FAQ
Repository: hive
Updated Branches:
   refs/heads/spark 725248174 -> a8c49ef41


HIVE-9139: Clean up GenSparkProcContext.clonedReduceSinks and related code [Spark Branch] (Chao Sun, reviewed by Xuefu Zhang)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/a8c49ef4
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/a8c49ef4
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/a8c49ef4

Branch: refs/heads/spark
Commit: a8c49ef419371cd827115f39a2a7b75c544e4eae
Parents: 7252481
Author: Chao Sun <sunchao@apache.org>
Authored: Mon Aug 17 10:40:53 2015 -0700
Committer: Chao Sun <sunchao@apache.org>
Committed: Mon Aug 17 10:40:53 2015 -0700

----------------------------------------------------------------------
  .../hadoop/hive/ql/parse/spark/GenSparkProcContext.java | 2 --
  .../org/apache/hadoop/hive/ql/parse/spark/GenSparkWork.java | 7 -------
  2 files changed, 9 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/a8c49ef4/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/GenSparkProcContext.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/GenSparkProcContext.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/GenSparkProcContext.java
index 0a0c791..62237e1 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/GenSparkProcContext.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/GenSparkProcContext.java
@@ -129,7 +129,6 @@ public class GenSparkProcContext implements NodeProcessorCtx {
    public final Map<Operator<?>, BaseWork> unionWorkMap;
    public final List<UnionOperator> currentUnionOperators;
    public final Set<BaseWork> workWithUnionOperators;
- public final Set<ReduceSinkOperator> clonedReduceSinks;

    public final Set<FileSinkOperator> fileSinkSet;
    public final Map<FileSinkOperator, List<FileSinkOperator>> fileSinkMap;
@@ -180,7 +179,6 @@ public class GenSparkProcContext implements NodeProcessorCtx {
      this.unionWorkMap = new LinkedHashMap<Operator<?>, BaseWork>();
      this.currentUnionOperators = new LinkedList<UnionOperator>();
      this.workWithUnionOperators = new LinkedHashSet<BaseWork>();
- this.clonedReduceSinks = new LinkedHashSet<ReduceSinkOperator>();
      this.fileSinkSet = new LinkedHashSet<FileSinkOperator>();
      this.fileSinkMap = new LinkedHashMap<FileSinkOperator, List<FileSinkOperator>>();
      this.pruningSinkSet = new LinkedHashSet<Operator<?>>();

http://git-wip-us.apache.org/repos/asf/hive/blob/a8c49ef4/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/GenSparkWork.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/GenSparkWork.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/GenSparkWork.java
index 3dd6d92..2d5bb62 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/GenSparkWork.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/GenSparkWork.java
@@ -94,12 +94,6 @@ public class GenSparkWork implements NodeProcessor {
      LOG.debug("Root operator: " + root);
      LOG.debug("Leaf operator: " + operator);

- if (context.clonedReduceSinks.contains(operator)) {
- // if we're visiting a terminal we've created ourselves,
- // just skip and keep going
- return null;
- }
-
      SparkWork sparkWork = context.currentTask.getWork();
      SMBMapJoinOperator smbOp = GenSparkUtils.getChildOperator(root, SMBMapJoinOperator.class);

@@ -192,7 +186,6 @@ public class GenSparkWork implements NodeProcessor {
                    // we've already set this one up. Need to clone for the next work.
                    r = (ReduceSinkOperator) OperatorFactory.getAndMakeChild(
                        (ReduceSinkDesc)r.getConf().clone(), r.getParentOperators());
- context.clonedReduceSinks.add(r);
                  }
                  r.getConf().setOutputName(work.getName());
                }

Search Discussions

Related Discussions

Discussion Navigation
viewthread | post
Discussion Overview
groupcommits @
categorieshive, hadoop
postedAug 17, '15 at 5:41p
activeAug 17, '15 at 5:41p
posts1
users1
websitehive.apache.org

1 user in discussion

Sunchao: 1 post

People

Translate

site design / logo © 2021 Grokbase