FAQ
Repository: hive
Updated Branches:
   refs/heads/branch-1 fa6f5cf67 -> c5b6b8df2


HIVE-9139: Clean up GenSparkProcContext.clonedReduceSinks and related code [Spark Branch] (Chao Sun, reviewed by Xuefu Zhang)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/c5b6b8df
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/c5b6b8df
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/c5b6b8df

Branch: refs/heads/branch-1
Commit: c5b6b8df29bbd115f6640ccb1b3fedd665527270
Parents: fa6f5cf
Author: Chao Sun <sunchao@apache.org>
Authored: Mon Aug 17 10:40:53 2015 -0700
Committer: Xuefu Zhang <xzhang@cloudera.com>
Committed: Wed Nov 18 18:36:45 2015 -0800

----------------------------------------------------------------------
  .../hadoop/hive/ql/parse/spark/GenSparkProcContext.java | 2 --
  .../org/apache/hadoop/hive/ql/parse/spark/GenSparkWork.java | 7 -------
  2 files changed, 9 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/c5b6b8df/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/GenSparkProcContext.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/GenSparkProcContext.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/GenSparkProcContext.java
index 0a0c791..62237e1 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/GenSparkProcContext.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/GenSparkProcContext.java
@@ -129,7 +129,6 @@ public class GenSparkProcContext implements NodeProcessorCtx {
    public final Map<Operator<?>, BaseWork> unionWorkMap;
    public final List<UnionOperator> currentUnionOperators;
    public final Set<BaseWork> workWithUnionOperators;
- public final Set<ReduceSinkOperator> clonedReduceSinks;

    public final Set<FileSinkOperator> fileSinkSet;
    public final Map<FileSinkOperator, List<FileSinkOperator>> fileSinkMap;
@@ -180,7 +179,6 @@ public class GenSparkProcContext implements NodeProcessorCtx {
      this.unionWorkMap = new LinkedHashMap<Operator<?>, BaseWork>();
      this.currentUnionOperators = new LinkedList<UnionOperator>();
      this.workWithUnionOperators = new LinkedHashSet<BaseWork>();
- this.clonedReduceSinks = new LinkedHashSet<ReduceSinkOperator>();
      this.fileSinkSet = new LinkedHashSet<FileSinkOperator>();
      this.fileSinkMap = new LinkedHashMap<FileSinkOperator, List<FileSinkOperator>>();
      this.pruningSinkSet = new LinkedHashSet<Operator<?>>();

http://git-wip-us.apache.org/repos/asf/hive/blob/c5b6b8df/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/GenSparkWork.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/GenSparkWork.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/GenSparkWork.java
index 3dd6d92..2d5bb62 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/GenSparkWork.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/GenSparkWork.java
@@ -94,12 +94,6 @@ public class GenSparkWork implements NodeProcessor {
      LOG.debug("Root operator: " + root);
      LOG.debug("Leaf operator: " + operator);

- if (context.clonedReduceSinks.contains(operator)) {
- // if we're visiting a terminal we've created ourselves,
- // just skip and keep going
- return null;
- }
-
      SparkWork sparkWork = context.currentTask.getWork();
      SMBMapJoinOperator smbOp = GenSparkUtils.getChildOperator(root, SMBMapJoinOperator.class);

@@ -192,7 +186,6 @@ public class GenSparkWork implements NodeProcessor {
                    // we've already set this one up. Need to clone for the next work.
                    r = (ReduceSinkOperator) OperatorFactory.getAndMakeChild(
                        (ReduceSinkDesc)r.getConf().clone(), r.getParentOperators());
- context.clonedReduceSinks.add(r);
                  }
                  r.getConf().setOutputName(work.getName());
                }

Search Discussions

Related Discussions

Discussion Navigation
viewthread | post
Discussion Overview
groupcommits @
categorieshive, hadoop
postedNov 19, '15 at 2:37a
activeNov 19, '15 at 2:37a
posts1
users1
websitehive.apache.org

1 user in discussion

Xuefu: 1 post

People

Translate

site design / logo © 2021 Grokbase