FAQ
Author: xuefu
Date: Fri Dec 19 21:41:36 2014
New Revision: 1646883

URL: http://svn.apache.org/r1646883
Log:
HIVE-9164: Profile query compiler #2 [Spark Branch] (Chao via Xuefu)

Modified:
     hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/log/PerfLogger.java
     hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/SparkCompiler.java

Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/log/PerfLogger.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/log/PerfLogger.java?rev=1646883&r1=1646882&r2=1646883&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/log/PerfLogger.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/log/PerfLogger.java Fri Dec 19 21:41:36 2014
@@ -74,6 +74,8 @@ public class PerfLogger {
    public static final String SPARK_RUN_STAGE = "SparkRunStage.";
    public static final String SPARK_INIT_OPERATORS = "SparkInitializeOperators";
    public static final String SPARK_GENERATE_TASK_TREE = "SparkGenerateTaskTree";
+ public static final String SPARK_OPTIMIZE_OPERATOR_TREE = "SparkOptimizeOperatorTree";
+ public static final String SPARK_OPTIMIZE_TASK_TREE = "SparkOptimizeTaskTree";
    public static final String SPARK_FLUSH_HASHTABLE = "SparkFlushHashTable.";

    protected static final ThreadLocal<PerfLogger> perfLogger = new ThreadLocal<PerfLogger>();

Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/SparkCompiler.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/SparkCompiler.java?rev=1646883&r1=1646882&r2=1646883&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/SparkCompiler.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/SparkCompiler.java Fri Dec 19 21:41:36 2014
@@ -110,6 +110,7 @@ public class SparkCompiler extends TaskC
    @Override
    protected void optimizeOperatorPlan(ParseContext pCtx, Set<ReadEntity> inputs,
        Set<WriteEntity> outputs) throws SemanticException {
+ perfLogger.PerfLogBegin(CLASS_NAME, PerfLogger.SPARK_OPTIMIZE_OPERATOR_TREE);
      // Sequence of TableScan operators to be walked
      Deque<Operator<? extends OperatorDesc>> deque = new LinkedList<Operator<? extends OperatorDesc>>();
      deque.addAll(pCtx.getTopOps().values());
@@ -134,6 +135,7 @@ public class SparkCompiler extends TaskC
      ArrayList<Node> topNodes = new ArrayList<Node>();
      topNodes.addAll(pCtx.getTopOps().values());
      ogw.startWalking(topNodes, null);
+ perfLogger.PerfLogEnd(CLASS_NAME, PerfLogger.SPARK_OPTIMIZE_OPERATOR_TREE);
    }

    /**
@@ -297,6 +299,7 @@ public class SparkCompiler extends TaskC
    @Override
    protected void optimizeTaskPlan(List<Task<? extends Serializable>> rootTasks, ParseContext pCtx,
        Context ctx) throws SemanticException {
+ perfLogger.PerfLogBegin(CLASS_NAME, PerfLogger.SPARK_OPTIMIZE_TASK_TREE);
      PhysicalContext physicalCtx = new PhysicalContext(conf, pCtx, pCtx.getContext(), rootTasks,
         pCtx.getFetchTask());

@@ -339,6 +342,8 @@ public class SparkCompiler extends TaskC
      } else {
        LOG.debug("Skipping stage id rearranger");
      }
+
+ perfLogger.PerfLogEnd(CLASS_NAME, PerfLogger.SPARK_OPTIMIZE_TASK_TREE);
      return;
    }
  }

Search Discussions

Related Discussions

Discussion Navigation
viewthread | post
Discussion Overview
groupcommits @
categorieshive, hadoop
postedDec 19, '14 at 9:41p
activeDec 19, '14 at 9:41p
posts1
users1
websitehive.apache.org

1 user in discussion

Xuefu: 1 post

People

Translate

site design / logo © 2021 Grokbase