FAQ
Repository: hive
Updated Branches:
   refs/heads/master 0ef92b2c7 -> a2b7b4d9d


HIVE-13407 : Add more subtlety to TezCompiler Perf Logging (Hari Sankar Subramaniyan via Ashutosh Chauhan)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/a2b7b4d9
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/a2b7b4d9
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/a2b7b4d9

Branch: refs/heads/master
Commit: a2b7b4d9dca82aa599b4b15fa7927f1d6403da79
Parents: 0ef92b2
Author: Hari Subramaniyan <harisankar@apache.org>
Authored: Thu Apr 21 13:57:00 2016 -0800
Committer: Ashutosh Chauhan <hashutosh@apache.org>
Committed: Fri Apr 22 17:57:20 2016 -0700

----------------------------------------------------------------------
  .../apache/hadoop/hive/ql/log/PerfLogger.java | 1 +
  .../apache/hadoop/hive/ql/parse/TezCompiler.java | 19 ++++++++++++++-----
  2 files changed, 15 insertions(+), 5 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/a2b7b4d9/common/src/java/org/apache/hadoop/hive/ql/log/PerfLogger.java
----------------------------------------------------------------------
diff --git a/common/src/java/org/apache/hadoop/hive/ql/log/PerfLogger.java b/common/src/java/org/apache/hadoop/hive/ql/log/PerfLogger.java
index 6f9347c..87b82f8 100644
--- a/common/src/java/org/apache/hadoop/hive/ql/log/PerfLogger.java
+++ b/common/src/java/org/apache/hadoop/hive/ql/log/PerfLogger.java
@@ -63,6 +63,7 @@ public class PerfLogger {
    public static final String FAILURE_HOOK = "FailureHook.";
    public static final String DRIVER_RUN = "Driver.run";
    public static final String TIME_TO_SUBMIT = "TimeToSubmit";
+ public static final String TEZ_COMPILER = "TezCompiler";
    public static final String TEZ_SUBMIT_TO_RUNNING = "TezSubmitToRunningDag";
    public static final String TEZ_BUILD_DAG = "TezBuildDag";
    public static final String TEZ_SUBMIT_DAG = "TezSubmitDag";

http://git-wip-us.apache.org/repos/asf/hive/blob/a2b7b4d9/ql/src/java/org/apache/hadoop/hive/ql/parse/TezCompiler.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/TezCompiler.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/TezCompiler.java
index ff971ac..8e67ce8 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/TezCompiler.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/TezCompiler.java
@@ -115,23 +115,28 @@ public class TezCompiler extends TaskCompiler {
    protected void optimizeOperatorPlan(ParseContext pCtx, Set<ReadEntity> inputs,
        Set<WriteEntity> outputs) throws SemanticException {
      PerfLogger perfLogger = SessionState.getPerfLogger();
- perfLogger.PerfLogBegin(this.getClass().getName(), PerfLogger.OPTIMIZER);
      // Create the context for the walker
      OptimizeTezProcContext procCtx = new OptimizeTezProcContext(conf, pCtx, inputs, outputs);
-
+ perfLogger.PerfLogBegin(this.getClass().getName(), PerfLogger.TEZ_COMPILER);
      // setup dynamic partition pruning where possible
      runDynamicPartitionPruning(procCtx, inputs, outputs);
+ perfLogger.PerfLogEnd(this.getClass().getName(), PerfLogger.TEZ_COMPILER, "Setup dynamic partition pruning");

+ perfLogger.PerfLogBegin(this.getClass().getName(), PerfLogger.TEZ_COMPILER);
      // setup stats in the operator plan
      runStatsAnnotation(procCtx);
+ perfLogger.PerfLogEnd(this.getClass().getName(), PerfLogger.TEZ_COMPILER, "Setup stats in the operator plan");

+ perfLogger.PerfLogBegin(this.getClass().getName(), PerfLogger.TEZ_COMPILER);
      // run the optimizations that use stats for optimization
      runStatsDependentOptimizations(procCtx, inputs, outputs);
+ perfLogger.PerfLogEnd(this.getClass().getName(), PerfLogger.TEZ_COMPILER, "Run the optimizations that use stats for optimization");

+ perfLogger.PerfLogBegin(this.getClass().getName(), PerfLogger.TEZ_COMPILER);
      // after the stats phase we might have some cyclic dependencies that we need
      // to take care of.
      runCycleAnalysisForPartitionPruning(procCtx, inputs, outputs);
- perfLogger.PerfLogEnd(this.getClass().getName(), PerfLogger.OPTIMIZER, "Tez compiler");
+ perfLogger.PerfLogEnd(this.getClass().getName(), PerfLogger.TEZ_COMPILER, "Run cycle analysis for partition pruning");

    }

@@ -323,7 +328,8 @@ public class TezCompiler extends TaskCompiler {
        List<Task<MoveWork>> mvTask, Set<ReadEntity> inputs, Set<WriteEntity> outputs)
        throws SemanticException {

-
+ PerfLogger perfLogger = SessionState.getPerfLogger();
+ perfLogger.PerfLogBegin(this.getClass().getName(), PerfLogger.TEZ_COMPILER);
      ParseContext tempParseContext = getParseContext(pCtx, rootTasks);
      GenTezUtils utils = new GenTezUtils();
      GenTezWork genTezWork = new GenTezWork(utils);
@@ -389,6 +395,7 @@ public class TezCompiler extends TaskCompiler {
        LOG.debug("Handling AppMasterEventOperator: " + event);
        GenTezUtils.processAppMasterEvent(procCtx, event);
      }
+ perfLogger.PerfLogEnd(this.getClass().getName(), PerfLogger.TEZ_COMPILER, "generateTaskTree");
    }

    @Override
@@ -449,6 +456,8 @@ public class TezCompiler extends TaskCompiler {
    @Override
    protected void optimizeTaskPlan(List<Task<? extends Serializable>> rootTasks, ParseContext pCtx,
        Context ctx) throws SemanticException {
+ PerfLogger perfLogger = SessionState.getPerfLogger();
+ perfLogger.PerfLogBegin(this.getClass().getName(), PerfLogger.TEZ_COMPILER);
      PhysicalContext physicalCtx = new PhysicalContext(conf, pCtx, pCtx.getContext(), rootTasks,
         pCtx.getFetchTask());

@@ -498,7 +507,7 @@ public class TezCompiler extends TaskCompiler {
      // the backend. If you have a physical optimization that changes
      // table scans or filters, you have to invoke it before this one.
      physicalCtx = new SerializeFilter().resolve(physicalCtx);
-
+ perfLogger.PerfLogEnd(this.getClass().getName(), PerfLogger.TEZ_COMPILER, "optimizeTaskPlan");
      return;
    }
  }

Search Discussions

Related Discussions

Discussion Navigation
viewthread | post
posts ‹ prev | 1 of 1 | next ›
Discussion Overview
groupcommits @
categorieshive, hadoop
postedApr 23, '16 at 12:58a
activeApr 23, '16 at 12:58a
posts1
users1
websitehive.apache.org

1 user in discussion

Hashutosh: 1 post

People

Translate

site design / logo © 2021 Grokbase