Grokbase Groups Hive commits May 2016
FAQ
Repository: hive
Updated Branches:
   refs/heads/master e3b6c9b1e -> e087ba845


HIVE-13342: Improve logging in llap decider and throw exception in case llap mode is all but we cannot run in llap. (Vikram Dixit K, reviewed by Sergey Shelukhin and Gunther Hagleitner)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/e087ba84
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/e087ba84
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/e087ba84

Branch: refs/heads/master
Commit: e087ba84537555eb37b0047e7ace3f1b9dc06d08
Parents: e3b6c9b
Author: vikram <vikram@hortonworks.com>
Authored: Tue May 10 14:01:30 2016 -0700
Committer: vikram <vikram@hortonworks.com>
Committed: Tue May 10 14:03:29 2016 -0700

----------------------------------------------------------------------
  .../hive/ql/optimizer/physical/LlapDecider.java | 37 +++++++++++++++-----
  ql/src/test/queries/clientpositive/llap_udf.q | 5 +++
  .../results/clientpositive/llap/llap_udf.q.out | 2 --
  3 files changed, 34 insertions(+), 10 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/e087ba84/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/LlapDecider.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/LlapDecider.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/LlapDecider.java
index c0b7a32..e6f8f75 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/LlapDecider.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/LlapDecider.java
@@ -132,6 +132,12 @@ public class LlapDecider implements PhysicalPlanResolver {
        throws SemanticException {
        if (evaluateWork(tezWork, work)) {
          convertWork(tezWork, work);
+ } else {
+ if (mode == all) {
+ throw new SemanticException("Llap mode is set to all but cannot run work in llap mode." +
+ "Set " + HiveConf.ConfVars.LLAP_EXECUTION_MODE + " = auto or set " +
+ HiveConf.ConfVars.HIVE_EXECUTION_MODE + " = container");
+ }
        }
      }

@@ -228,6 +234,7 @@ public class LlapDecider implements PhysicalPlanResolver {
        }

        // couldn't convince you otherwise? well then let's llap.
+ LOG.info("Can run work " + work.getName() + " in llap mode.");
        return true;
      }

@@ -304,6 +311,7 @@ public class LlapDecider implements PhysicalPlanResolver {
            @Override
            public Object process(Node n, Stack<Node> s, NodeProcessorCtx c,
                Object... os) {
+ LOG.debug("Cannot run operator [" + n + "] in llap mode.");
              return new Boolean(false);
            }
          });
@@ -313,7 +321,11 @@ public class LlapDecider implements PhysicalPlanResolver {
            public Object process(Node n, Stack<Node> s, NodeProcessorCtx c,
                Object... os) {
              ExprNodeDesc expr = ((FilterOperator)n).getConf().getPredicate();
- return new Boolean(checkExpression(expr));
+ Boolean retval = new Boolean(checkExpression(expr));
+ if (!retval) {
+ LOG.info("Cannot run filter operator [" + n + "] in llap mode");
+ }
+ return new Boolean(retval);
            }
          });
        opRules.put(new RuleRegExp("No user code in gby", GroupByOperator.getOperatorName() + "%"),
@@ -321,9 +333,13 @@ public class LlapDecider implements PhysicalPlanResolver {
            @Override
            public Object process(Node n, Stack<Node> s, NodeProcessorCtx c,
                Object... os) {
- @SuppressWarnings("unchecked")
- List<AggregationDesc> aggs = ((Operator<GroupByDesc>) n).getConf().getAggregators();
- return new Boolean(checkAggregators(aggs));
+ @SuppressWarnings("unchecked")
+ List<AggregationDesc> aggs = ((Operator<GroupByDesc>) n).getConf().getAggregators();
+ Boolean retval = new Boolean(checkAggregators(aggs));
+ if (!retval) {
+ LOG.info("Cannot run group by operator [" + n + "] in llap mode");
+ }
+ return new Boolean(retval);
            }
          });
        opRules.put(new RuleRegExp("No user code in select", SelectOperator.getOperatorName() + "%"),
@@ -331,9 +347,13 @@ public class LlapDecider implements PhysicalPlanResolver {
            @Override
            public Object process(Node n, Stack<Node> s, NodeProcessorCtx c,
                Object... os) {
- @SuppressWarnings({ "unchecked" })
- List<ExprNodeDesc> exprs = ((Operator<SelectDesc>) n).getConf().getColList();
- return new Boolean(checkExpressions(exprs));
+ @SuppressWarnings({ "unchecked" })
+ List<ExprNodeDesc> exprs = ((Operator<SelectDesc>) n).getConf().getColList();
+ Boolean retval = new Boolean(checkExpressions(exprs));
+ if (!retval) {
+ LOG.info("Cannot run select operator [" + n + "] in llap mode");
+ }
+ return new Boolean(retval);
            }
          });

@@ -355,6 +375,7 @@ public class LlapDecider implements PhysicalPlanResolver {
        for (Node n : nodeOutput.keySet()) {
          if (nodeOutput.get(n) != null) {
            if (!((Boolean)nodeOutput.get(n))) {
+ LOG.info("Cannot run in LLAP mode.");
              return false;
            }
          }
@@ -379,7 +400,7 @@ public class LlapDecider implements PhysicalPlanResolver {
            Arrays.asList(pd.getInputFileFormatClass().getInterfaces());
          if (!interfaceList.contains(VectorizedInputFormatInterface.class)) {
            LOG.info("Input format: " + pd.getInputFileFormatClassName()
- + ", doesn't provide vectorized input");
+ + ", doesn't provide vectorized input");
            return false;
          }
        }

http://git-wip-us.apache.org/repos/asf/hive/blob/e087ba84/ql/src/test/queries/clientpositive/llap_udf.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/llap_udf.q b/ql/src/test/queries/clientpositive/llap_udf.q
index 2224bd5..8b1dcb7 100644
--- a/ql/src/test/queries/clientpositive/llap_udf.q
+++ b/ql/src/test/queries/clientpositive/llap_udf.q
@@ -12,8 +12,10 @@ create table src_orc stored as orc as select * from src;

  CREATE TEMPORARY FUNCTION test_udf0 AS 'org.apache.hadoop.hive.ql.udf.generic.GenericUDFEvaluateNPE';

+set hive.llap.execution.mode=auto;
  EXPLAIN SELECT test_udf0(cast(key as string)) from src_orc;

+set hive.llap.execution.mode=all;
  CREATE FUNCTION test_udf2 AS 'org.apache.hadoop.hive.ql.udf.generic.GenericUDFTestGetJavaString';
  CREATE FUNCTION test_udf3 AS 'org.apache.hadoop.hive.ql.udf.generic.GenericUDFTestGetJavaString';
  CREATE FUNCTION test_udf4 AS 'org.apache.hadoop.hive.ql.udf.generic.GenericUDFEvaluateNPE';
@@ -21,18 +23,21 @@ CREATE FUNCTION test_udf4 AS 'org.apache.hadoop.hive.ql.udf.generic.GenericUDFEv
  EXPLAIN
  SELECT test_udf2(cast(key as string)), test_udf3(cast(key as string)), test_udf4(cast(key as string)) from src_orc;

+set hive.llap.execution.mode=auto;
  -- Verification is based on classes, so 0 would work based on 4.
  EXPLAIN
  SELECT test_udf0(cast(key as string)) from src_orc;

  DROP FUNCTION test_udf2;

+set hive.llap.execution.mode=all;
  -- ...verify that 3 still works
  EXPLAIN
  SELECT test_udf3(cast(key as string)), test_udf4(cast(key as string)) from src_orc;

  DROP FUNCTION test_udf4;

+set hive.llap.execution.mode=auto;
  -- ...now 0 should stop working
  EXPLAIN
  SELECT test_udf0(cast(key as string)) from src_orc;

http://git-wip-us.apache.org/repos/asf/hive/blob/e087ba84/ql/src/test/results/clientpositive/llap/llap_udf.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/llap_udf.q.out b/ql/src/test/results/clientpositive/llap/llap_udf.q.out
index 6197cf0..e517942 100644
--- a/ql/src/test/results/clientpositive/llap/llap_udf.q.out
+++ b/ql/src/test/results/clientpositive/llap/llap_udf.q.out
@@ -157,8 +157,6 @@ STAGE PLANS:
                            input format: org.apache.hadoop.mapred.SequenceFileInputFormat
                            output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
                            serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- Execution mode: llap
- LLAP IO: no inputs

    Stage: Stage-0
      Fetch Operator

Search Discussions

Related Discussions

Discussion Navigation
viewthread | post
Discussion Overview
groupcommits @
categorieshive, hadoop
postedMay 10, '16 at 9:04p
activeMay 10, '16 at 9:04p
posts1
users1
websitehive.apache.org

1 user in discussion

Vikram: 1 post

People

Translate

site design / logo © 2021 Grokbase