FAQ
Repository: hive
Updated Branches:
   refs/heads/master be0000608 -> e44198f5e


HIVE-12857 : LLAP: modify the decider to allow using LLAP with whitelisted UDFs (Sergey Shelukhin, reviewed by Gunther Hagleitner)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/e44198f5
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/e44198f5
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/e44198f5

Branch: refs/heads/master
Commit: e44198f5e6b2b5a83c34d70bd12bb4e772f737dd
Parents: be00006
Author: Sergey Shelukhin <sershe@apache.org>
Authored: Thu Feb 25 19:41:56 2016 -0800
Committer: Sergey Shelukhin <sershe@apache.org>
Committed: Thu Feb 25 19:41:56 2016 -0800

----------------------------------------------------------------------
  itests/qtest/pom.xml | 4 +-
  .../test/resources/testconfiguration.properties | 6 +-
  .../hadoop/hive/ql/exec/FunctionRegistry.java | 15 +
  .../hadoop/hive/ql/exec/FunctionTask.java | 10 +-
  .../apache/hadoop/hive/ql/exec/Registry.java | 44 +++
  .../hive/ql/optimizer/physical/LlapDecider.java | 50 +--
  ql/src/test/queries/clientpositive/llap_udf.q | 47 +++
  .../results/clientpositive/llap/llap_udf.q.out | 307 +++++++++++++++++++
  8 files changed, 458 insertions(+), 25 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/e44198f5/itests/qtest/pom.xml
----------------------------------------------------------------------
diff --git a/itests/qtest/pom.xml b/itests/qtest/pom.xml
index 292e562..cb68e39 100644
--- a/itests/qtest/pom.xml
+++ b/itests/qtest/pom.xml
@@ -431,7 +431,7 @@
                    templatePath="${basedir}/${hive.path.to.root}/ql/src/test/templates/" template="TestCliDriver.vm"
                    queryDirectory="${basedir}/${hive.path.to.root}/ql/src/test/queries/clientpositive/"
                    queryFile="${qfile}"
- excludeQueryFile="${minimr.query.files},${minitez.query.files},${encrypted.query.files},${spark.only.query.files},${disabled.query.files}"
+ excludeQueryFile="${minillap.query.files},${minimr.query.files},${minitez.query.files},${encrypted.query.files},${spark.only.query.files},${disabled.query.files}"
                    queryFileRegex="${qfile_regex}"
                    clusterMode="${clustermode}"
                    runDisabled="${run_disabled}"
@@ -535,7 +535,7 @@
                    templatePath="${basedir}/${hive.path.to.root}/ql/src/test/templates/" template="TestCliDriver.vm"
                    queryDirectory="${basedir}/${hive.path.to.root}/ql/src/test/queries/clientpositive/"
                    queryFile="${qfile}"
- includeQueryFile="${minillap.query.files}"
+ includeQueryFile="${minillap.query.files},${minillap.shared.query.files}"
                    queryFileRegex="${qfile_regex}"
                    clusterMode="llap"
                    runDisabled="${run_disabled}"

http://git-wip-us.apache.org/repos/asf/hive/blob/e44198f5/itests/src/test/resources/testconfiguration.properties
----------------------------------------------------------------------
diff --git a/itests/src/test/resources/testconfiguration.properties b/itests/src/test/resources/testconfiguration.properties
index 58d0a45..68dbd0c 100644
--- a/itests/src/test/resources/testconfiguration.properties
+++ b/itests/src/test/resources/testconfiguration.properties
@@ -445,7 +445,7 @@ minitez.query.files=bucket_map_join_tez1.q,\



-minillap.query.files=bucket_map_join_tez1.q,\
+minillap.shared.query.files=bucket_map_join_tez1.q,\
    bucket_map_join_tez2.q,\
    constprog_dpp.q,\
    cte_1.q,\
@@ -496,6 +496,10 @@ minillap.query.files=bucket_map_join_tez1.q,\
    tez_join.q,\
    tez_union_multiinsert.q

+
+minillap.query.files=llap_udf.q
+
+
  encrypted.query.files=encryption_join_unencrypted_tbl.q,\
    encryption_insert_partition_static.q,\
    encryption_insert_partition_dynamic.q,\

http://git-wip-us.apache.org/repos/asf/hive/blob/e44198f5/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java
index e0e030f..d1e1441 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java
@@ -1480,6 +1480,21 @@ public final class FunctionRegistry {
      return system.registerPermanentFunction(functionName, className, registerToSession, resources);
    }

+ public static boolean isPermanentFunction(ExprNodeGenericFuncDesc fnExpr) {
+ GenericUDF udf = fnExpr.getGenericUDF();
+ if (udf == null) return false;
+
+ Class<?> clazz = udf.getClass();
+ if (udf instanceof GenericUDFBridge) {
+ clazz = ((GenericUDFBridge)udf).getUdfClass();
+ }
+
+ if (clazz != null) {
+ return system.isPermanentFunc(clazz);
+ }
+ return false;
+ }
+
    public static void unregisterPermanentFunction(String functionName) throws HiveException {
      system.unregisterFunction(functionName);
      unregisterTemporaryUDF(functionName);

http://git-wip-us.apache.org/repos/asf/hive/blob/e44198f5/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionTask.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionTask.java
index 77f11b9..1b971fc 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionTask.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionTask.java
@@ -136,8 +136,16 @@ public class FunctionTask extends Task<FunctionWork> {
      // For permanent functions, check for any resources from local filesystem.
      checkLocalFunctionResources(db, createFunctionDesc.getResources());

- FunctionInfo registered = FunctionRegistry.registerPermanentFunction(
+ FunctionInfo registered = null;
+ try {
+ registered = FunctionRegistry.registerPermanentFunction(
          registeredName, className, true, toFunctionResource(resources));
+ } catch (RuntimeException ex) {
+ Throwable t = ex;
+ while (t.getCause() != null) {
+ t = t.getCause();
+ }
+ }
      if (registered == null) {
        console.printError("Failed to register " + registeredName
            + " using class " + createFunctionDesc.getClassName());

http://git-wip-us.apache.org/repos/asf/hive/blob/e44198f5/ql/src/java/org/apache/hadoop/hive/ql/exec/Registry.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/Registry.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/Registry.java
index 16c8c8f..d5f4a37 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/Registry.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/Registry.java
@@ -55,6 +55,7 @@ import java.util.List;
  import java.util.Map;
  import java.util.Set;
  import java.util.TreeSet;
+import java.util.concurrent.ConcurrentHashMap;
  import java.util.regex.Pattern;
  import java.util.regex.PatternSyntaxException;

@@ -71,6 +72,11 @@ public class Registry {
     */
    private final Map<String, FunctionInfo> mFunctions = new LinkedHashMap<String, FunctionInfo>();
    private final Set<Class<?>> builtIns = Collections.synchronizedSet(new HashSet<Class<?>>());
+ /**
+ * Persistent map contains refcounts that are only modified in synchronized methods for now,
+ * so there's no separate effort to make refcount operations thread-safe.
+ */
+ private final Map<Class<?>, Integer> persistent = new ConcurrentHashMap<>();
    private final Set<ClassLoader> mSessionUDFLoaders = new LinkedHashSet<ClassLoader>();

    private final boolean isNative;
@@ -285,6 +291,10 @@ public class Registry {
      return udfClass != null && builtIns.contains(udfClass);
    }

+ public boolean isPermanentFunc(Class<?> udfClass) {
+ return udfClass != null && persistent.containsKey(udfClass);
+ }
+
    public synchronized Set<String> getCurrentFunctionNames() {
      return getFunctionNames((Pattern)null);
    }
@@ -415,9 +425,27 @@ public class Registry {
      mFunctions.put(functionName, function);
      if (function.isBuiltIn()) {
        builtIns.add(function.getFunctionClass());
+ } else if (function.isPersistent()) {
+ Class<?> functionClass = getPermanentUdfClass(function);
+ Integer refCount = persistent.get(functionClass);
+ persistent.put(functionClass, Integer.valueOf(refCount == null ? 1 : refCount + 1));
      }
    }

+ private Class<?> getPermanentUdfClass(FunctionInfo function) {
+ Class<?> functionClass = function.getFunctionClass();
+ if (functionClass == null) {
+ // Expected for permanent UDFs at this point.
+ ClassLoader loader = Utilities.getSessionSpecifiedClassLoader();
+ try {
+ functionClass = Class.forName(function.getClassName(), true, loader);
+ } catch (ClassNotFoundException ex) {
+ throw new RuntimeException(ex);
+ }
+ }
+ return functionClass;
+ }
+
    public synchronized void unregisterFunction(String functionName) throws HiveException {
      functionName = functionName.toLowerCase();
      FunctionInfo fi = mFunctions.get(functionName);
@@ -427,6 +455,21 @@ public class Registry {
        }
        mFunctions.remove(functionName);
        fi.discarded();
+ if (fi.isPersistent()) {
+ removePersistentFunction(fi);
+ }
+ }
+ }
+
+ /** Should only be called from synchronized methods. */
+ private void removePersistentFunction(FunctionInfo fi) {
+ Class<?> functionClass = getPermanentUdfClass(fi);
+ Integer refCount = persistent.get(functionClass);
+ assert refCount != null;
+ if (refCount == 1) {
+ persistent.remove(functionClass);
+ } else {
+ persistent.put(functionClass, Integer.valueOf(refCount - 1));
      }
    }

@@ -517,6 +560,7 @@ public class Registry {
      }
      mFunctions.clear();
      builtIns.clear();
+ persistent.clear();
    }

    public synchronized void closeCUDFLoaders() {

http://git-wip-us.apache.org/repos/asf/hive/blob/e44198f5/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/LlapDecider.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/LlapDecider.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/LlapDecider.java
index f2998af..80dbd31 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/LlapDecider.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/LlapDecider.java
@@ -101,15 +101,18 @@ public class LlapDecider implements PhysicalPlanResolver {
    class LlapDecisionDispatcher implements Dispatcher {
      private final HiveConf conf;
      private final boolean doSkipUdfCheck;
+ private final boolean arePermanentFnsAllowed;

      public LlapDecisionDispatcher(PhysicalContext pctx) {
        conf = pctx.getConf();
        doSkipUdfCheck = HiveConf.getBoolVar(conf, ConfVars.LLAP_SKIP_COMPILE_UDF_CHECK);
+ arePermanentFnsAllowed = HiveConf.getBoolVar(conf, ConfVars.LLAP_DAEMON_ALLOW_PERMANENT_FNS);
      }

      @Override
      public Object dispatch(Node nd, Stack<Node> stack, Object... nodeOutputs)
        throws SemanticException {
+ @SuppressWarnings("unchecked")
        Task<? extends Serializable> currTask = (Task<? extends Serializable>) nd;
        if (currTask instanceof TezTask) {
          TezWork work = ((TezTask) currTask).getWork();
@@ -237,10 +240,20 @@ public class LlapDecider implements PhysicalPlanResolver {
            exprs.addAll(cur.getChildren());
          }

- if (!doSkipUdfCheck && cur instanceof ExprNodeGenericFuncDesc
- && !FunctionRegistry.isBuiltInFuncExpr((ExprNodeGenericFuncDesc)cur)) {
- LOG.info("Not a built-in function: " + cur.getExprString());
- return false;
+ if (!doSkipUdfCheck && cur instanceof ExprNodeGenericFuncDesc) {
+ ExprNodeGenericFuncDesc funcDesc = (ExprNodeGenericFuncDesc)cur;
+ boolean isBuiltIn = FunctionRegistry.isBuiltInFuncExpr(funcDesc);
+ if (!isBuiltIn) {
+ if (!arePermanentFnsAllowed) {
+ LOG.info("Not a built-in function: " + cur.getExprString()
+ + " (permanent functions are disabled)");
+ return false;
+ }
+ if (!FunctionRegistry.isPermanentFunction(funcDesc)) {
+ LOG.info("Not a built-in or permanent function: " + cur.getExprString());
+ return false;
+ }
+ }
          }
        }
        return true;
@@ -261,24 +274,22 @@ public class LlapDecider implements PhysicalPlanResolver {
      }

      private boolean checkExpressions(Collection<ExprNodeDesc> exprs) {
- boolean result = true;
- for (ExprNodeDesc expr: exprs) {
- result = result && checkExpression(expr);
+ for (ExprNodeDesc expr : exprs) {
+ if (!checkExpression(expr)) return false;
        }
- return result;
+ return true;
      }

      private boolean checkAggregators(Collection<AggregationDesc> aggs) {
- boolean result = true;
        try {
- for (AggregationDesc agg: aggs) {
- result = result && checkAggregator(agg);
- }
+ for (AggregationDesc agg: aggs) {
+ if (!checkAggregator(agg)) return false;
+ }
        } catch (SemanticException e) {
- LOG.warn("Exception testing aggregators.",e);
- result = false;
+ LOG.warn("Exception testing aggregators.",e);
+ return false;
        }
- return result;
+ return true;
      }

      private Map<Rule, NodeProcessor> getRules() {
@@ -291,8 +302,7 @@ public class LlapDecider implements PhysicalPlanResolver {
              return new Boolean(false);
            }
          });
- opRules.put(new RuleRegExp("No user code in fil",
- FilterOperator.getOperatorName() + "%"),
+ opRules.put(new RuleRegExp("No user code in fil", FilterOperator.getOperatorName() + "%"),
            new NodeProcessor() {
            @Override
            public Object process(Node n, Stack<Node> s, NodeProcessorCtx c,
@@ -301,8 +311,7 @@ public class LlapDecider implements PhysicalPlanResolver {
              return new Boolean(checkExpression(expr));
            }
          });
- opRules.put(new RuleRegExp("No user code in gby",
- GroupByOperator.getOperatorName() + "%"),
+ opRules.put(new RuleRegExp("No user code in gby", GroupByOperator.getOperatorName() + "%"),
            new NodeProcessor() {
            @Override
            public Object process(Node n, Stack<Node> s, NodeProcessorCtx c,
@@ -311,8 +320,7 @@ public class LlapDecider implements PhysicalPlanResolver {
              return new Boolean(checkAggregators(aggs));
            }
          });
- opRules.put(new RuleRegExp("No user code in select",
- SelectOperator.getOperatorName() + "%"),
+ opRules.put(new RuleRegExp("No user code in select", SelectOperator.getOperatorName() + "%"),
            new NodeProcessor() {
            @Override
            public Object process(Node n, Stack<Node> s, NodeProcessorCtx c,

http://git-wip-us.apache.org/repos/asf/hive/blob/e44198f5/ql/src/test/queries/clientpositive/llap_udf.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/llap_udf.q b/ql/src/test/queries/clientpositive/llap_udf.q
new file mode 100644
index 0000000..c964f2b
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/llap_udf.q
@@ -0,0 +1,47 @@
+set hive.mapred.mode=nonstrict;
+set hive.explain.user=false;
+set hive.execution.mode=llap;
+set hive.llap.execution.mode=all;
+set hive.fetch.task.conversion=none;
+set hive.llap.daemon.allow.permanent.fns=true;
+
+drop table if exists src_orc;
+create table src_orc stored as orc as select * from src;
+
+-- Not using GenericUDFTestGetJavaBoolean; that is already registered when tests begin
+
+CREATE TEMPORARY FUNCTION test_udf0 AS 'org.apache.hadoop.hive.ql.udf.generic.GenericUDFEvaluateNPE';
+
+EXPLAIN SELECT test_udf0(cast(key as string)) from src_orc;
+
+CREATE FUNCTION test_udf2 AS 'org.apache.hadoop.hive.ql.udf.generic.GenericUDFTestGetJavaString';
+CREATE FUNCTION test_udf3 AS 'org.apache.hadoop.hive.ql.udf.generic.GenericUDFTestGetJavaString';
+CREATE FUNCTION test_udf4 AS 'org.apache.hadoop.hive.ql.udf.generic.GenericUDFEvaluateNPE';
+
+EXPLAIN
+SELECT test_udf2(cast(key as string)), test_udf3(cast(key as string)), test_udf4(cast(key as string)) from src_orc;
+
+-- Verification is based on classes, so 0 would work based on 4.
+EXPLAIN
+SELECT test_udf0(cast(key as string)) from src_orc;
+
+DROP FUNCTION test_udf2;
+
+-- ...verify that 3 still works
+EXPLAIN
+SELECT test_udf3(cast(key as string)), test_udf4(cast(key as string)) from src_orc;
+
+DROP FUNCTION test_udf4;
+
+-- ...now 0 should stop working
+EXPLAIN
+SELECT test_udf0(cast(key as string)) from src_orc;
+
+set hive.llap.daemon.allow.permanent.fns=false;
+
+EXPLAIN
+SELECT test_udf3(cast(key as string)) from src_orc;
+
+
+drop table if exists src_orc;
+set hive.execution.mode=container;
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hive/blob/e44198f5/ql/src/test/results/clientpositive/llap/llap_udf.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/llap_udf.q.out b/ql/src/test/results/clientpositive/llap/llap_udf.q.out
new file mode 100644
index 0000000..29ed978
--- /dev/null
+++ b/ql/src/test/results/clientpositive/llap/llap_udf.q.out
@@ -0,0 +1,307 @@
+PREHOOK: query: drop table if exists src_orc
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: drop table if exists src_orc
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: create table src_orc stored as orc as select * from src
+PREHOOK: type: CREATETABLE_AS_SELECT
+PREHOOK: Input: default@src
+PREHOOK: Output: database:default
+PREHOOK: Output: default@src_orc
+POSTHOOK: query: create table src_orc stored as orc as select * from src
+POSTHOOK: type: CREATETABLE_AS_SELECT
+POSTHOOK: Input: default@src
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@src_orc
+PREHOOK: query: -- Not using GenericUDFTestGetJavaBoolean; that is already registered when tests begin
+
+CREATE TEMPORARY FUNCTION test_udf0 AS 'org.apache.hadoop.hive.ql.udf.generic.GenericUDFEvaluateNPE'
+PREHOOK: type: CREATEFUNCTION
+PREHOOK: Output: test_udf0
+POSTHOOK: query: -- Not using GenericUDFTestGetJavaBoolean; that is already registered when tests begin
+
+CREATE TEMPORARY FUNCTION test_udf0 AS 'org.apache.hadoop.hive.ql.udf.generic.GenericUDFEvaluateNPE'
+POSTHOOK: type: CREATEFUNCTION
+POSTHOOK: Output: test_udf0
+PREHOOK: query: EXPLAIN SELECT test_udf0(cast(key as string)) from src_orc
+PREHOOK: type: QUERY
+POSTHOOK: query: EXPLAIN SELECT test_udf0(cast(key as string)) from src_orc
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: src_orc
+ Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: evaluate_npe(key) (type: string)
+ outputColumnNames: _col0
+ Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: CREATE FUNCTION test_udf2 AS 'org.apache.hadoop.hive.ql.udf.generic.GenericUDFTestGetJavaString'
+PREHOOK: type: CREATEFUNCTION
+PREHOOK: Output: database:default
+PREHOOK: Output: default.test_udf2
+POSTHOOK: query: CREATE FUNCTION test_udf2 AS 'org.apache.hadoop.hive.ql.udf.generic.GenericUDFTestGetJavaString'
+POSTHOOK: type: CREATEFUNCTION
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default.test_udf2
+PREHOOK: query: CREATE FUNCTION test_udf3 AS 'org.apache.hadoop.hive.ql.udf.generic.GenericUDFTestGetJavaString'
+PREHOOK: type: CREATEFUNCTION
+PREHOOK: Output: database:default
+PREHOOK: Output: default.test_udf3
+POSTHOOK: query: CREATE FUNCTION test_udf3 AS 'org.apache.hadoop.hive.ql.udf.generic.GenericUDFTestGetJavaString'
+POSTHOOK: type: CREATEFUNCTION
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default.test_udf3
+PREHOOK: query: CREATE FUNCTION test_udf4 AS 'org.apache.hadoop.hive.ql.udf.generic.GenericUDFEvaluateNPE'
+PREHOOK: type: CREATEFUNCTION
+PREHOOK: Output: database:default
+PREHOOK: Output: default.test_udf4
+POSTHOOK: query: CREATE FUNCTION test_udf4 AS 'org.apache.hadoop.hive.ql.udf.generic.GenericUDFEvaluateNPE'
+POSTHOOK: type: CREATEFUNCTION
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default.test_udf4
+PREHOOK: query: EXPLAIN
+SELECT test_udf2(cast(key as string)), test_udf3(cast(key as string)), test_udf4(cast(key as string)) from src_orc
+PREHOOK: type: QUERY
+POSTHOOK: query: EXPLAIN
+SELECT test_udf2(cast(key as string)), test_udf3(cast(key as string)), test_udf4(cast(key as string)) from src_orc
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: src_orc
+ Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: GenericUDFTestGetJavaString(key) (type: string), GenericUDFTestGetJavaString(key) (type: string), evaluate_npe(key) (type: string)
+ outputColumnNames: _col0, _col1, _col2
+ Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ Execution mode: llap
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: -- Verification is based on classes, so 0 would work based on 4.
+EXPLAIN
+SELECT test_udf0(cast(key as string)) from src_orc
+PREHOOK: type: QUERY
+POSTHOOK: query: -- Verification is based on classes, so 0 would work based on 4.
+EXPLAIN
+SELECT test_udf0(cast(key as string)) from src_orc
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: src_orc
+ Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: evaluate_npe(key) (type: string)
+ outputColumnNames: _col0
+ Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ Execution mode: llap
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: DROP FUNCTION test_udf2
+PREHOOK: type: DROPFUNCTION
+PREHOOK: Output: database:default
+PREHOOK: Output: default.test_udf2
+POSTHOOK: query: DROP FUNCTION test_udf2
+POSTHOOK: type: DROPFUNCTION
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default.test_udf2
+PREHOOK: query: -- ...verify that 3 still works
+EXPLAIN
+SELECT test_udf3(cast(key as string)), test_udf4(cast(key as string)) from src_orc
+PREHOOK: type: QUERY
+POSTHOOK: query: -- ...verify that 3 still works
+EXPLAIN
+SELECT test_udf3(cast(key as string)), test_udf4(cast(key as string)) from src_orc
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: src_orc
+ Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: GenericUDFTestGetJavaString(key) (type: string), evaluate_npe(key) (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ Execution mode: llap
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: DROP FUNCTION test_udf4
+PREHOOK: type: DROPFUNCTION
+PREHOOK: Output: database:default
+PREHOOK: Output: default.test_udf4
+POSTHOOK: query: DROP FUNCTION test_udf4
+POSTHOOK: type: DROPFUNCTION
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default.test_udf4
+PREHOOK: query: -- ...now 0 should stop working
+EXPLAIN
+SELECT test_udf0(cast(key as string)) from src_orc
+PREHOOK: type: QUERY
+POSTHOOK: query: -- ...now 0 should stop working
+EXPLAIN
+SELECT test_udf0(cast(key as string)) from src_orc
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: src_orc
+ Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: evaluate_npe(key) (type: string)
+ outputColumnNames: _col0
+ Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: EXPLAIN
+SELECT test_udf3(cast(key as string)) from src_orc
+PREHOOK: type: QUERY
+POSTHOOK: query: EXPLAIN
+SELECT test_udf3(cast(key as string)) from src_orc
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: src_orc
+ Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: GenericUDFTestGetJavaString(key) (type: string)
+ outputColumnNames: _col0
+ Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: drop table if exists src_orc
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@src_orc
+PREHOOK: Output: default@src_orc
+POSTHOOK: query: drop table if exists src_orc
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@src_orc
+POSTHOOK: Output: default@src_orc

Search Discussions

Related Discussions

Discussion Navigation
viewthread | post
Discussion Overview
groupcommits @
categorieshive, hadoop
postedFeb 26, '16 at 3:48a
activeFeb 26, '16 at 3:48a
posts1
users1
websitehive.apache.org

1 user in discussion

Sershe: 1 post

People

Translate

site design / logo © 2021 Grokbase