FAQ
Author: sershe
Date: Tue Oct 14 22:45:05 2014
New Revision: 1631921

URL: http://svn.apache.org/r1631921
Log:
HIVE-8389 : Fix CBO when indexes are used (Pengcheng Xiong, reviewed by Sergey Shelukhin)

Modified:
     hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
     hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/index/IndexPredicateAnalyzer.java
     hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/Optimizer.java
     hive/trunk/ql/src/test/queries/clientnegative/set_hiveconf_validation2.q
     hive/trunk/ql/src/test/queries/clientpositive/index_auto_partitioned.q
     hive/trunk/ql/src/test/queries/clientpositive/index_bitmap_auto_partitioned.q
     hive/trunk/ql/src/test/results/clientnegative/set_hiveconf_validation2.q.out

Modified: hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
URL: http://svn.apache.org/viewvc/hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java?rev=1631921&r1=1631920&r2=1631921&view=diff
==============================================================================
--- hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java (original)
+++ hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java Tue Oct 14 22:45:05 2014
@@ -1317,10 +1317,11 @@ public class HiveConf extends Configurat

      HIVEOUTERJOINSUPPORTSFILTERS("hive.outerjoin.supports.filters", true, ""),

- HIVEFETCHTASKCONVERSION("hive.fetch.task.conversion", "more", new StringSet("minimal", "more"),
+ HIVEFETCHTASKCONVERSION("hive.fetch.task.conversion", "more", new StringSet("none", "minimal", "more"),
          "Some select queries can be converted to single FETCH task minimizing latency.\n" +
          "Currently the query should be single sourced not having any subquery and should not have\n" +
          "any aggregations or distincts (which incurs RS), lateral views and joins.\n" +
+ "0. none : disable hive.fetch.task.conversion\n" +
          "1. minimal : SELECT STAR, FILTER on partition columns, LIMIT only\n" +
          "2. more : SELECT, FILTER, LIMIT only (support TABLESAMPLE and virtual columns)"
      ),

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/index/IndexPredicateAnalyzer.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/index/IndexPredicateAnalyzer.java?rev=1631921&r1=1631920&r2=1631921&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/index/IndexPredicateAnalyzer.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/index/IndexPredicateAnalyzer.java Tue Oct 14 22:45:05 2014
@@ -182,6 +182,29 @@ public class IndexPredicateAnalyzer {
      }
      ExprNodeDesc expr1 = (ExprNodeDesc) nodeOutputs[0];
      ExprNodeDesc expr2 = (ExprNodeDesc) nodeOutputs[1];
+ // We may need to peel off the GenericUDFBridge that is added by CBO or user
+ boolean peelOffGenericUDFBridge = false;
+ while (expr1 instanceof ExprNodeGenericFuncDesc && expr2 instanceof ExprNodeGenericFuncDesc) {
+ GenericUDF udf1 = ((ExprNodeGenericFuncDesc) expr1).getGenericUDF();
+ GenericUDF udf2 = ((ExprNodeGenericFuncDesc) expr2).getGenericUDF();
+ // We assume that GenericUDFBridge that is added by CBO or user if they
+ // have the same udf names.
+ if (udf1.getUdfName() == udf2.getUdfName()) {
+ peelOffGenericUDFBridge = true;
+ expr1 = expr1.getChildren().get(0);
+ expr2 = expr2.getChildren().get(0);
+ } else {
+ break;
+ }
+ }
+ // We also need to update the expr so that the index query can be generated.
+ // Note that, hive does not support UDFToDouble in the query text.
+ if (peelOffGenericUDFBridge) {
+ List<ExprNodeDesc> list = new ArrayList<ExprNodeDesc>();
+ list.add(expr1);
+ list.add(expr2);
+ expr = new ExprNodeGenericFuncDesc(expr.getTypeInfo(), expr.getGenericUDF(), list);
+ }
      ExprNodeDesc[] extracted = ExprNodeDescUtils.extractComparePair(expr1, expr2);
      if (extracted == null || (extracted.length > 2 && !acceptsFields)) {
        return expr;

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/Optimizer.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/Optimizer.java?rev=1631921&r1=1631920&r2=1631921&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/Optimizer.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/Optimizer.java Tue Oct 14 22:45:05 2014
@@ -143,8 +143,10 @@ public class Optimizer {
        transformations.add(new AnnotateWithOpTraits());
      }

- transformations.add(new SimpleFetchOptimizer()); // must be called last
-
+ if (!HiveConf.getVar(hiveConf, HiveConf.ConfVars.HIVEFETCHTASKCONVERSION).equals("none")) {
+ transformations.add(new SimpleFetchOptimizer()); // must be called last
+ }
+
      if (HiveConf.getBoolVar(hiveConf, HiveConf.ConfVars.HIVEFETCHTASKAGGR)) {
        transformations.add(new SimpleFetchAggregation());
      }

Modified: hive/trunk/ql/src/test/queries/clientnegative/set_hiveconf_validation2.q
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientnegative/set_hiveconf_validation2.q?rev=1631921&r1=1631920&r2=1631921&view=diff
==============================================================================
--- hive/trunk/ql/src/test/queries/clientnegative/set_hiveconf_validation2.q (original)
+++ hive/trunk/ql/src/test/queries/clientnegative/set_hiveconf_validation2.q Tue Oct 14 22:45:05 2014
@@ -1,4 +1,4 @@
--- should fail: hive.fetch.task.conversion accepts minimal or more
+-- should fail: hive.fetch.task.conversion accepts none, minimal or more
  desc src;

  set hive.conf.validation=true;

Modified: hive/trunk/ql/src/test/queries/clientpositive/index_auto_partitioned.q
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientpositive/index_auto_partitioned.q?rev=1631921&r1=1631920&r2=1631921&view=diff
==============================================================================
--- hive/trunk/ql/src/test/queries/clientpositive/index_auto_partitioned.q (original)
+++ hive/trunk/ql/src/test/queries/clientpositive/index_auto_partitioned.q Tue Oct 14 22:45:05 2014
@@ -1,4 +1,5 @@
  set hive.stats.dbclass=fs;
+set hive.fetch.task.conversion=none;

  -- SORT_QUERY_RESULTS
  -- test automatic use of index on table with partitions

Modified: hive/trunk/ql/src/test/queries/clientpositive/index_bitmap_auto_partitioned.q
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientpositive/index_bitmap_auto_partitioned.q?rev=1631921&r1=1631920&r2=1631921&view=diff
==============================================================================
--- hive/trunk/ql/src/test/queries/clientpositive/index_bitmap_auto_partitioned.q (original)
+++ hive/trunk/ql/src/test/queries/clientpositive/index_bitmap_auto_partitioned.q Tue Oct 14 22:45:05 2014
@@ -1,4 +1,5 @@
  set hive.stats.dbclass=fs;
+set hive.fetch.task.conversion=none;

  -- SORT_QUERY_RESULTS


Modified: hive/trunk/ql/src/test/results/clientnegative/set_hiveconf_validation2.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientnegative/set_hiveconf_validation2.q.out?rev=1631921&r1=1631920&r2=1631921&view=diff
==============================================================================
--- hive/trunk/ql/src/test/results/clientnegative/set_hiveconf_validation2.q.out (original)
+++ hive/trunk/ql/src/test/results/clientnegative/set_hiveconf_validation2.q.out Tue Oct 14 22:45:05 2014
@@ -1,11 +1,11 @@
-PREHOOK: query: -- should fail: hive.fetch.task.conversion accepts minimal or more
+PREHOOK: query: -- should fail: hive.fetch.task.conversion accepts none, minimal or more
  desc src
  PREHOOK: type: DESCTABLE
  PREHOOK: Input: default@src
-POSTHOOK: query: -- should fail: hive.fetch.task.conversion accepts minimal or more
+POSTHOOK: query: -- should fail: hive.fetch.task.conversion accepts none, minimal or more
  desc src
  POSTHOOK: type: DESCTABLE
  POSTHOOK: Input: default@src
  key string default
  value string default
-Query returned non-zero code: 1, cause: 'SET hive.fetch.task.conversion=true' FAILED in validation : Invalid value.. expects one of [minimal, more].
+Query returned non-zero code: 1, cause: 'SET hive.fetch.task.conversion=true' FAILED in validation : Invalid value.. expects one of [none, minimal, more].

Search Discussions

Related Discussions

Discussion Navigation
viewthread | post
posts ‹ prev | 1 of 1 | next ›
Discussion Overview
groupcommits @
categorieshive, hadoop
postedOct 14, '14 at 10:45p
activeOct 14, '14 at 10:45p
posts1
users1
websitehive.apache.org

1 user in discussion

Sershe: 1 post

People

Translate

site design / logo © 2021 Grokbase