FAQ
Author: hashutosh
Date: Fri Sep 19 15:10:37 2014
New Revision: 1626244

URL: http://svn.apache.org/r1626244
Log:
HIVE-7359 : Stats based compute query replies fail to do simple column transforms (Ashutosh Chauhan via Gopal V)

Modified:
     hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/StatsOptimizer.java
     hive/trunk/ql/src/test/queries/clientpositive/metadata_only_queries_with_filters.q
     hive/trunk/ql/src/test/results/clientpositive/metadata_only_queries_with_filters.q.out

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/StatsOptimizer.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/StatsOptimizer.java?rev=1626244&r1=1626243&r2=1626244&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/StatsOptimizer.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/StatsOptimizer.java Fri Sep 19 15:10:37 2014
@@ -193,11 +193,12 @@ public class StatsOptimizer implements T
          }
          SelectOperator selOp = (SelectOperator)tsOp.getChildren().get(0);
          for(ExprNodeDesc desc : selOp.getConf().getColList()) {
- if (!(desc instanceof ExprNodeColumnDesc)) {
+ if (!((desc instanceof ExprNodeColumnDesc) || (desc instanceof ExprNodeConstantDesc))) {
              // Probably an expression, cant handle that
              return null;
            }
          }
+ Map<String, ExprNodeDesc> exprMap = selOp.getColumnExprMap();
          // Since we have done an exact match on TS-SEL-GBY-RS-GBY-SEL-FS
          // we need not to do any instanceof checks for following.
          GroupByOperator gbyOp = (GroupByOperator)selOp.getChildren().get(0);
@@ -215,6 +216,12 @@ public class StatsOptimizer implements T
            return null;

          }
+ for(ExprNodeDesc desc : selOp.getConf().getColList()) {
+ if (!(desc instanceof ExprNodeColumnDesc)) {
+ // Probably an expression, cant handle that
+ return null;
+ }
+ }
          FileSinkOperator fsOp = (FileSinkOperator)(selOp.getChildren().get(0));
          if (fsOp.getChildOperators() != null && fsOp.getChildOperators().size() > 0) {
            // looks like a subq plan.
@@ -236,22 +243,28 @@ public class StatsOptimizer implements T
            GenericUDAFResolver udaf =
                FunctionRegistry.getGenericUDAFResolver(aggr.getGenericUDAFName());
            if (udaf instanceof GenericUDAFSum) {
- if(!(aggr.getParameters().get(0) instanceof ExprNodeConstantDesc)){
+ ExprNodeDesc desc = aggr.getParameters().get(0);
+ String constant;
+ if (desc instanceof ExprNodeConstantDesc) {
+ constant = ((ExprNodeConstantDesc) desc).getValue().toString();
+ } else if (desc instanceof ExprNodeColumnDesc && exprMap.get(((ExprNodeColumnDesc)desc).getColumn()) instanceof ExprNodeConstantDesc) {
+ constant = ((ExprNodeConstantDesc)exprMap.get(((ExprNodeColumnDesc)desc).getColumn())).getValue().toString();
+ } else {
                return null;
              }
              Long rowCnt = getRowCnt(pctx, tsOp, tbl);
              if(rowCnt == null) {
                return null;
              }
- oneRow.add(HiveDecimal.create(((ExprNodeConstantDesc) aggr.getParameters().get(0))
- .getValue().toString()).multiply(HiveDecimal.create(rowCnt)));
+ oneRow.add(HiveDecimal.create(constant).multiply(HiveDecimal.create(rowCnt)));
              ois.add(PrimitiveObjectInspectorFactory.getPrimitiveJavaObjectInspector(
                  PrimitiveCategory.DECIMAL));
            }
            else if (udaf instanceof GenericUDAFCount) {
              Long rowCnt = 0L;
- if ((aggr.getParameters().isEmpty() || aggr.getParameters().get(0) instanceof
- ExprNodeConstantDesc)) {
+ if (aggr.getParameters().isEmpty() || aggr.getParameters().get(0) instanceof
+ ExprNodeConstantDesc || ((aggr.getParameters().get(0) instanceof ExprNodeColumnDesc) &&
+ exprMap.get(((ExprNodeColumnDesc)aggr.getParameters().get(0)).getColumn()) instanceof ExprNodeConstantDesc)) {
                // Its either count (*) or count(1) case
                rowCnt = getRowCnt(pctx, tsOp, tbl);
                if(rowCnt == null) {
@@ -259,12 +272,7 @@ public class StatsOptimizer implements T
                }
              } else {
                // Its count(col) case
- if (!(aggr.getParameters().get(0) instanceof ExprNodeColumnDesc)) {
- // this is weird, we got expr or something in there, bail out
- Log.debug("Unexpected expression : " + aggr.getParameters().get(0));
- return null;
- }
- ExprNodeColumnDesc desc = (ExprNodeColumnDesc)aggr.getParameters().get(0);
+ ExprNodeColumnDesc desc = (ExprNodeColumnDesc)exprMap.get(((ExprNodeColumnDesc)aggr.getParameters().get(0)).getColumn());
                String colName = desc.getColumn();
                StatType type = getType(desc.getTypeString());
                if(!tbl.isPartitioned()) {
@@ -330,7 +338,7 @@ public class StatsOptimizer implements T
              ois.add(PrimitiveObjectInspectorFactory.
                  getPrimitiveJavaObjectInspector(PrimitiveCategory.LONG));
            } else if (udaf instanceof GenericUDAFMax) {
- ExprNodeColumnDesc colDesc = (ExprNodeColumnDesc)aggr.getParameters().get(0);
+ ExprNodeColumnDesc colDesc = (ExprNodeColumnDesc)exprMap.get(((ExprNodeColumnDesc)aggr.getParameters().get(0)).getColumn());
              String colName = colDesc.getColumn();
              StatType type = getType(colDesc.getTypeString());
              if(!tbl.isPartitioned()) {
@@ -419,7 +427,7 @@ public class StatsOptimizer implements T
                }
              }
            } else if (udaf instanceof GenericUDAFMin) {
- ExprNodeColumnDesc colDesc = (ExprNodeColumnDesc)aggr.getParameters().get(0);
+ ExprNodeColumnDesc colDesc = (ExprNodeColumnDesc)exprMap.get(((ExprNodeColumnDesc)aggr.getParameters().get(0)).getColumn());
              String colName = colDesc.getColumn();
              StatType type = getType(colDesc.getTypeString());
              if (!tbl.isPartitioned()) {

Modified: hive/trunk/ql/src/test/queries/clientpositive/metadata_only_queries_with_filters.q
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientpositive/metadata_only_queries_with_filters.q?rev=1626244&r1=1626243&r2=1626244&view=diff
==============================================================================
--- hive/trunk/ql/src/test/queries/clientpositive/metadata_only_queries_with_filters.q (original)
+++ hive/trunk/ql/src/test/queries/clientpositive/metadata_only_queries_with_filters.q Fri Sep 19 15:10:37 2014
@@ -46,6 +46,8 @@ explain
  select count(*), count(1), sum(1), sum(2), count(s), count(bo), count(bin), count(si), max(i), min(b), max(f), min(d) from stats_tbl_part where dt > 2010;
  select count(*), count(1), sum(1), sum(2), count(s), count(bo), count(bin), count(si), max(i), min(b), max(f), min(d) from stats_tbl_part where dt > 2010;

+select count(*) from stats_tbl_part;
+select count(*)/2 from stats_tbl_part;
  drop table stats_tbl_part;
  set hive.compute.query.using.stats=false;
  set hive.stats.dbclass=jdbc:derby;

Modified: hive/trunk/ql/src/test/results/clientpositive/metadata_only_queries_with_filters.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/metadata_only_queries_with_filters.q.out?rev=1626244&r1=1626243&r2=1626244&view=diff
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/metadata_only_queries_with_filters.q.out (original)
+++ hive/trunk/ql/src/test/results/clientpositive/metadata_only_queries_with_filters.q.out Fri Sep 19 15:10:37 2014
@@ -188,6 +188,26 @@ POSTHOOK: query: select count(*), count(
  POSTHOOK: type: QUERY
  #### A masked pattern was here ####
  2219 2219 2219 4438 2219 2219 2219 2219 65791 4294967296 99.95999908447266 0.04
+PREHOOK: query: select count(*) from stats_tbl_part
+PREHOOK: type: QUERY
+#### A masked pattern was here ####
+POSTHOOK: query: select count(*) from stats_tbl_part
+POSTHOOK: type: QUERY
+#### A masked pattern was here ####
+4541
+PREHOOK: query: select count(*)/2 from stats_tbl_part
+PREHOOK: type: QUERY
+PREHOOK: Input: default@stats_tbl_part
+PREHOOK: Input: default@stats_tbl_part@dt=2010
+PREHOOK: Input: default@stats_tbl_part@dt=2014
+#### A masked pattern was here ####
+POSTHOOK: query: select count(*)/2 from stats_tbl_part
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@stats_tbl_part
+POSTHOOK: Input: default@stats_tbl_part@dt=2010
+POSTHOOK: Input: default@stats_tbl_part@dt=2014
+#### A masked pattern was here ####
+2270.5
  PREHOOK: query: drop table stats_tbl_part
  PREHOOK: type: DROPTABLE
  PREHOOK: Input: default@stats_tbl_part

Search Discussions

Related Discussions

Discussion Navigation
viewthread | post
Discussion Overview
groupcommits @
categorieshive, hadoop
postedSep 19, '14 at 3:11p
activeSep 19, '14 at 3:11p
posts1
users1
websitehive.apache.org

1 user in discussion

Hashutosh: 1 post

People

Translate

site design / logo © 2021 Grokbase