FAQ
Author: gunther
Date: Fri Mar 28 05:30:27 2014
New Revision: 1582602

URL: http://svn.apache.org/r1582602
Log:
HIVE-6760: Scalable dynamic partitioning should bail out properly for list bucketing (Prasanth J via Gunther Hagleitner)

Modified:
     hive/branches/branch-0.13/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SortedDynPartitionOptimizer.java

Modified: hive/branches/branch-0.13/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SortedDynPartitionOptimizer.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.13/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SortedDynPartitionOptimizer.java?rev=1582602&r1=1582601&r2=1582602&view=diff
==============================================================================
--- hive/branches/branch-0.13/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SortedDynPartitionOptimizer.java (original)
+++ hive/branches/branch-0.13/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SortedDynPartitionOptimizer.java Fri Mar 28 05:30:27 2014
@@ -63,6 +63,7 @@ import org.apache.hadoop.hive.ql.plan.Ex
  import org.apache.hadoop.hive.ql.plan.ExprNodeDesc;
  import org.apache.hadoop.hive.ql.plan.ExtractDesc;
  import org.apache.hadoop.hive.ql.plan.FileSinkDesc;
+import org.apache.hadoop.hive.ql.plan.ListBucketingCtx;
  import org.apache.hadoop.hive.ql.plan.OperatorDesc;
  import org.apache.hadoop.hive.ql.plan.PlanUtils;
  import org.apache.hadoop.hive.ql.plan.ReduceSinkDesc;
@@ -124,9 +125,19 @@ public class SortedDynPartitionOptimizer
        // RS then ReduceSinkDeDuplication optimization should merge them
        FileSinkOperator fsOp = (FileSinkOperator) nd;

+ LOG.info("Sorted dynamic partitioning optimization kicked in..");
+
        // if not dynamic partitioning then bail out
        if (fsOp.getConf().getDynPartCtx() == null) {
- LOG.debug("Bailing out of sort dynamic partition optimization as dpCtx is null");
+ LOG.debug("Bailing out of sort dynamic partition optimization as dynamic partitioning context is null");
+ return null;
+ }
+
+ // if list bucketing then bail out
+ ListBucketingCtx lbCtx = fsOp.getConf().getLbCtx();
+ if (lbCtx != null && !lbCtx.getSkewedColNames().isEmpty()
+ && !lbCtx.getSkewedColValues().isEmpty()) {
+ LOG.debug("Bailing out of sort dynamic partition optimization as list bucketing is enabled");
          return null;
        }

Search Discussions

Related Discussions

Discussion Navigation
viewthread | post
Discussion Overview
groupcommits @
categorieshive, hadoop
postedMar 28, '14 at 5:30a
activeMar 28, '14 at 5:30a
posts1
users1
websitehive.apache.org

1 user in discussion

Gunther: 1 post

People

Translate

site design / logo © 2022 Grokbase