Author: gunther
Date: Fri Mar 28 05:28:13 2014
New Revision: 1582600
URL: http://svn.apache.org/r1582600
Log:
HIVE-6760: Scalable dynamic partitioning should bail out properly for list bucketing (Prasanth J via Gunther Hagleitner)
Modified:
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SortedDynPartitionOptimizer.java
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SortedDynPartitionOptimizer.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SortedDynPartitionOptimizer.java?rev=1582600&r1=1582599&r2=1582600&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SortedDynPartitionOptimizer.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SortedDynPartitionOptimizer.java Fri Mar 28 05:28:13 2014
@@ -63,6 +63,7 @@ import org.apache.hadoop.hive.ql.plan.Ex
import org.apache.hadoop.hive.ql.plan.ExprNodeDesc;
import org.apache.hadoop.hive.ql.plan.ExtractDesc;
import org.apache.hadoop.hive.ql.plan.FileSinkDesc;
+import org.apache.hadoop.hive.ql.plan.ListBucketingCtx;
import org.apache.hadoop.hive.ql.plan.OperatorDesc;
import org.apache.hadoop.hive.ql.plan.PlanUtils;
import org.apache.hadoop.hive.ql.plan.ReduceSinkDesc;
@@ -124,9 +125,19 @@ public class SortedDynPartitionOptimizer
// RS then ReduceSinkDeDuplication optimization should merge them
FileSinkOperator fsOp = (FileSinkOperator) nd;
+ LOG.info("Sorted dynamic partitioning optimization kicked in..");
+
// if not dynamic partitioning then bail out
if (fsOp.getConf().getDynPartCtx() == null) {
- LOG.debug("Bailing out of sort dynamic partition optimization as dpCtx is null");
+ LOG.debug("Bailing out of sort dynamic partition optimization as dynamic partitioning context is null");
+ return null;
+ }
+
+ // if list bucketing then bail out
+ ListBucketingCtx lbCtx = fsOp.getConf().getLbCtx();
+ if (lbCtx != null && !lbCtx.getSkewedColNames().isEmpty()
+ && !lbCtx.getSkewedColValues().isEmpty()) {
+ LOG.debug("Bailing out of sort dynamic partition optimization as list bucketing is enabled");
return null;
}
