FAQ
Repository: hive
Updated Branches:
   refs/heads/master 27172bcb4 -> bdf20ef57


HIVE-12805 : CBO: Calcite Operator To Hive Operator (Calcite Return Path): MiniTezCliDriver skewjoin.q failure(Hari Subramaniyan, reviewed by Ashutosh Chauhan)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/bdf20ef5
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/bdf20ef5
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/bdf20ef5

Branch: refs/heads/master
Commit: bdf20ef5709b62471c1c6cd2a0c317791465c7c1
Parents: 27172bc
Author: Hari Subramaniyan <harisankar@apache.org>
Authored: Tue Jan 26 19:08:45 2016 -0800
Committer: Hari Subramaniyan <harisankar@apache.org>
Committed: Tue Jan 26 19:08:45 2016 -0800

----------------------------------------------------------------------
  .../calcite/reloperators/HiveMultiJoin.java | 29 +++++++++++----
  .../rules/HiveInsertExchange4JoinRule.java | 10 ++---
  .../calcite/rules/HiveJoinToMultiJoinRule.java | 39 +++++++++++++++++++-
  3 files changed, 64 insertions(+), 14 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/bdf20ef5/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveMultiJoin.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveMultiJoin.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveMultiJoin.java
index 660f01d..cff737c 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveMultiJoin.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveMultiJoin.java
@@ -70,6 +70,7 @@ public final class HiveMultiJoin extends AbstractRelNode {
     * INNER
     * @param filters filters associated with each join
     * input
+ * @param joinPredicateInfo join predicate information
     */
    public HiveMultiJoin(
        RelOptCluster cluster,
@@ -78,9 +79,10 @@ public final class HiveMultiJoin extends AbstractRelNode {
        RelDataType rowType,
        List<Pair<Integer,Integer>> joinInputs,
        List<JoinRelType> joinTypes,
- List<RexNode> filters) {
+ List<RexNode> filters,
+ JoinPredicateInfo joinPredicateInfo) {
      super(cluster, TraitsUtil.getDefaultTraitSet(cluster));
- this.inputs = Lists.newArrayList(inputs);
+ this.inputs = inputs;
      this.condition = condition;
      this.rowType = rowType;

@@ -89,14 +91,27 @@ public final class HiveMultiJoin extends AbstractRelNode {
      this.joinTypes = ImmutableList.copyOf(joinTypes);
      this.filters = ImmutableList.copyOf(filters);
      this.outerJoin = containsOuter();
-
- try {
- this.joinPredInfo = HiveCalciteUtil.JoinPredicateInfo.constructJoinPredicateInfo(this);
- } catch (CalciteSemanticException e) {
- throw new RuntimeException(e);
+ if (joinPredicateInfo == null) {
+ try {
+ this.joinPredInfo = HiveCalciteUtil.JoinPredicateInfo.constructJoinPredicateInfo(this);
+ } catch (CalciteSemanticException e) {
+ throw new RuntimeException(e);
+ }
+ } else {
+ this.joinPredInfo = joinPredicateInfo;
      }
    }

+ public HiveMultiJoin(
+ RelOptCluster cluster,
+ List<RelNode> inputs,
+ RexNode condition,
+ RelDataType rowType,
+ List<Pair<Integer,Integer>> joinInputs,
+ List<JoinRelType> joinTypes,
+ List<RexNode> filters) {
+ this(cluster, Lists.newArrayList(inputs), condition, rowType, joinInputs, joinTypes, filters, null);
+ }

    @Override
    public void replaceInput(int ordinalInParent, RelNode p) {

http://git-wip-us.apache.org/repos/asf/hive/blob/bdf20ef5/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveInsertExchange4JoinRule.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveInsertExchange4JoinRule.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveInsertExchange4JoinRule.java
index d6e3915..c9cf396 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveInsertExchange4JoinRule.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveInsertExchange4JoinRule.java
@@ -38,6 +38,7 @@ import org.apache.hadoop.hive.ql.optimizer.calcite.HiveCalciteUtil.JoinLeafPredi
  import org.apache.hadoop.hive.ql.optimizer.calcite.HiveCalciteUtil.JoinPredicateInfo;
  import org.apache.hadoop.hive.ql.optimizer.calcite.HiveRelCollation;
  import org.apache.hadoop.hive.ql.optimizer.calcite.HiveRelDistribution;
+import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveJoin;
  import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveMultiJoin;
  import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveSortExchange;

@@ -76,11 +77,10 @@ public class HiveInsertExchange4JoinRule extends RelOptRule {
      JoinPredicateInfo joinPredInfo;
      if (call.rel(0) instanceof HiveMultiJoin) {
        HiveMultiJoin multiJoin = call.rel(0);
- try {
- joinPredInfo = HiveCalciteUtil.JoinPredicateInfo.constructJoinPredicateInfo(multiJoin);
- } catch (CalciteSemanticException e) {
- throw new RuntimeException(e);
- }
+ joinPredInfo = multiJoin.getJoinPredicateInfo();
+ } else if (call.rel(0) instanceof HiveJoin) {
+ HiveJoin hiveJoin = call.rel(0);
+ joinPredInfo = hiveJoin.getJoinPredicateInfo();
      } else if (call.rel(0) instanceof Join) {
        Join join = call.rel(0);
        try {

http://git-wip-us.apache.org/repos/asf/hive/blob/bdf20ef5/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveJoinToMultiJoinRule.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveJoinToMultiJoinRule.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveJoinToMultiJoinRule.java
index 5d169a1..ce9535f 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveJoinToMultiJoinRule.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveJoinToMultiJoinRule.java
@@ -239,14 +239,49 @@ public class HiveJoinToMultiJoinRule extends RelOptRule {
      // We can now create a multijoin operator
      RexNode newCondition = RexUtil.flatten(rexBuilder,
              RexUtil.composeConjunction(rexBuilder, newJoinCondition, false));
+ List<RelNode> newInputsArray = Lists.newArrayList(newInputs);
+ JoinPredicateInfo joinPredInfo = null;
+ try {
+ joinPredInfo = HiveCalciteUtil.JoinPredicateInfo.constructJoinPredicateInfo(newInputsArray, systemFieldList, newCondition);
+ } catch (CalciteSemanticException e) {
+ throw new RuntimeException(e);
+ }
+
+ // If the number of joins < number of input tables-1, this is not a star join.
+ if (joinPredInfo.getEquiJoinPredicateElements().size() < newInputs.size()-1) {
+ return null;
+ }
+ // Validate that the multi-join is a valid star join before returning it.
+ for (int i=0; i<newInputs.size(); i++) {
+ List<RexNode> joinKeys = null;
+ for (int j = 0; j < joinPredInfo.getEquiJoinPredicateElements().size(); j++) {
+ List<RexNode> currJoinKeys = joinPredInfo.
+ getEquiJoinPredicateElements().get(j).getJoinExprs(i);
+ if (currJoinKeys.isEmpty()) {
+ continue;
+ }
+ if (joinKeys == null) {
+ joinKeys = currJoinKeys;
+ } else {
+ // If we join on different keys on different tables, we can no longer apply
+ // multi-join conversion as this is no longer a valid star join.
+ // Bail out if this is the case.
+ if (!joinKeys.containsAll(currJoinKeys) || !currJoinKeys.containsAll(joinKeys)) {
+ return null;
+ }
+ }
+ }
+ }
+
      return new HiveMultiJoin(
              join.getCluster(),
- newInputs,
+ newInputsArray,
              newCondition,
              join.getRowType(),
              joinInputs,
              joinTypes,
- joinFilters);
+ joinFilters,
+ joinPredInfo);
    }

    /*

Search Discussions

Related Discussions

Discussion Navigation
viewthread | post
Discussion Overview
groupcommits @
categorieshive, hadoop
postedJan 27, '16 at 3:08a
activeJan 27, '16 at 3:08a
posts1
users1
websitehive.apache.org

1 user in discussion

Harisankar: 1 post

People

Translate

site design / logo © 2021 Grokbase