FAQ
Author: xuefu
Date: Fri Apr 18 02:19:41 2014
New Revision: 1588401

URL: http://svn.apache.org/r1588401
Log:
HIVE-6913: Hive unable to find the hashtable file during complex multi-staged map join (Brock via Xuefu)

Modified:
     hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/HashTableSinkOperator.java
     hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/MapJoinOperator.java
     hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Operator.java
     hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/MapredLocalTask.java

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/HashTableSinkOperator.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/HashTableSinkOperator.java?rev=1588401&r1=1588400&r2=1588401&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/HashTableSinkOperator.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/HashTableSinkOperator.java Fri Apr 18 02:19:41 2014
@@ -263,7 +263,9 @@ public class HashTableSinkOperator exten
    @Override
    public void closeOp(boolean abort) throws HiveException {
      try {
- if (mapJoinTables != null) {
+ if (mapJoinTables == null) {
+ LOG.debug("mapJoinTables is null");
+ } else {
          flushToFile();
        }
        super.closeOp(abort);

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/MapJoinOperator.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/MapJoinOperator.java?rev=1588401&r1=1588400&r2=1588401&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/MapJoinOperator.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/MapJoinOperator.java Fri Apr 18 02:19:41 2014
@@ -248,8 +248,10 @@ public class MapJoinOperator extends Abs
            storage[pos] = null;
          }
        }
- } catch (SerDeException e) {
- throw new HiveException(e);
+ } catch (Exception e) {
+ String msg = "Unxpected exception: " + e.getMessage();
+ LOG.error(msg, e);
+ throw new HiveException(msg, e);
      }
    }


Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Operator.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Operator.java?rev=1588401&r1=1588400&r2=1588401&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Operator.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Operator.java Fri Apr 18 02:19:41 2014
@@ -546,6 +546,7 @@ public abstract class Operator<T extends
          if(parent==null){
            continue;
          }
+ LOG.debug("allInitializedParentsAreClosed? parent.state = " + parent.state);
          if (!(parent.state == State.CLOSE || parent.state == State.UNINIT)) {
            return false;
          }
@@ -565,6 +566,7 @@ public abstract class Operator<T extends

      // check if all parents are finished
      if (!allInitializedParentsAreClosed()) {
+ LOG.debug("Not all parent operators are closed. Not closing.");
        return;
      }

@@ -585,6 +587,7 @@ public abstract class Operator<T extends
        }

        for (Operator<? extends OperatorDesc> op : childOperators) {
+ LOG.debug("Closing child = " + op);
          op.close(abort);
        }


Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/MapredLocalTask.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/MapredLocalTask.java?rev=1588401&r1=1588400&r2=1588401&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/MapredLocalTask.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/MapredLocalTask.java Fri Apr 18 02:19:41 2014
@@ -349,14 +349,15 @@ public class MapredLocalTask extends Tas
          setUpFetchOpContext(fetchOp, alias, bigTableBucket);
        }

+ // get the root operator
+ Operator<? extends OperatorDesc> forwardOp = work.getAliasToWork().get(alias);
        if (fetchOp.isEmptyTable()) {
          //generate empty hashtable for empty table
          this.generateDummyHashTable(alias, bigTableBucket);
+ forwardOp.close(false);
          continue;
        }

- // get the root operator
- Operator<? extends OperatorDesc> forwardOp = work.getAliasToWork().get(alias);
        // walk through the operator tree
        while (!forwardOp.getDone()) {
          InspectableObject row = fetchOp.getNextRow();
@@ -375,6 +376,9 @@ public class MapredLocalTask extends Tas

    private void initializeOperators(Map<FetchOperator, JobConf> fetchOpJobConfMap)
        throws HiveException {
+ for (Map.Entry<String, Operator<? extends OperatorDesc>> entry : work.getAliasToWork().entrySet()) {
+ LOG.debug("initializeOperators: " + entry.getKey() + ", children = " + entry.getValue().getChildOperators());
+ }
      // this mapper operator is used to initialize all the operators
      for (Map.Entry<String, FetchWork> entry : work.getAliasToFetchWork().entrySet()) {
        if (entry.getValue() == null) {
@@ -419,6 +423,7 @@ public class MapredLocalTask extends Tas

    private void generateDummyHashTable(String alias, String bigBucketFileName)
        throws HiveException,IOException {
+ LOG.debug("generating dummy for " + alias);
      // find the (byte)tag for the map join(HashTableSinkOperator)
      Operator<? extends OperatorDesc> parentOp = work.getAliasToWork().get(alias);
      Operator<? extends OperatorDesc> childOp = parentOp.getChildOperators().get(0);

Search Discussions

Related Discussions

Discussion Navigation
viewthread | post
Discussion Overview
groupcommits @
categorieshive, hadoop
postedApr 18, '14 at 9:26a
activeApr 18, '14 at 9:26a
posts1
users1
websitehive.apache.org

1 user in discussion

Xuefu: 1 post

People

Translate

site design / logo © 2021 Grokbase