FAQ
Repository: hive
Updated Branches:
   refs/heads/branch-1.0 3dccb381e -> 2801d2c4b


HIVE-10083 - SMBJoin fails in case one table is uninitialized (Na via Chao)

git-svn-id: https://svn.apache.org/repos/asf/hive/trunk@1670718 13f79535-47bb-0310-9956-ffa450edef68


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/a7618dfb
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/a7618dfb
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/a7618dfb

Branch: refs/heads/branch-1.0
Commit: a7618dfb9f93eab922f1939680dca4ae5d5a8f6b
Parents: 3dccb38
Author: Sun Chao <sunchao@apache.org>
Authored: Wed Apr 1 16:27:50 2015 +0000
Committer: Pengcheng Xiong <pxiong@apache.org>
Committed: Thu Sep 24 17:10:50 2015 -0700

----------------------------------------------------------------------
  .../ql/optimizer/AbstractBucketJoinProc.java | 24 +++++++++++---------
  1 file changed, 13 insertions(+), 11 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/a7618dfb/ql/src/java/org/apache/hadoop/hive/ql/optimizer/AbstractBucketJoinProc.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/AbstractBucketJoinProc.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/AbstractBucketJoinProc.java
index 70c23a6..13ede1b 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/AbstractBucketJoinProc.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/AbstractBucketJoinProc.java
@@ -479,17 +479,19 @@ abstract public class AbstractBucketJoinProc implements NodeProcessor {
        for (int sindex = 0; sindex < smallTblBucketNums.size(); sindex++) {
          int smallTblBucketNum = smallTblBucketNums.get(sindex);
          List<String> smallTblFileNames = smallTblFilesList.get(sindex);
- if (bigTblBucketNum >= smallTblBucketNum) {
- // if the big table has more buckets than the current small table,
- // use "MOD" to get small table bucket names. For example, if the big
- // table has 4 buckets and the small table has 2 buckets, then the
- // mapping should be 0->0, 1->1, 2->0, 3->1.
- int toAddSmallIndex = bindex % smallTblBucketNum;
- resultFileNames.add(smallTblFileNames.get(toAddSmallIndex));
- } else {
- int jump = smallTblBucketNum / bigTblBucketNum;
- for (int i = bindex; i < smallTblFileNames.size(); i = i + jump) {
- resultFileNames.add(smallTblFileNames.get(i));
+ if (smallTblFileNames.size() > 0) {
+ if (bigTblBucketNum >= smallTblBucketNum) {
+ // if the big table has more buckets than the current small table,
+ // use "MOD" to get small table bucket names. For example, if the big
+ // table has 4 buckets and the small table has 2 buckets, then the
+ // mapping should be 0->0, 1->1, 2->0, 3->1.
+ int toAddSmallIndex = bindex % smallTblBucketNum;
+ resultFileNames.add(smallTblFileNames.get(toAddSmallIndex));
+ } else {
+ int jump = smallTblBucketNum / bigTblBucketNum;
+ for (int i = bindex; i < smallTblFileNames.size(); i = i + jump) {
+ resultFileNames.add(smallTblFileNames.get(i));
+ }
            }
          }
        }

Search Discussions

  • Pxiong at Sep 25, 2015 at 3:39 am
    HIVE-10209 - FetchTask with VC may fail because ExecMapper.done is true (Chao)

    git-svn-id: https://svn.apache.org/repos/asf/hive/branches/spark@1672509 13f79535-47bb-0310-9956-ffa450edef68


    Project: http://git-wip-us.apache.org/repos/asf/hive/repo
    Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/2801d2c4
    Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/2801d2c4
    Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/2801d2c4

    Branch: refs/heads/branch-1.0
    Commit: 2801d2c4b1a61315ae7f28c0ea825580e30f411b
    Parents: a7618df
    Author: Sun Chao <sunchao@apache.org>
    Authored: Thu Apr 9 22:30:15 2015 +0000
    Committer: Pengcheng Xiong <pxiong@apache.org>
    Committed: Thu Sep 24 20:35:14 2015 -0700

    ----------------------------------------------------------------------
      ql/src/java/org/apache/hadoop/hive/ql/exec/FetchTask.java | 2 ++
      1 file changed, 2 insertions(+)
    ----------------------------------------------------------------------


    http://git-wip-us.apache.org/repos/asf/hive/blob/2801d2c4/ql/src/java/org/apache/hadoop/hive/ql/exec/FetchTask.java
    ----------------------------------------------------------------------
    diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/FetchTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/FetchTask.java
    index c4f04cb..3e2187d 100644
    --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/FetchTask.java
    +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/FetchTask.java
    @@ -28,6 +28,7 @@ import org.apache.hadoop.hive.conf.HiveConf;
      import org.apache.hadoop.hive.ql.CommandNeedRetryException;
      import org.apache.hadoop.hive.ql.DriverContext;
      import org.apache.hadoop.hive.ql.QueryPlan;
    +import org.apache.hadoop.hive.ql.exec.mr.ExecMapper;
      import org.apache.hadoop.hive.ql.io.HiveInputFormat;
      import org.apache.hadoop.hive.ql.metadata.HiveException;
      import org.apache.hadoop.hive.ql.metadata.VirtualColumn;
    @@ -79,6 +80,7 @@ public class FetchTask extends Task<FetchWork> implements Serializable {
            fetch = new FetchOperator(work, job, source, getVirtualColumns(source));
            source.initialize(conf, new ObjectInspector[]{fetch.getOutputObjectInspector()});
            totalRows = 0;
    + ExecMapper.setDone(false);

          } catch (Exception e) {
            // Bail out ungracefully - we should never hit

Related Discussions

Discussion Navigation
viewthread | post
Discussion Overview
groupcommits @
categorieshive, hadoop
postedSep 25, '15 at 3:39a
activeSep 25, '15 at 3:39a
posts2
users1
websitehive.apache.org

1 user in discussion

Pxiong: 2 posts

People

Translate

site design / logo © 2021 Grokbase