FAQ
Repository: hive
Updated Branches:
   refs/heads/master 368bcaead -> 76a3d5e37


HIVE-13932:Hive SMB Map Join with small set of LIMIT failed with NPE (Yongzhi Chen, reviewed by Chaoyu Tang)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/76a3d5e3
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/76a3d5e3
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/76a3d5e3

Branch: refs/heads/master
Commit: 76a3d5e37acfed87ce5eb153616579acee9a8852
Parents: 368bcae
Author: Yongzhi Chen <ychena@apache.org>
Authored: Thu Jun 2 18:47:13 2016 -0400
Committer: Yongzhi Chen <ychena@apache.org>
Committed: Tue Jun 7 11:17:45 2016 -0400

----------------------------------------------------------------------
  data/files/smbdata.txt | 99 ++++++++++++++++++++
  .../hadoop/hive/ql/exec/SMBMapJoinOperator.java | 2 +-
  ql/src/test/queries/clientpositive/smblimit.q | 12 +++
  .../test/results/clientpositive/smblimit.q.out | 50 ++++++++++
  4 files changed, 162 insertions(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/76a3d5e3/data/files/smbdata.txt
----------------------------------------------------------------------
diff --git a/data/files/smbdata.txt b/data/files/smbdata.txt
new file mode 100644
index 0000000..4422a54
--- /dev/null
+++ b/data/files/smbdata.txt
@@ -0,0 +1,99 @@
+1
+2
+3
+4
+5
+6
+7
+8
+9
+10
+11
+12
+13
+14
+15
+16
+17
+18
+19
+20
+21
+22
+23
+24
+25
+26
+27
+28
+29
+30
+31
+32
+33
+34
+35
+36
+37
+38
+39
+40
+41
+42
+43
+44
+45
+46
+47
+48
+49
+50
+51
+52
+53
+54
+55
+56
+57
+58
+59
+60
+61
+62
+63
+64
+65
+66
+67
+68
+69
+70
+71
+72
+73
+74
+75
+76
+77
+78
+79
+80
+81
+82
+83
+84
+85
+86
+87
+88
+89
+90
+91
+92
+93
+94
+95
+96
+97
+98
+99

http://git-wip-us.apache.org/repos/asf/hive/blob/76a3d5e3/ql/src/java/org/apache/hadoop/hive/ql/exec/SMBMapJoinOperator.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/SMBMapJoinOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/SMBMapJoinOperator.java
index 23abec3..27b1673 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/SMBMapJoinOperator.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/SMBMapJoinOperator.java
@@ -350,7 +350,7 @@ public class SMBMapJoinOperator extends AbstractMapJoinOperator<SMBJoinDesc> imp
        joinOneGroup();
        dataInCache = false;
        for (byte pos = 0; pos < order.length; pos++) {
- if (this.candidateStorage[pos].hasRows()) {
+ if (this.candidateStorage[pos] != null && this.candidateStorage[pos].hasRows()) {
            dataInCache = true;
            break;
          }

http://git-wip-us.apache.org/repos/asf/hive/blob/76a3d5e3/ql/src/test/queries/clientpositive/smblimit.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/smblimit.q b/ql/src/test/queries/clientpositive/smblimit.q
new file mode 100644
index 0000000..1db2167
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/smblimit.q
@@ -0,0 +1,12 @@
+drop table if exists hlp1;
+drop table if exists btl;
+create table hlp1(c string);
+load data local inpath '../../data/files/smbdata.txt' into table hlp1;
+create table btl(c string) clustered by (c) sorted by (c) into 5 buckets;
+insert overwrite table btl select * from hlp1;
+SET hive.auto.convert.sortmerge.join = true;
+SET hive.auto.convert.sortmerge.join.bigtable.selection.policy = org.apache.hadoop.hive.ql.optimizer.LeftmostBigTableSelectorForAutoSMJ;
+SET hive.optimize.bucketmapjoin = true;
+SET hive.optimize.bucketmapjoin.sortedmerge = true;
+select 1 from btl join btl t1 on btl.c=t1.c limit 1;
+

http://git-wip-us.apache.org/repos/asf/hive/blob/76a3d5e3/ql/src/test/results/clientpositive/smblimit.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/smblimit.q.out b/ql/src/test/results/clientpositive/smblimit.q.out
new file mode 100644
index 0000000..64ca604
--- /dev/null
+++ b/ql/src/test/results/clientpositive/smblimit.q.out
@@ -0,0 +1,50 @@
+PREHOOK: query: drop table if exists hlp1
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: drop table if exists hlp1
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: drop table if exists btl
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: drop table if exists btl
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: create table hlp1(c string)
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@hlp1
+POSTHOOK: query: create table hlp1(c string)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@hlp1
+PREHOOK: query: load data local inpath '../../data/files/smbdata.txt' into table hlp1
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@hlp1
+POSTHOOK: query: load data local inpath '../../data/files/smbdata.txt' into table hlp1
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@hlp1
+PREHOOK: query: create table btl(c string) clustered by (c) sorted by (c) into 5 buckets
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@btl
+POSTHOOK: query: create table btl(c string) clustered by (c) sorted by (c) into 5 buckets
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@btl
+PREHOOK: query: insert overwrite table btl select * from hlp1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@hlp1
+PREHOOK: Output: default@btl
+POSTHOOK: query: insert overwrite table btl select * from hlp1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@hlp1
+POSTHOOK: Output: default@btl
+POSTHOOK: Lineage: btl.c SIMPLE [(hlp1)hlp1.FieldSchema(name:c, type:string, comment:null), ]
+PREHOOK: query: select 1 from btl join btl t1 on btl.c=t1.c limit 1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@btl
+#### A masked pattern was here ####
+POSTHOOK: query: select 1 from btl join btl t1 on btl.c=t1.c limit 1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@btl
+#### A masked pattern was here ####
+1

Search Discussions

Discussion Posts

Follow ups

Related Discussions

Discussion Navigation
viewthread | post
posts ‹ prev | 1 of 2 | next ›
Discussion Overview
groupcommits @
categorieshive, hadoop
postedJun 7, '16 at 3:20p
activeJun 7, '16 at 4:09p
posts2
users1
websitehive.apache.org

1 user in discussion

Ychena: 2 posts

People

Translate

site design / logo © 2021 Grokbase