FAQ
Repository: hive
Updated Branches:
   refs/heads/branch-1 39285e5cc -> 9b8fbd486


HIVE-13065: Hive throws NPE when writing map type data to a HBase backed table (Yongzhi Chen, reviewed by Aihua Xu)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/9b8fbd48
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/9b8fbd48
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/9b8fbd48

Branch: refs/heads/branch-1
Commit: 9b8fbd4866f28aa3174a41f461a07afc330a172a
Parents: 39285e5
Author: Yongzhi Chen <ychena@apache.org>
Authored: Tue Feb 16 16:12:28 2016 -0500
Committer: Yongzhi Chen <ychena@apache.org>
Committed: Thu Feb 18 13:41:26 2016 -0500

----------------------------------------------------------------------
  .../hadoop/hive/hbase/HBaseRowSerializer.java | 7 +-
  .../src/test/queries/positive/hbase_queries.q | 26 ++++
  .../test/results/positive/hbase_queries.q.out | 154 +++++++++++++++++++
  3 files changed, 185 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/9b8fbd48/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HBaseRowSerializer.java
----------------------------------------------------------------------
diff --git a/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HBaseRowSerializer.java b/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HBaseRowSerializer.java
index 7c141fc..c6f3b0f 100644
--- a/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HBaseRowSerializer.java
+++ b/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HBaseRowSerializer.java
@@ -271,8 +271,11 @@ public class HBaseRowSerializer {
                ss.write(sep);
              }
              serialize(entry.getKey(), koi, level+2, ss);
- ss.write(keyValueSeparator);
- serialize(entry.getValue(), voi, level+2, ss);
+
+ if ( entry.getValue() != null) {
+ ss.write(keyValueSeparator);
+ serialize(entry.getValue(), voi, level+2, ss);
+ }
            }
          }
          return true;

http://git-wip-us.apache.org/repos/asf/hive/blob/9b8fbd48/hbase-handler/src/test/queries/positive/hbase_queries.q
----------------------------------------------------------------------
diff --git a/hbase-handler/src/test/queries/positive/hbase_queries.q b/hbase-handler/src/test/queries/positive/hbase_queries.q
index b445c4b..49fa829 100644
--- a/hbase-handler/src/test/queries/positive/hbase_queries.q
+++ b/hbase-handler/src/test/queries/positive/hbase_queries.q
@@ -164,6 +164,30 @@ SELECT COUNT(*) FROM hbase_table_1_like;

  SHOW CREATE TABLE hbase_table_1_like;

+DROP TABLE IF EXISTS hbase_table_9;
+CREATE TABLE hbase_table_9 (id bigint, data map<string, string>, str string)
+stored by 'org.apache.hadoop.hive.hbase.HBaseStorageHandler'
+with serdeproperties ("hbase.columns.mapping" = ":key,cf:map_col#s:s,cf:str_col");
+
+insert overwrite table hbase_table_9 select 1 as id, map('abcd', null) as data , null as str from src limit 1;
+insert into table hbase_table_9 select 2 as id, map('efgh', null) as data , '1234' as str from src limit 1;
+insert into table hbase_table_9 select 3 as id, map('hij', '') as data , '1234' as str from src limit 1;
+insert into table hbase_table_9 select 4 as id, map('klm', 'avalue') as data , '1234' as str from src limit 1;
+insert into table hbase_table_9 select 5 as id, map('key1',null, 'key2', 'avalue') as data , '1234' as str from src limit 1;
+select * from hbase_table_9;
+
+DROP TABLE IF EXISTS hbase_table_10;
+CREATE TABLE hbase_table_10 (id bigint, data map<int, int>, str string)
+stored by 'org.apache.hadoop.hive.hbase.HBaseStorageHandler'
+with serdeproperties ("hbase.columns.mapping" = ":key,cf:map_col2,cf:str2_col");
+insert overwrite table hbase_table_10 select 1 as id, map(10, cast(null as int)) as data , null as str from src limit 1;
+insert into table hbase_table_10 select 2 as id, map(20, cast(null as int)) as data , '1234' as str from src limit 1;
+insert into table hbase_table_10 select 3 as id, map(30, 31) as data , '1234' as str from src limit 1;
+insert into table hbase_table_10 select 4 as id, map(40, cast(null as int), 45, cast(null as int)) as data , '1234' as str from src limit 1;
+insert into table hbase_table_10 select 5 as id, map(50,cast(null as int), 55, 58) as data , '1234' as str from src limit 1;
+select * from hbase_table_10;
+
+
  DROP TABLE hbase_table_1;
  DROP TABLE hbase_table_1_like;
  DROP TABLE hbase_table_2;
@@ -176,3 +200,5 @@ DROP TABLE hbase_table_7;
  DROP TABLE hbase_table_8;
  DROP TABLE empty_hbase_table;
  DROP TABLE empty_normal_table;
+DROP TABLE hbase_table_9;
+DROP TABLE hbase_table_10;

http://git-wip-us.apache.org/repos/asf/hive/blob/9b8fbd48/hbase-handler/src/test/results/positive/hbase_queries.q.out
----------------------------------------------------------------------
diff --git a/hbase-handler/src/test/results/positive/hbase_queries.q.out b/hbase-handler/src/test/results/positive/hbase_queries.q.out
index d044c7e..4b51969 100644
--- a/hbase-handler/src/test/results/positive/hbase_queries.q.out
+++ b/hbase-handler/src/test/results/positive/hbase_queries.q.out
@@ -918,6 +918,144 @@ WITH SERDEPROPERTIES (
  TBLPROPERTIES (
    'hbase.table.name'='hbase_table_0',
  #### A masked pattern was here ####
+PREHOOK: query: DROP TABLE IF EXISTS hbase_table_9
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: DROP TABLE IF EXISTS hbase_table_9
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: CREATE TABLE hbase_table_9 (id bigint, data map<string, string>, str string)
+stored by 'org.apache.hadoop.hive.hbase.HBaseStorageHandler'
+with serdeproperties ("hbase.columns.mapping" = ":key,cf:map_col#s:s,cf:str_col")
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@hbase_table_9
+POSTHOOK: query: CREATE TABLE hbase_table_9 (id bigint, data map<string, string>, str string)
+stored by 'org.apache.hadoop.hive.hbase.HBaseStorageHandler'
+with serdeproperties ("hbase.columns.mapping" = ":key,cf:map_col#s:s,cf:str_col")
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@hbase_table_9
+PREHOOK: query: insert overwrite table hbase_table_9 select 1 as id, map('abcd', null) as data , null as str from src limit 1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Output: default@hbase_table_9
+POSTHOOK: query: insert overwrite table hbase_table_9 select 1 as id, map('abcd', null) as data , null as str from src limit 1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Output: default@hbase_table_9
+PREHOOK: query: insert into table hbase_table_9 select 2 as id, map('efgh', null) as data , '1234' as str from src limit 1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Output: default@hbase_table_9
+POSTHOOK: query: insert into table hbase_table_9 select 2 as id, map('efgh', null) as data , '1234' as str from src limit 1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Output: default@hbase_table_9
+PREHOOK: query: insert into table hbase_table_9 select 3 as id, map('hij', '') as data , '1234' as str from src limit 1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Output: default@hbase_table_9
+POSTHOOK: query: insert into table hbase_table_9 select 3 as id, map('hij', '') as data , '1234' as str from src limit 1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Output: default@hbase_table_9
+PREHOOK: query: insert into table hbase_table_9 select 4 as id, map('klm', 'avalue') as data , '1234' as str from src limit 1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Output: default@hbase_table_9
+POSTHOOK: query: insert into table hbase_table_9 select 4 as id, map('klm', 'avalue') as data , '1234' as str from src limit 1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Output: default@hbase_table_9
+PREHOOK: query: insert into table hbase_table_9 select 5 as id, map('key1',null, 'key2', 'avalue') as data , '1234' as str from src limit 1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Output: default@hbase_table_9
+POSTHOOK: query: insert into table hbase_table_9 select 5 as id, map('key1',null, 'key2', 'avalue') as data , '1234' as str from src limit 1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Output: default@hbase_table_9
+PREHOOK: query: select * from hbase_table_9
+PREHOOK: type: QUERY
+PREHOOK: Input: default@hbase_table_9
+#### A masked pattern was here ####
+POSTHOOK: query: select * from hbase_table_9
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@hbase_table_9
+#### A masked pattern was here ####
+1 {"abcd":null} NULL
+2 {"efgh":null} 1234
+3 {"hij":""} 1234
+4 {"klm":"avalue"} 1234
+5 {"key1":null,"key2":"avalue"} 1234
+PREHOOK: query: DROP TABLE IF EXISTS hbase_table_10
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: DROP TABLE IF EXISTS hbase_table_10
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: CREATE TABLE hbase_table_10 (id bigint, data map<int, int>, str string)
+stored by 'org.apache.hadoop.hive.hbase.HBaseStorageHandler'
+with serdeproperties ("hbase.columns.mapping" = ":key,cf:map_col2,cf:str2_col")
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@hbase_table_10
+POSTHOOK: query: CREATE TABLE hbase_table_10 (id bigint, data map<int, int>, str string)
+stored by 'org.apache.hadoop.hive.hbase.HBaseStorageHandler'
+with serdeproperties ("hbase.columns.mapping" = ":key,cf:map_col2,cf:str2_col")
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@hbase_table_10
+PREHOOK: query: insert overwrite table hbase_table_10 select 1 as id, map(10, cast(null as int)) as data , null as str from src limit 1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Output: default@hbase_table_10
+POSTHOOK: query: insert overwrite table hbase_table_10 select 1 as id, map(10, cast(null as int)) as data , null as str from src limit 1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Output: default@hbase_table_10
+PREHOOK: query: insert into table hbase_table_10 select 2 as id, map(20, cast(null as int)) as data , '1234' as str from src limit 1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Output: default@hbase_table_10
+POSTHOOK: query: insert into table hbase_table_10 select 2 as id, map(20, cast(null as int)) as data , '1234' as str from src limit 1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Output: default@hbase_table_10
+PREHOOK: query: insert into table hbase_table_10 select 3 as id, map(30, 31) as data , '1234' as str from src limit 1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Output: default@hbase_table_10
+POSTHOOK: query: insert into table hbase_table_10 select 3 as id, map(30, 31) as data , '1234' as str from src limit 1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Output: default@hbase_table_10
+PREHOOK: query: insert into table hbase_table_10 select 4 as id, map(40, cast(null as int), 45, cast(null as int)) as data , '1234' as str from src limit 1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Output: default@hbase_table_10
+POSTHOOK: query: insert into table hbase_table_10 select 4 as id, map(40, cast(null as int), 45, cast(null as int)) as data , '1234' as str from src limit 1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Output: default@hbase_table_10
+PREHOOK: query: insert into table hbase_table_10 select 5 as id, map(50,cast(null as int), 55, 58) as data , '1234' as str from src limit 1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Output: default@hbase_table_10
+POSTHOOK: query: insert into table hbase_table_10 select 5 as id, map(50,cast(null as int), 55, 58) as data , '1234' as str from src limit 1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Output: default@hbase_table_10
+PREHOOK: query: select * from hbase_table_10
+PREHOOK: type: QUERY
+PREHOOK: Input: default@hbase_table_10
+#### A masked pattern was here ####
+POSTHOOK: query: select * from hbase_table_10
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@hbase_table_10
+#### A masked pattern was here ####
+1 {10:null} NULL
+2 {20:null} 1234
+3 {30:31} 1234
+4 {40:null,45:null} 1234
+5 {50:null,55:58} 1234
  PREHOOK: query: DROP TABLE hbase_table_1
  PREHOOK: type: DROPTABLE
  PREHOOK: Input: default@hbase_table_1
@@ -1014,3 +1152,19 @@ POSTHOOK: query: DROP TABLE empty_normal_table
  POSTHOOK: type: DROPTABLE
  POSTHOOK: Input: default@empty_normal_table
  POSTHOOK: Output: default@empty_normal_table
+PREHOOK: query: DROP TABLE hbase_table_9
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@hbase_table_9
+PREHOOK: Output: default@hbase_table_9
+POSTHOOK: query: DROP TABLE hbase_table_9
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@hbase_table_9
+POSTHOOK: Output: default@hbase_table_9
+PREHOOK: query: DROP TABLE hbase_table_10
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@hbase_table_10
+PREHOOK: Output: default@hbase_table_10
+POSTHOOK: query: DROP TABLE hbase_table_10
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@hbase_table_10
+POSTHOOK: Output: default@hbase_table_10

Search Discussions

Discussion Posts

Previous

Related Discussions

Discussion Navigation
viewthread | post
posts ‹ prev | 2 of 2 | next ›
Discussion Overview
groupcommits @
categorieshive, hadoop
postedFeb 18, '16 at 6:40p
activeFeb 18, '16 at 6:55p
posts2
users1
websitehive.apache.org

1 user in discussion

Ychena: 2 posts

People

Translate

site design / logo © 2021 Grokbase