FAQ
Repository: hive
Updated Branches:
   refs/heads/master 9d84ad4a1 -> 6ab945c52


HIVE-14006: Hive query with UNION ALL fails with ArrayIndexOutOfBoundsException. (Naveen Gangam via Yongzhi Chen)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/6ab945c5
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/6ab945c5
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/6ab945c5

Branch: refs/heads/master
Commit: 6ab945c5225be915327d6a0d2126f3b41c50caba
Parents: 9d84ad4
Author: Yongzhi Chen <ychena@apache.org>
Authored: Thu Jun 16 10:56:56 2016 -0400
Committer: Yongzhi Chen <ychena@apache.org>
Committed: Thu Jun 16 10:56:56 2016 -0400

----------------------------------------------------------------------
  .../hadoop/hive/ql/exec/UnionOperator.java | 2 +-
  .../clientpositive/unionall_join_nullconstant.q | 41 ++++++
  .../unionall_join_nullconstant.q.out | 135 +++++++++++++++++++
  3 files changed, 177 insertions(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/6ab945c5/ql/src/java/org/apache/hadoop/hive/ql/exec/UnionOperator.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/UnionOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/UnionOperator.java
index 39b2776..3df5533 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/UnionOperator.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/UnionOperator.java
@@ -139,7 +139,7 @@ public class UnionOperator extends Operator<UnionDesc> implements Serializable {
      StructObjectInspector soi = parentObjInspectors[tag];
      List<? extends StructField> fields = parentFields[tag];

- if (needsTransform[tag]) {
+ if (needsTransform[tag] && outputRow.size() > 0) {
        for (int c = 0; c < fields.size(); c++) {
          outputRow.set(c, columnTypeResolvers[c].convertIfNecessary(soi
              .getStructFieldData(row, fields.get(c)), fields.get(c)

http://git-wip-us.apache.org/repos/asf/hive/blob/6ab945c5/ql/src/test/queries/clientpositive/unionall_join_nullconstant.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/unionall_join_nullconstant.q b/ql/src/test/queries/clientpositive/unionall_join_nullconstant.q
new file mode 100644
index 0000000..4f0ffa6
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/unionall_join_nullconstant.q
@@ -0,0 +1,41 @@
+set hive.cbo.enable=false;
+DROP VIEW IF EXISTS a_view;
+
+DROP TABLE IF EXISTS table_a1;
+DROP TABLE IF EXISTS table_a2;
+DROP TABLE IF EXISTS table_b1;
+DROP TABLE IF EXISTS table_b2;
+
+CREATE TABLE table_a1
+(composite_key STRING);
+
+CREATE TABLE table_a2
+(composite_key STRING);
+
+CREATE TABLE table_b1
+(composite_key STRING, col1 STRING);
+
+CREATE TABLE table_b2
+(composite_key STRING);
+
+CREATE VIEW a_view AS
+SELECT
+substring(a1.composite_key, 1, locate('|',a1.composite_key) - 1) AS autoname,
+NULL AS col1
+FROM table_a1 a1
+FULL OUTER JOIN table_a2 a2
+ON a1.composite_key = a2.composite_key
+UNION ALL
+SELECT
+substring(b1.composite_key, 1, locate('|',b1.composite_key) - 1) AS autoname,
+b1.col1 AS col1
+FROM table_b1 b1
+FULL OUTER JOIN table_b2 b2
+ON b1.composite_key = b2.composite_key;
+
+INSERT INTO TABLE table_b1
+SELECT * FROM (
+SELECT 'something|awful', 'col1'
+)s ;
+
+SELECT autoname FROM a_view WHERE autoname='something';

http://git-wip-us.apache.org/repos/asf/hive/blob/6ab945c5/ql/src/test/results/clientpositive/unionall_join_nullconstant.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/unionall_join_nullconstant.q.out b/ql/src/test/results/clientpositive/unionall_join_nullconstant.q.out
new file mode 100644
index 0000000..fca26b4
--- /dev/null
+++ b/ql/src/test/results/clientpositive/unionall_join_nullconstant.q.out
@@ -0,0 +1,135 @@
+PREHOOK: query: DROP VIEW IF EXISTS a_view
+PREHOOK: type: DROPVIEW
+POSTHOOK: query: DROP VIEW IF EXISTS a_view
+POSTHOOK: type: DROPVIEW
+PREHOOK: query: DROP TABLE IF EXISTS table_a1
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: DROP TABLE IF EXISTS table_a1
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: DROP TABLE IF EXISTS table_a2
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: DROP TABLE IF EXISTS table_a2
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: DROP TABLE IF EXISTS table_b1
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: DROP TABLE IF EXISTS table_b1
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: DROP TABLE IF EXISTS table_b2
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: DROP TABLE IF EXISTS table_b2
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: CREATE TABLE table_a1
+(composite_key STRING)
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@table_a1
+POSTHOOK: query: CREATE TABLE table_a1
+(composite_key STRING)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@table_a1
+PREHOOK: query: CREATE TABLE table_a2
+(composite_key STRING)
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@table_a2
+POSTHOOK: query: CREATE TABLE table_a2
+(composite_key STRING)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@table_a2
+PREHOOK: query: CREATE TABLE table_b1
+(composite_key STRING, col1 STRING)
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@table_b1
+POSTHOOK: query: CREATE TABLE table_b1
+(composite_key STRING, col1 STRING)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@table_b1
+PREHOOK: query: CREATE TABLE table_b2
+(composite_key STRING)
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@table_b2
+POSTHOOK: query: CREATE TABLE table_b2
+(composite_key STRING)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@table_b2
+PREHOOK: query: CREATE VIEW a_view AS
+SELECT
+substring(a1.composite_key, 1, locate('|',a1.composite_key) - 1) AS autoname,
+NULL AS col1
+FROM table_a1 a1
+FULL OUTER JOIN table_a2 a2
+ON a1.composite_key = a2.composite_key
+UNION ALL
+SELECT
+substring(b1.composite_key, 1, locate('|',b1.composite_key) - 1) AS autoname,
+b1.col1 AS col1
+FROM table_b1 b1
+FULL OUTER JOIN table_b2 b2
+ON b1.composite_key = b2.composite_key
+PREHOOK: type: CREATEVIEW
+PREHOOK: Input: default@table_a1
+PREHOOK: Input: default@table_a2
+PREHOOK: Input: default@table_b1
+PREHOOK: Input: default@table_b2
+PREHOOK: Output: database:default
+PREHOOK: Output: default@a_view
+POSTHOOK: query: CREATE VIEW a_view AS
+SELECT
+substring(a1.composite_key, 1, locate('|',a1.composite_key) - 1) AS autoname,
+NULL AS col1
+FROM table_a1 a1
+FULL OUTER JOIN table_a2 a2
+ON a1.composite_key = a2.composite_key
+UNION ALL
+SELECT
+substring(b1.composite_key, 1, locate('|',b1.composite_key) - 1) AS autoname,
+b1.col1 AS col1
+FROM table_b1 b1
+FULL OUTER JOIN table_b2 b2
+ON b1.composite_key = b2.composite_key
+POSTHOOK: type: CREATEVIEW
+POSTHOOK: Input: default@table_a1
+POSTHOOK: Input: default@table_a2
+POSTHOOK: Input: default@table_b1
+POSTHOOK: Input: default@table_b2
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@a_view
+PREHOOK: query: INSERT INTO TABLE table_b1
+SELECT * FROM (
+SELECT 'something|awful', 'col1'
+)s
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@table_b1
+POSTHOOK: query: INSERT INTO TABLE table_b1
+SELECT * FROM (
+SELECT 'something|awful', 'col1'
+)s
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@table_b1
+POSTHOOK: Lineage: table_b1.col1 SIMPLE []
+POSTHOOK: Lineage: table_b1.composite_key SIMPLE []
+PREHOOK: query: SELECT autoname FROM a_view WHERE autoname='something'
+PREHOOK: type: QUERY
+PREHOOK: Input: default@a_view
+PREHOOK: Input: default@table_a1
+PREHOOK: Input: default@table_a2
+PREHOOK: Input: default@table_b1
+PREHOOK: Input: default@table_b2
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT autoname FROM a_view WHERE autoname='something'
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@a_view
+POSTHOOK: Input: default@table_a1
+POSTHOOK: Input: default@table_a2
+POSTHOOK: Input: default@table_b1
+POSTHOOK: Input: default@table_b2
+#### A masked pattern was here ####
+something

Search Discussions

  • Ychena at Jun 16, 2016 at 3:13 pm
    Repository: hive
    Updated Branches:
       refs/heads/branch-2.1 74d2bdc80 -> 5633df3c2


    HIVE-14006: Hive query with UNION ALL fails with ArrayIndexOutOfBoundsException. (Naveen Gangam via Yongzhi Chen)


    Project: http://git-wip-us.apache.org/repos/asf/hive/repo
    Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/5633df3c
    Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/5633df3c
    Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/5633df3c

    Branch: refs/heads/branch-2.1
    Commit: 5633df3c2c2c9a3a41ac1f498ee497a24a81ce86
    Parents: 74d2bdc
    Author: Yongzhi Chen <ychena@apache.org>
    Authored: Thu Jun 16 10:56:56 2016 -0400
    Committer: Yongzhi Chen <ychena@apache.org>
    Committed: Thu Jun 16 11:07:15 2016 -0400

    ----------------------------------------------------------------------
      .../hadoop/hive/ql/exec/UnionOperator.java | 2 +-
      .../clientpositive/unionall_join_nullconstant.q | 41 ++++++
      .../unionall_join_nullconstant.q.out | 135 +++++++++++++++++++
      3 files changed, 177 insertions(+), 1 deletion(-)
    ----------------------------------------------------------------------


    http://git-wip-us.apache.org/repos/asf/hive/blob/5633df3c/ql/src/java/org/apache/hadoop/hive/ql/exec/UnionOperator.java
    ----------------------------------------------------------------------
    diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/UnionOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/UnionOperator.java
    index 39b2776..3df5533 100644
    --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/UnionOperator.java
    +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/UnionOperator.java
    @@ -139,7 +139,7 @@ public class UnionOperator extends Operator<UnionDesc> implements Serializable {
          StructObjectInspector soi = parentObjInspectors[tag];
          List<? extends StructField> fields = parentFields[tag];

    - if (needsTransform[tag]) {
    + if (needsTransform[tag] && outputRow.size() > 0) {
            for (int c = 0; c < fields.size(); c++) {
              outputRow.set(c, columnTypeResolvers[c].convertIfNecessary(soi
                  .getStructFieldData(row, fields.get(c)), fields.get(c)

    http://git-wip-us.apache.org/repos/asf/hive/blob/5633df3c/ql/src/test/queries/clientpositive/unionall_join_nullconstant.q
    ----------------------------------------------------------------------
    diff --git a/ql/src/test/queries/clientpositive/unionall_join_nullconstant.q b/ql/src/test/queries/clientpositive/unionall_join_nullconstant.q
    new file mode 100644
    index 0000000..4f0ffa6
    --- /dev/null
    +++ b/ql/src/test/queries/clientpositive/unionall_join_nullconstant.q
    @@ -0,0 +1,41 @@
    +set hive.cbo.enable=false;
    +DROP VIEW IF EXISTS a_view;
    +
    +DROP TABLE IF EXISTS table_a1;
    +DROP TABLE IF EXISTS table_a2;
    +DROP TABLE IF EXISTS table_b1;
    +DROP TABLE IF EXISTS table_b2;
    +
    +CREATE TABLE table_a1
    +(composite_key STRING);
    +
    +CREATE TABLE table_a2
    +(composite_key STRING);
    +
    +CREATE TABLE table_b1
    +(composite_key STRING, col1 STRING);
    +
    +CREATE TABLE table_b2
    +(composite_key STRING);
    +
    +CREATE VIEW a_view AS
    +SELECT
    +substring(a1.composite_key, 1, locate('|',a1.composite_key) - 1) AS autoname,
    +NULL AS col1
    +FROM table_a1 a1
    +FULL OUTER JOIN table_a2 a2
    +ON a1.composite_key = a2.composite_key
    +UNION ALL
    +SELECT
    +substring(b1.composite_key, 1, locate('|',b1.composite_key) - 1) AS autoname,
    +b1.col1 AS col1
    +FROM table_b1 b1
    +FULL OUTER JOIN table_b2 b2
    +ON b1.composite_key = b2.composite_key;
    +
    +INSERT INTO TABLE table_b1
    +SELECT * FROM (
    +SELECT 'something|awful', 'col1'
    +)s ;
    +
    +SELECT autoname FROM a_view WHERE autoname='something';

    http://git-wip-us.apache.org/repos/asf/hive/blob/5633df3c/ql/src/test/results/clientpositive/unionall_join_nullconstant.q.out
    ----------------------------------------------------------------------
    diff --git a/ql/src/test/results/clientpositive/unionall_join_nullconstant.q.out b/ql/src/test/results/clientpositive/unionall_join_nullconstant.q.out
    new file mode 100644
    index 0000000..fca26b4
    --- /dev/null
    +++ b/ql/src/test/results/clientpositive/unionall_join_nullconstant.q.out
    @@ -0,0 +1,135 @@
    +PREHOOK: query: DROP VIEW IF EXISTS a_view
    +PREHOOK: type: DROPVIEW
    +POSTHOOK: query: DROP VIEW IF EXISTS a_view
    +POSTHOOK: type: DROPVIEW
    +PREHOOK: query: DROP TABLE IF EXISTS table_a1
    +PREHOOK: type: DROPTABLE
    +POSTHOOK: query: DROP TABLE IF EXISTS table_a1
    +POSTHOOK: type: DROPTABLE
    +PREHOOK: query: DROP TABLE IF EXISTS table_a2
    +PREHOOK: type: DROPTABLE
    +POSTHOOK: query: DROP TABLE IF EXISTS table_a2
    +POSTHOOK: type: DROPTABLE
    +PREHOOK: query: DROP TABLE IF EXISTS table_b1
    +PREHOOK: type: DROPTABLE
    +POSTHOOK: query: DROP TABLE IF EXISTS table_b1
    +POSTHOOK: type: DROPTABLE
    +PREHOOK: query: DROP TABLE IF EXISTS table_b2
    +PREHOOK: type: DROPTABLE
    +POSTHOOK: query: DROP TABLE IF EXISTS table_b2
    +POSTHOOK: type: DROPTABLE
    +PREHOOK: query: CREATE TABLE table_a1
    +(composite_key STRING)
    +PREHOOK: type: CREATETABLE
    +PREHOOK: Output: database:default
    +PREHOOK: Output: default@table_a1
    +POSTHOOK: query: CREATE TABLE table_a1
    +(composite_key STRING)
    +POSTHOOK: type: CREATETABLE
    +POSTHOOK: Output: database:default
    +POSTHOOK: Output: default@table_a1
    +PREHOOK: query: CREATE TABLE table_a2
    +(composite_key STRING)
    +PREHOOK: type: CREATETABLE
    +PREHOOK: Output: database:default
    +PREHOOK: Output: default@table_a2
    +POSTHOOK: query: CREATE TABLE table_a2
    +(composite_key STRING)
    +POSTHOOK: type: CREATETABLE
    +POSTHOOK: Output: database:default
    +POSTHOOK: Output: default@table_a2
    +PREHOOK: query: CREATE TABLE table_b1
    +(composite_key STRING, col1 STRING)
    +PREHOOK: type: CREATETABLE
    +PREHOOK: Output: database:default
    +PREHOOK: Output: default@table_b1
    +POSTHOOK: query: CREATE TABLE table_b1
    +(composite_key STRING, col1 STRING)
    +POSTHOOK: type: CREATETABLE
    +POSTHOOK: Output: database:default
    +POSTHOOK: Output: default@table_b1
    +PREHOOK: query: CREATE TABLE table_b2
    +(composite_key STRING)
    +PREHOOK: type: CREATETABLE
    +PREHOOK: Output: database:default
    +PREHOOK: Output: default@table_b2
    +POSTHOOK: query: CREATE TABLE table_b2
    +(composite_key STRING)
    +POSTHOOK: type: CREATETABLE
    +POSTHOOK: Output: database:default
    +POSTHOOK: Output: default@table_b2
    +PREHOOK: query: CREATE VIEW a_view AS
    +SELECT
    +substring(a1.composite_key, 1, locate('|',a1.composite_key) - 1) AS autoname,
    +NULL AS col1
    +FROM table_a1 a1
    +FULL OUTER JOIN table_a2 a2
    +ON a1.composite_key = a2.composite_key
    +UNION ALL
    +SELECT
    +substring(b1.composite_key, 1, locate('|',b1.composite_key) - 1) AS autoname,
    +b1.col1 AS col1
    +FROM table_b1 b1
    +FULL OUTER JOIN table_b2 b2
    +ON b1.composite_key = b2.composite_key
    +PREHOOK: type: CREATEVIEW
    +PREHOOK: Input: default@table_a1
    +PREHOOK: Input: default@table_a2
    +PREHOOK: Input: default@table_b1
    +PREHOOK: Input: default@table_b2
    +PREHOOK: Output: database:default
    +PREHOOK: Output: default@a_view
    +POSTHOOK: query: CREATE VIEW a_view AS
    +SELECT
    +substring(a1.composite_key, 1, locate('|',a1.composite_key) - 1) AS autoname,
    +NULL AS col1
    +FROM table_a1 a1
    +FULL OUTER JOIN table_a2 a2
    +ON a1.composite_key = a2.composite_key
    +UNION ALL
    +SELECT
    +substring(b1.composite_key, 1, locate('|',b1.composite_key) - 1) AS autoname,
    +b1.col1 AS col1
    +FROM table_b1 b1
    +FULL OUTER JOIN table_b2 b2
    +ON b1.composite_key = b2.composite_key
    +POSTHOOK: type: CREATEVIEW
    +POSTHOOK: Input: default@table_a1
    +POSTHOOK: Input: default@table_a2
    +POSTHOOK: Input: default@table_b1
    +POSTHOOK: Input: default@table_b2
    +POSTHOOK: Output: database:default
    +POSTHOOK: Output: default@a_view
    +PREHOOK: query: INSERT INTO TABLE table_b1
    +SELECT * FROM (
    +SELECT 'something|awful', 'col1'
    +)s
    +PREHOOK: type: QUERY
    +PREHOOK: Input: _dummy_database@_dummy_table
    +PREHOOK: Output: default@table_b1
    +POSTHOOK: query: INSERT INTO TABLE table_b1
    +SELECT * FROM (
    +SELECT 'something|awful', 'col1'
    +)s
    +POSTHOOK: type: QUERY
    +POSTHOOK: Input: _dummy_database@_dummy_table
    +POSTHOOK: Output: default@table_b1
    +POSTHOOK: Lineage: table_b1.col1 SIMPLE []
    +POSTHOOK: Lineage: table_b1.composite_key SIMPLE []
    +PREHOOK: query: SELECT autoname FROM a_view WHERE autoname='something'
    +PREHOOK: type: QUERY
    +PREHOOK: Input: default@a_view
    +PREHOOK: Input: default@table_a1
    +PREHOOK: Input: default@table_a2
    +PREHOOK: Input: default@table_b1
    +PREHOOK: Input: default@table_b2
    +#### A masked pattern was here ####
    +POSTHOOK: query: SELECT autoname FROM a_view WHERE autoname='something'
    +POSTHOOK: type: QUERY
    +POSTHOOK: Input: default@a_view
    +POSTHOOK: Input: default@table_a1
    +POSTHOOK: Input: default@table_a2
    +POSTHOOK: Input: default@table_b1
    +POSTHOOK: Input: default@table_b2
    +#### A masked pattern was here ####
    +something

Related Discussions

Discussion Navigation
viewthread | post
Discussion Overview
groupcommits @
categorieshive, hadoop
postedJun 16, '16 at 3:06p
activeJun 16, '16 at 3:13p
posts2
users1
websitehive.apache.org

1 user in discussion

Ychena: 2 posts

People

Translate

site design / logo © 2021 Grokbase