FAQ
Repository: hive
Updated Branches:
   refs/heads/master 4ee17e53c -> 0ec5185a2


HIVE-11214: Insert into ACID table switches vectorization off (Matt McCline, reviewed by Eugene Koifman)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/0ec5185a
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/0ec5185a
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/0ec5185a

Branch: refs/heads/master
Commit: 0ec5185a28177a04cf4f59a38b7c93da3b6e4565
Parents: 4ee17e5
Author: Matt McCline <mmccline@hortonworks.com>
Authored: Wed Jul 29 15:03:20 2015 -0700
Committer: Matt McCline <mmccline@hortonworks.com>
Committed: Wed Jul 29 15:03:52 2015 -0700

----------------------------------------------------------------------
  .../test/resources/testconfiguration.properties | 1 +
  .../hadoop/hive/ql/parse/SemanticAnalyzer.java | 4 ---
  .../test/queries/clientpositive/vector_acid3.q | 17 +++++++++++
  .../clientpositive/tez/vector_acid3.q.out | 31 ++++++++++++++++++++
  .../results/clientpositive/vector_acid3.q.out | 31 ++++++++++++++++++++
  5 files changed, 80 insertions(+), 4 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/0ec5185a/itests/src/test/resources/testconfiguration.properties
----------------------------------------------------------------------
diff --git a/itests/src/test/resources/testconfiguration.properties b/itests/src/test/resources/testconfiguration.properties
index fbde465..aabf6fc 100644
--- a/itests/src/test/resources/testconfiguration.properties
+++ b/itests/src/test/resources/testconfiguration.properties
@@ -186,6 +186,7 @@ minitez.query.files.shared=alter_merge_2_orc.q,\
    update_where_non_partitioned.q,\
    update_where_partitioned.q,\
    update_two_cols.q,\
+ vector_acid3.q,\
    vector_aggregate_9.q,\
    vector_between_in.q,\
    vector_binary_join_groupby.q,\

http://git-wip-us.apache.org/repos/asf/hive/blob/0ec5185a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
index ad4efef..f05407d 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
@@ -6678,10 +6678,6 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
        LOG.debug("Couldn't find table " + tableName + " in insertIntoTable");
        throw new SemanticException(ErrorMsg.NO_INSERT_OVERWRITE_WITH_ACID.getMsg());
      }
- if (conf.getBoolVar(ConfVars.HIVE_VECTORIZATION_ENABLED)) {
- LOG.info("Turning off vectorization for acid write operation");
- conf.setBoolVar(ConfVars.HIVE_VECTORIZATION_ENABLED, false);
- }
      LOG.info("Modifying config values for ACID write");
      conf.setBoolVar(ConfVars.HIVEOPTREDUCEDEDUPLICATION, true);
      conf.setIntVar(ConfVars.HIVEOPTREDUCEDEDUPLICATIONMINREDUCER, 1);

http://git-wip-us.apache.org/repos/asf/hive/blob/0ec5185a/ql/src/test/queries/clientpositive/vector_acid3.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/vector_acid3.q b/ql/src/test/queries/clientpositive/vector_acid3.q
new file mode 100644
index 0000000..d4313f4
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/vector_acid3.q
@@ -0,0 +1,17 @@
+set hive.support.concurrency=true;
+set hive.txn.manager=org.apache.hadoop.hive.ql.lockmgr.DbTxnManager;
+set hive.exec.dynamic.partition.mode=nonstrict;
+set hive.exec.dynamic.partition=true;
+set hive.vectorized.execution.enabled=true;
+
+drop table if exists testacid1;
+
+create table testacid1(id int) clustered by (id) into 2 buckets stored as orc tblproperties("transactional"="true");
+
+insert into table testacid1 values (1),(2),(3),(4);
+
+set hive.compute.query.using.stats=false;
+
+set hive.vectorized.execution.enabled;
+
+select count(1) from testacid1;

http://git-wip-us.apache.org/repos/asf/hive/blob/0ec5185a/ql/src/test/results/clientpositive/tez/vector_acid3.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/tez/vector_acid3.q.out b/ql/src/test/results/clientpositive/tez/vector_acid3.q.out
new file mode 100644
index 0000000..4299c73
--- /dev/null
+++ b/ql/src/test/results/clientpositive/tez/vector_acid3.q.out
@@ -0,0 +1,31 @@
+PREHOOK: query: drop table if exists testacid1
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: drop table if exists testacid1
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: create table testacid1(id int) clustered by (id) into 2 buckets stored as orc tblproperties("transactional"="true")
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@testacid1
+POSTHOOK: query: create table testacid1(id int) clustered by (id) into 2 buckets stored as orc tblproperties("transactional"="true")
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@testacid1
+PREHOOK: query: insert into table testacid1 values (1),(2),(3),(4)
+PREHOOK: type: QUERY
+PREHOOK: Input: default@values__tmp__table__1
+PREHOOK: Output: default@testacid1
+POSTHOOK: query: insert into table testacid1 values (1),(2),(3),(4)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@values__tmp__table__1
+POSTHOOK: Output: default@testacid1
+POSTHOOK: Lineage: testacid1.id EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col1, type:string, comment:), ]
+hive.vectorized.execution.enabled=true
+PREHOOK: query: select count(1) from testacid1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@testacid1
+#### A masked pattern was here ####
+POSTHOOK: query: select count(1) from testacid1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@testacid1
+#### A masked pattern was here ####
+4

http://git-wip-us.apache.org/repos/asf/hive/blob/0ec5185a/ql/src/test/results/clientpositive/vector_acid3.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/vector_acid3.q.out b/ql/src/test/results/clientpositive/vector_acid3.q.out
new file mode 100644
index 0000000..4299c73
--- /dev/null
+++ b/ql/src/test/results/clientpositive/vector_acid3.q.out
@@ -0,0 +1,31 @@
+PREHOOK: query: drop table if exists testacid1
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: drop table if exists testacid1
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: create table testacid1(id int) clustered by (id) into 2 buckets stored as orc tblproperties("transactional"="true")
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@testacid1
+POSTHOOK: query: create table testacid1(id int) clustered by (id) into 2 buckets stored as orc tblproperties("transactional"="true")
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@testacid1
+PREHOOK: query: insert into table testacid1 values (1),(2),(3),(4)
+PREHOOK: type: QUERY
+PREHOOK: Input: default@values__tmp__table__1
+PREHOOK: Output: default@testacid1
+POSTHOOK: query: insert into table testacid1 values (1),(2),(3),(4)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@values__tmp__table__1
+POSTHOOK: Output: default@testacid1
+POSTHOOK: Lineage: testacid1.id EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col1, type:string, comment:), ]
+hive.vectorized.execution.enabled=true
+PREHOOK: query: select count(1) from testacid1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@testacid1
+#### A masked pattern was here ####
+POSTHOOK: query: select count(1) from testacid1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@testacid1
+#### A masked pattern was here ####
+4

Search Discussions

  • Mmccline at Jul 29, 2015 at 11:23 pm
    Repository: hive
    Updated Branches:
       refs/heads/branch-1 d5f81a423 -> fb57bd134


    HIVE-11214: Insert into ACID table switches vectorization off (Matt McCline, reviewed by Eugene Koifman)


    Project: http://git-wip-us.apache.org/repos/asf/hive/repo
    Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/fb57bd13
    Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/fb57bd13
    Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/fb57bd13

    Branch: refs/heads/branch-1
    Commit: fb57bd13452f676e603279dd75645526c17cc2e5
    Parents: d5f81a4
    Author: Matt McCline <mmccline@hortonworks.com>
    Authored: Wed Jul 29 16:23:33 2015 -0700
    Committer: Matt McCline <mmccline@hortonworks.com>
    Committed: Wed Jul 29 16:23:33 2015 -0700

    ----------------------------------------------------------------------
      .../test/resources/testconfiguration.properties | 1 +
      .../hadoop/hive/ql/parse/SemanticAnalyzer.java | 4 ---
      .../test/queries/clientpositive/vector_acid3.q | 17 +++++++++++
      .../clientpositive/tez/vector_acid3.q.out | 31 ++++++++++++++++++++
      .../results/clientpositive/vector_acid3.q.out | 31 ++++++++++++++++++++
      5 files changed, 80 insertions(+), 4 deletions(-)
    ----------------------------------------------------------------------


    http://git-wip-us.apache.org/repos/asf/hive/blob/fb57bd13/itests/src/test/resources/testconfiguration.properties
    ----------------------------------------------------------------------
    diff --git a/itests/src/test/resources/testconfiguration.properties b/itests/src/test/resources/testconfiguration.properties
    index 75c1a5c..98fd87b 100644
    --- a/itests/src/test/resources/testconfiguration.properties
    +++ b/itests/src/test/resources/testconfiguration.properties
    @@ -186,6 +186,7 @@ minitez.query.files.shared=alter_merge_2_orc.q,\
        update_where_non_partitioned.q,\
        update_where_partitioned.q,\
        update_two_cols.q,\
    + vector_acid3.q,\
        vector_aggregate_9.q,\
        vector_between_in.q,\
        vector_binary_join_groupby.q,\

    http://git-wip-us.apache.org/repos/asf/hive/blob/fb57bd13/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
    ----------------------------------------------------------------------
    diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
    index 2ae6309..b4878bf 100644
    --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
    +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
    @@ -6690,10 +6690,6 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
            LOG.debug("Couldn't find table " + tableName + " in insertIntoTable");
            throw new SemanticException(ErrorMsg.NO_INSERT_OVERWRITE_WITH_ACID.getMsg());
          }
    - if (conf.getBoolVar(ConfVars.HIVE_VECTORIZATION_ENABLED)) {
    - LOG.info("Turning off vectorization for acid write operation");
    - conf.setBoolVar(ConfVars.HIVE_VECTORIZATION_ENABLED, false);
    - }
          LOG.info("Modifying config values for ACID write");
          conf.setBoolVar(ConfVars.HIVEOPTREDUCEDEDUPLICATION, true);
          conf.setIntVar(ConfVars.HIVEOPTREDUCEDEDUPLICATIONMINREDUCER, 1);

    http://git-wip-us.apache.org/repos/asf/hive/blob/fb57bd13/ql/src/test/queries/clientpositive/vector_acid3.q
    ----------------------------------------------------------------------
    diff --git a/ql/src/test/queries/clientpositive/vector_acid3.q b/ql/src/test/queries/clientpositive/vector_acid3.q
    new file mode 100644
    index 0000000..d4313f4
    --- /dev/null
    +++ b/ql/src/test/queries/clientpositive/vector_acid3.q
    @@ -0,0 +1,17 @@
    +set hive.support.concurrency=true;
    +set hive.txn.manager=org.apache.hadoop.hive.ql.lockmgr.DbTxnManager;
    +set hive.exec.dynamic.partition.mode=nonstrict;
    +set hive.exec.dynamic.partition=true;
    +set hive.vectorized.execution.enabled=true;
    +
    +drop table if exists testacid1;
    +
    +create table testacid1(id int) clustered by (id) into 2 buckets stored as orc tblproperties("transactional"="true");
    +
    +insert into table testacid1 values (1),(2),(3),(4);
    +
    +set hive.compute.query.using.stats=false;
    +
    +set hive.vectorized.execution.enabled;
    +
    +select count(1) from testacid1;

    http://git-wip-us.apache.org/repos/asf/hive/blob/fb57bd13/ql/src/test/results/clientpositive/tez/vector_acid3.q.out
    ----------------------------------------------------------------------
    diff --git a/ql/src/test/results/clientpositive/tez/vector_acid3.q.out b/ql/src/test/results/clientpositive/tez/vector_acid3.q.out
    new file mode 100644
    index 0000000..4299c73
    --- /dev/null
    +++ b/ql/src/test/results/clientpositive/tez/vector_acid3.q.out
    @@ -0,0 +1,31 @@
    +PREHOOK: query: drop table if exists testacid1
    +PREHOOK: type: DROPTABLE
    +POSTHOOK: query: drop table if exists testacid1
    +POSTHOOK: type: DROPTABLE
    +PREHOOK: query: create table testacid1(id int) clustered by (id) into 2 buckets stored as orc tblproperties("transactional"="true")
    +PREHOOK: type: CREATETABLE
    +PREHOOK: Output: database:default
    +PREHOOK: Output: default@testacid1
    +POSTHOOK: query: create table testacid1(id int) clustered by (id) into 2 buckets stored as orc tblproperties("transactional"="true")
    +POSTHOOK: type: CREATETABLE
    +POSTHOOK: Output: database:default
    +POSTHOOK: Output: default@testacid1
    +PREHOOK: query: insert into table testacid1 values (1),(2),(3),(4)
    +PREHOOK: type: QUERY
    +PREHOOK: Input: default@values__tmp__table__1
    +PREHOOK: Output: default@testacid1
    +POSTHOOK: query: insert into table testacid1 values (1),(2),(3),(4)
    +POSTHOOK: type: QUERY
    +POSTHOOK: Input: default@values__tmp__table__1
    +POSTHOOK: Output: default@testacid1
    +POSTHOOK: Lineage: testacid1.id EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col1, type:string, comment:), ]
    +hive.vectorized.execution.enabled=true
    +PREHOOK: query: select count(1) from testacid1
    +PREHOOK: type: QUERY
    +PREHOOK: Input: default@testacid1
    +#### A masked pattern was here ####
    +POSTHOOK: query: select count(1) from testacid1
    +POSTHOOK: type: QUERY
    +POSTHOOK: Input: default@testacid1
    +#### A masked pattern was here ####
    +4

    http://git-wip-us.apache.org/repos/asf/hive/blob/fb57bd13/ql/src/test/results/clientpositive/vector_acid3.q.out
    ----------------------------------------------------------------------
    diff --git a/ql/src/test/results/clientpositive/vector_acid3.q.out b/ql/src/test/results/clientpositive/vector_acid3.q.out
    new file mode 100644
    index 0000000..4299c73
    --- /dev/null
    +++ b/ql/src/test/results/clientpositive/vector_acid3.q.out
    @@ -0,0 +1,31 @@
    +PREHOOK: query: drop table if exists testacid1
    +PREHOOK: type: DROPTABLE
    +POSTHOOK: query: drop table if exists testacid1
    +POSTHOOK: type: DROPTABLE
    +PREHOOK: query: create table testacid1(id int) clustered by (id) into 2 buckets stored as orc tblproperties("transactional"="true")
    +PREHOOK: type: CREATETABLE
    +PREHOOK: Output: database:default
    +PREHOOK: Output: default@testacid1
    +POSTHOOK: query: create table testacid1(id int) clustered by (id) into 2 buckets stored as orc tblproperties("transactional"="true")
    +POSTHOOK: type: CREATETABLE
    +POSTHOOK: Output: database:default
    +POSTHOOK: Output: default@testacid1
    +PREHOOK: query: insert into table testacid1 values (1),(2),(3),(4)
    +PREHOOK: type: QUERY
    +PREHOOK: Input: default@values__tmp__table__1
    +PREHOOK: Output: default@testacid1
    +POSTHOOK: query: insert into table testacid1 values (1),(2),(3),(4)
    +POSTHOOK: type: QUERY
    +POSTHOOK: Input: default@values__tmp__table__1
    +POSTHOOK: Output: default@testacid1
    +POSTHOOK: Lineage: testacid1.id EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col1, type:string, comment:), ]
    +hive.vectorized.execution.enabled=true
    +PREHOOK: query: select count(1) from testacid1
    +PREHOOK: type: QUERY
    +PREHOOK: Input: default@testacid1
    +#### A masked pattern was here ####
    +POSTHOOK: query: select count(1) from testacid1
    +POSTHOOK: type: QUERY
    +POSTHOOK: Input: default@testacid1
    +#### A masked pattern was here ####
    +4

Related Discussions

Discussion Navigation
viewthread | post
Discussion Overview
groupcommits @
categorieshive, hadoop
postedJul 29, '15 at 10:04p
activeJul 29, '15 at 11:23p
posts2
users1
websitehive.apache.org

1 user in discussion

Mmccline: 2 posts

People

Translate

site design / logo © 2021 Grokbase