FAQ
HIVE-13082: Enable constant propagation optimization in query with left semi join (Chaoyu Tang, reviewed by Aihua Xu and Gopal V)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/c659b202
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/c659b202
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/c659b202

Branch: refs/heads/master
Commit: c659b20264ec5c07102f18a5ab82a2f10a26afda
Parents: cf42684
Author: ctang <ctang.ma@gmail.com>
Authored: Mon Feb 22 16:25:57 2016 -0500
Committer: ctang <ctang.ma@gmail.com>
Committed: Mon Feb 22 16:25:57 2016 -0500

----------------------------------------------------------------------
  .../test/resources/testconfiguration.properties | 2 +
  .../optimizer/ConstantPropagateProcFactory.java | 3 +-
  .../queries/clientpositive/constprog_semijoin.q | 34 +
  .../clientpositive/constprog_partitioner.q.out | 12 +-
  .../clientpositive/constprog_semijoin.q.out | 814 +++++++++++++++++
  .../test/results/clientpositive/semijoin2.q.out | 4 +-
  .../test/results/clientpositive/semijoin4.q.out | 24 +-
  .../spark/constprog_partitioner.q.out | 12 +-
  .../spark/constprog_semijoin.q.out | 862 +++++++++++++++++++
  .../spark/vector_mapjoin_reduce.q.out | 12 +-
  .../tez/constprog_partitioner.q.out | 119 +++
  .../clientpositive/tez/constprog_semijoin.q.out | 492 +++++++++++
  .../clientpositive/tez/explainuser_1.q.out | 8 +-
  .../tez/vector_mapjoin_reduce.q.out | 12 +-
  .../clientpositive/vector_mapjoin_reduce.q.out | 12 +-
  15 files changed, 2373 insertions(+), 49 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/c659b202/itests/src/test/resources/testconfiguration.properties
----------------------------------------------------------------------
diff --git a/itests/src/test/resources/testconfiguration.properties b/itests/src/test/resources/testconfiguration.properties
index 2d0a1e3..58d0a45 100644
--- a/itests/src/test/resources/testconfiguration.properties
+++ b/itests/src/test/resources/testconfiguration.properties
@@ -93,6 +93,7 @@ minitez.query.files.shared=acid_globallimit.q,\
    cbo_union.q,\
    cbo_views.q,\
    cbo_windowing.q,\
+ constprog_semijoin.q,\
    correlationoptimizer1.q,\
    count.q,\
    create_merge_compressed.q,\
@@ -1283,6 +1284,7 @@ miniSparkOnYarn.query.files=auto_sortmerge_join_16.q,\
    bucketmapjoin6.q,\
    bucketmapjoin7.q,\
    constprog_partitioner.q,\
+ constprog_semijoin.q,\
    disable_merge_for_bucketing.q,\
    empty_dir_in_table.q,\
    external_table_with_space_in_location_path.q,\

http://git-wip-us.apache.org/repos/asf/hive/blob/c659b202/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConstantPropagateProcFactory.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConstantPropagateProcFactory.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConstantPropagateProcFactory.java
index e02d277..edb258a 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConstantPropagateProcFactory.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConstantPropagateProcFactory.java
@@ -1317,7 +1317,8 @@ public final class ConstantPropagateProcFactory {
       */
      private boolean skipFolding(JoinDesc joinDesc) {
        for (JoinCondDesc cond : joinDesc.getConds()) {
- if (cond.getType() == JoinDesc.INNER_JOIN || cond.getType() == JoinDesc.UNIQUE_JOIN) {
+ if (cond.getType() == JoinDesc.INNER_JOIN || cond.getType() == JoinDesc.UNIQUE_JOIN
+ || cond.getType() == JoinDesc.LEFT_SEMI_JOIN) {
            continue;
          }
          return true;

http://git-wip-us.apache.org/repos/asf/hive/blob/c659b202/ql/src/test/queries/clientpositive/constprog_semijoin.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/constprog_semijoin.q b/ql/src/test/queries/clientpositive/constprog_semijoin.q
new file mode 100644
index 0000000..34f90a0
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/constprog_semijoin.q
@@ -0,0 +1,34 @@
+set hive.optimize.constant.propagation=true;
+
+create table table1 (id int, val string, val1 string, dimid int);
+insert into table1 (id, val, val1, dimid) values (1, 't1val01', 'val101', 100), (2, 't1val02', 'val102', 200), (3, 't1val03', 'val103', 103), (3, 't1val01', 'val104', 100), (2, 't1val05', 'val105', 200), (3, 't1val01', 'val106', 103), (1, 't1val07', 'val107', 200), (2, 't1val01', 'val108', 200), (3, 't1val09', 'val109', 103), (4,'t1val01', 'val110', 200);
+
+create table table2 (id int, val2 string);
+insert into table2 (id, val2) values (1, 't2val201'), (2, 't2val202'), (3, 't2val203');
+
+create table table3 (id int);
+insert into table3 (id) values (100), (100), (101), (102), (103);
+
+explain select table1.id, table1.val, table1.val1 from table1 left semi join table3 on table1.dimid = table3.id where table1.val = 't1val01';
+select table1.id, table1.val, table1.val1 from table1 left semi join table3 on table1.dimid = table3.id where table1.val = 't1val01';
+
+explain select table1.id, table1.val, table2.val2 from table1 inner join table2 on table1.val = 't1val01' and table1.id = table2.id left semi join table3 on table1.dimid = table3.id;
+select table1.id, table1.val, table2.val2 from table1 inner join table2 on table1.val = 't1val01' and table1.id = table2.id left semi join table3 on table1.dimid = table3.id;
+
+explain select table1.id, table1.val, table2.val2 from table1 left semi join table3 on table1.dimid = table3.id inner join table2 on table1.val = 't1val01' and table1.id = table2.id;
+select table1.id, table1.val, table2.val2 from table1 left semi join table3 on table1.dimid = table3.id inner join table2 on table1.val = 't1val01' and table1.id = table2.id;
+
+explain select table1.id, table1.val, table1.val1 from table1 left semi join table3 on table1.dimid = table3.id and table3.id = 100 where table1.dimid <> 100;
+select table1.id, table1.val, table1.val1 from table1 left semi join table3 on table1.dimid = table3.id and table3.id = 100 where table1.dimid <> 100;
+
+explain select table1.id, table1.val, table1.val1 from table1 left semi join table3 on table1.dimid = table3.id and table3.id = 100 where table1.dimid IN (100,200);
+select table1.id, table1.val, table1.val1 from table1 left semi join table3 on table1.dimid = table3.id and table3.id = 100 where table1.dimid IN (100,200);
+
+explain select table1.id, table1.val, table1.val1 from table1 left semi join table3 on table1.dimid = table3.id and table3.id = 100 where table1.dimid = 200;
+select table1.id, table1.val, table1.val1 from table1 left semi join table3 on table1.dimid = table3.id and table3.id = 100 where table1.dimid = 200;
+
+explain select table1.id, table1.val, table1.val1 from table1 left semi join table3 on table1.dimid = table3.id and table3.id = 100 where table1.dimid = 100;
+select table1.id, table1.val, table1.val1 from table1 left semi join table3 on table1.dimid = table3.id and table3.id = 100 where table1.dimid = 100;
+
+explain select table1.id, table1.val, table1.val1 from table1 left semi join table3 on table1.dimid = table3.id and table3.id = 100;
+select table1.id, table1.val, table1.val1 from table1 left semi join table3 on table1.dimid = table3.id and table3.id = 100;

http://git-wip-us.apache.org/repos/asf/hive/blob/c659b202/ql/src/test/results/clientpositive/constprog_partitioner.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/constprog_partitioner.q.out b/ql/src/test/results/clientpositive/constprog_partitioner.q.out
index 12ac9c8..1dd9128 100644
--- a/ql/src/test/results/clientpositive/constprog_partitioner.q.out
+++ b/ql/src/test/results/clientpositive/constprog_partitioner.q.out
@@ -111,13 +111,13 @@ STAGE PLANS:
                predicate: ((l_linenumber = 1) and l_orderkey is not null) (type: boolean)
                Statistics: Num rows: 50 Data size: 5999 Basic stats: COMPLETE Column stats: NONE
                Select Operator
- expressions: l_orderkey (type: int), l_partkey (type: int), l_suppkey (type: int), 1 (type: int)
- outputColumnNames: _col0, _col1, _col2, _col3
+ expressions: l_orderkey (type: int), l_partkey (type: int), l_suppkey (type: int)
+ outputColumnNames: _col0, _col1, _col2
                  Statistics: Num rows: 50 Data size: 5999 Basic stats: COMPLETE Column stats: NONE
                  Reduce Output Operator
- key expressions: _col0 (type: int), _col3 (type: int)
+ key expressions: _col0 (type: int), 1 (type: int)
                    sort order: ++
- Map-reduce partition columns: _col0 (type: int), _col3 (type: int)
+ Map-reduce partition columns: _col0 (type: int), 1 (type: int)
                    Statistics: Num rows: 50 Data size: 5999 Basic stats: COMPLETE Column stats: NONE
                    value expressions: _col1 (type: int), _col2 (type: int)
            TableScan
@@ -136,9 +136,9 @@ STAGE PLANS:
                    outputColumnNames: _col0, _col1
                    Statistics: Num rows: 25 Data size: 2999 Basic stats: COMPLETE Column stats: NONE
                    Reduce Output Operator
- key expressions: _col0 (type: int), _col1 (type: int)
+ key expressions: _col0 (type: int), 1 (type: int)
                      sort order: ++
- Map-reduce partition columns: _col0 (type: int), _col1 (type: int)
+ Map-reduce partition columns: _col0 (type: int), 1 (type: int)
                      Statistics: Num rows: 25 Data size: 2999 Basic stats: COMPLETE Column stats: NONE
        Reduce Operator Tree:
          Join Operator

http://git-wip-us.apache.org/repos/asf/hive/blob/c659b202/ql/src/test/results/clientpositive/constprog_semijoin.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/constprog_semijoin.q.out b/ql/src/test/results/clientpositive/constprog_semijoin.q.out
new file mode 100644
index 0000000..0e0e883
--- /dev/null
+++ b/ql/src/test/results/clientpositive/constprog_semijoin.q.out
@@ -0,0 +1,814 @@
+PREHOOK: query: create table table1 (id int, val string, val1 string, dimid int)
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@table1
+POSTHOOK: query: create table table1 (id int, val string, val1 string, dimid int)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@table1
+PREHOOK: query: insert into table1 (id, val, val1, dimid) values (1, 't1val01', 'val101', 100), (2, 't1val02', 'val102', 200), (3, 't1val03', 'val103', 103), (3, 't1val01', 'val104', 100), (2, 't1val05', 'val105', 200), (3, 't1val01', 'val106', 103), (1, 't1val07', 'val107', 200), (2, 't1val01', 'val108', 200), (3, 't1val09', 'val109', 103), (4,'t1val01', 'val110', 200)
+PREHOOK: type: QUERY
+PREHOOK: Input: default@values__tmp__table__1
+PREHOOK: Output: default@table1
+POSTHOOK: query: insert into table1 (id, val, val1, dimid) values (1, 't1val01', 'val101', 100), (2, 't1val02', 'val102', 200), (3, 't1val03', 'val103', 103), (3, 't1val01', 'val104', 100), (2, 't1val05', 'val105', 200), (3, 't1val01', 'val106', 103), (1, 't1val07', 'val107', 200), (2, 't1val01', 'val108', 200), (3, 't1val09', 'val109', 103), (4,'t1val01', 'val110', 200)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@values__tmp__table__1
+POSTHOOK: Output: default@table1
+POSTHOOK: Lineage: table1.dimid EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col4, type:string, comment:), ]
+POSTHOOK: Lineage: table1.id EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col1, type:string, comment:), ]
+POSTHOOK: Lineage: table1.val SIMPLE [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col2, type:string, comment:), ]
+POSTHOOK: Lineage: table1.val1 SIMPLE [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col3, type:string, comment:), ]
+PREHOOK: query: create table table2 (id int, val2 string)
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@table2
+POSTHOOK: query: create table table2 (id int, val2 string)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@table2
+PREHOOK: query: insert into table2 (id, val2) values (1, 't2val201'), (2, 't2val202'), (3, 't2val203')
+PREHOOK: type: QUERY
+PREHOOK: Input: default@values__tmp__table__2
+PREHOOK: Output: default@table2
+POSTHOOK: query: insert into table2 (id, val2) values (1, 't2val201'), (2, 't2val202'), (3, 't2val203')
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@values__tmp__table__2
+POSTHOOK: Output: default@table2
+POSTHOOK: Lineage: table2.id EXPRESSION [(values__tmp__table__2)values__tmp__table__2.FieldSchema(name:tmp_values_col1, type:string, comment:), ]
+POSTHOOK: Lineage: table2.val2 SIMPLE [(values__tmp__table__2)values__tmp__table__2.FieldSchema(name:tmp_values_col2, type:string, comment:), ]
+PREHOOK: query: create table table3 (id int)
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@table3
+POSTHOOK: query: create table table3 (id int)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@table3
+PREHOOK: query: insert into table3 (id) values (100), (100), (101), (102), (103)
+PREHOOK: type: QUERY
+PREHOOK: Input: default@values__tmp__table__3
+PREHOOK: Output: default@table3
+POSTHOOK: query: insert into table3 (id) values (100), (100), (101), (102), (103)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@values__tmp__table__3
+POSTHOOK: Output: default@table3
+POSTHOOK: Lineage: table3.id EXPRESSION [(values__tmp__table__3)values__tmp__table__3.FieldSchema(name:tmp_values_col1, type:string, comment:), ]
+PREHOOK: query: explain select table1.id, table1.val, table1.val1 from table1 left semi join table3 on table1.dimid = table3.id where table1.val = 't1val01'
+PREHOOK: type: QUERY
+POSTHOOK: query: explain select table1.id, table1.val, table1.val1 from table1 left semi join table3 on table1.dimid = table3.id where table1.val = 't1val01'
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ alias: table1
+ Statistics: Num rows: 10 Data size: 200 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: ((val = 't1val01') and dimid is not null) (type: boolean)
+ Statistics: Num rows: 5 Data size: 100 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: id (type: int), val1 (type: string), dimid (type: int)
+ outputColumnNames: _col0, _col2, _col3
+ Statistics: Num rows: 5 Data size: 100 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col3 (type: int)
+ sort order: +
+ Map-reduce partition columns: _col3 (type: int)
+ Statistics: Num rows: 5 Data size: 100 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: int), _col2 (type: string)
+ TableScan
+ alias: table3
+ Statistics: Num rows: 5 Data size: 15 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: id is not null (type: boolean)
+ Statistics: Num rows: 5 Data size: 15 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: id (type: int)
+ outputColumnNames: _col0
+ Statistics: Num rows: 5 Data size: 15 Basic stats: COMPLETE Column stats: NONE
+ Group By Operator
+ keys: _col0 (type: int)
+ mode: hash
+ outputColumnNames: _col0
+ Statistics: Num rows: 5 Data size: 15 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: int)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: int)
+ Statistics: Num rows: 5 Data size: 15 Basic stats: COMPLETE Column stats: NONE
+ Reduce Operator Tree:
+ Join Operator
+ condition map:
+ Left Semi Join 0 to 1
+ keys:
+ 0 _col3 (type: int)
+ 1 _col0 (type: int)
+ outputColumnNames: _col0, _col2
+ Statistics: Num rows: 5 Data size: 110 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: _col0 (type: int), 't1val01' (type: string), _col2 (type: string)
+ outputColumnNames: _col0, _col1, _col2
+ Statistics: Num rows: 5 Data size: 110 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 5 Data size: 110 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: select table1.id, table1.val, table1.val1 from table1 left semi join table3 on table1.dimid = table3.id where table1.val = 't1val01'
+PREHOOK: type: QUERY
+PREHOOK: Input: default@table1
+PREHOOK: Input: default@table3
+#### A masked pattern was here ####
+POSTHOOK: query: select table1.id, table1.val, table1.val1 from table1 left semi join table3 on table1.dimid = table3.id where table1.val = 't1val01'
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@table1
+POSTHOOK: Input: default@table3
+#### A masked pattern was here ####
+3 t1val01 val104
+1 t1val01 val101
+3 t1val01 val106
+PREHOOK: query: explain select table1.id, table1.val, table2.val2 from table1 inner join table2 on table1.val = 't1val01' and table1.id = table2.id left semi join table3 on table1.dimid = table3.id
+PREHOOK: type: QUERY
+POSTHOOK: query: explain select table1.id, table1.val, table2.val2 from table1 inner join table2 on table1.val = 't1val01' and table1.id = table2.id left semi join table3 on table1.dimid = table3.id
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-2 depends on stages: Stage-1
+ Stage-0 depends on stages: Stage-2
+
+STAGE PLANS:
+ Stage: Stage-1
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ alias: table1
+ Statistics: Num rows: 10 Data size: 200 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: (((val = 't1val01') and id is not null) and dimid is not null) (type: boolean)
+ Statistics: Num rows: 5 Data size: 100 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: id (type: int), dimid (type: int)
+ outputColumnNames: _col0, _col2
+ Statistics: Num rows: 5 Data size: 100 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col2 (type: int)
+ sort order: +
+ Map-reduce partition columns: _col2 (type: int)
+ Statistics: Num rows: 5 Data size: 100 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: int)
+ TableScan
+ alias: table3
+ Statistics: Num rows: 5 Data size: 15 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: id is not null (type: boolean)
+ Statistics: Num rows: 5 Data size: 15 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: id (type: int)
+ outputColumnNames: _col0
+ Statistics: Num rows: 5 Data size: 15 Basic stats: COMPLETE Column stats: NONE
+ Group By Operator
+ keys: _col0 (type: int)
+ mode: hash
+ outputColumnNames: _col0
+ Statistics: Num rows: 5 Data size: 15 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: int)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: int)
+ Statistics: Num rows: 5 Data size: 15 Basic stats: COMPLETE Column stats: NONE
+ Reduce Operator Tree:
+ Join Operator
+ condition map:
+ Left Semi Join 0 to 1
+ keys:
+ 0 _col2 (type: int)
+ 1 _col0 (type: int)
+ outputColumnNames: _col0
+ Statistics: Num rows: 5 Data size: 110 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
+
+ Stage: Stage-2
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ Reduce Output Operator
+ key expressions: _col0 (type: int)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: int)
+ Statistics: Num rows: 5 Data size: 110 Basic stats: COMPLETE Column stats: NONE
+ TableScan
+ alias: table2
+ Statistics: Num rows: 3 Data size: 30 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: id is not null (type: boolean)
+ Statistics: Num rows: 3 Data size: 30 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: id (type: int), val2 (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 3 Data size: 30 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: int)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: int)
+ Statistics: Num rows: 3 Data size: 30 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: string)
+ Reduce Operator Tree:
+ Join Operator
+ condition map:
+ Inner Join 0 to 1
+ keys:
+ 0 _col0 (type: int)
+ 1 _col0 (type: int)
+ outputColumnNames: _col0, _col4
+ Statistics: Num rows: 5 Data size: 121 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: _col0 (type: int), 't1val01' (type: string), _col4 (type: string)
+ outputColumnNames: _col0, _col1, _col2
+ Statistics: Num rows: 5 Data size: 121 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 5 Data size: 121 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: select table1.id, table1.val, table2.val2 from table1 inner join table2 on table1.val = 't1val01' and table1.id = table2.id left semi join table3 on table1.dimid = table3.id
+PREHOOK: type: QUERY
+PREHOOK: Input: default@table1
+PREHOOK: Input: default@table2
+PREHOOK: Input: default@table3
+#### A masked pattern was here ####
+POSTHOOK: query: select table1.id, table1.val, table2.val2 from table1 inner join table2 on table1.val = 't1val01' and table1.id = table2.id left semi join table3 on table1.dimid = table3.id
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@table1
+POSTHOOK: Input: default@table2
+POSTHOOK: Input: default@table3
+#### A masked pattern was here ####
+1 t1val01 t2val201
+3 t1val01 t2val203
+3 t1val01 t2val203
+PREHOOK: query: explain select table1.id, table1.val, table2.val2 from table1 left semi join table3 on table1.dimid = table3.id inner join table2 on table1.val = 't1val01' and table1.id = table2.id
+PREHOOK: type: QUERY
+POSTHOOK: query: explain select table1.id, table1.val, table2.val2 from table1 left semi join table3 on table1.dimid = table3.id inner join table2 on table1.val = 't1val01' and table1.id = table2.id
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-2 depends on stages: Stage-1
+ Stage-0 depends on stages: Stage-2
+
+STAGE PLANS:
+ Stage: Stage-1
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ alias: table1
+ Statistics: Num rows: 10 Data size: 200 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: (((val = 't1val01') and dimid is not null) and id is not null) (type: boolean)
+ Statistics: Num rows: 5 Data size: 100 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: id (type: int), dimid (type: int)
+ outputColumnNames: _col0, _col2
+ Statistics: Num rows: 5 Data size: 100 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col2 (type: int)
+ sort order: +
+ Map-reduce partition columns: _col2 (type: int)
+ Statistics: Num rows: 5 Data size: 100 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: int)
+ TableScan
+ alias: table3
+ Statistics: Num rows: 5 Data size: 15 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: id is not null (type: boolean)
+ Statistics: Num rows: 5 Data size: 15 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: id (type: int)
+ outputColumnNames: _col0
+ Statistics: Num rows: 5 Data size: 15 Basic stats: COMPLETE Column stats: NONE
+ Group By Operator
+ keys: _col0 (type: int)
+ mode: hash
+ outputColumnNames: _col0
+ Statistics: Num rows: 5 Data size: 15 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: int)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: int)
+ Statistics: Num rows: 5 Data size: 15 Basic stats: COMPLETE Column stats: NONE
+ Reduce Operator Tree:
+ Join Operator
+ condition map:
+ Left Semi Join 0 to 1
+ keys:
+ 0 _col2 (type: int)
+ 1 _col0 (type: int)
+ outputColumnNames: _col0
+ Statistics: Num rows: 5 Data size: 110 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
+
+ Stage: Stage-2
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ Reduce Output Operator
+ key expressions: _col0 (type: int)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: int)
+ Statistics: Num rows: 5 Data size: 110 Basic stats: COMPLETE Column stats: NONE
+ TableScan
+ alias: table2
+ Statistics: Num rows: 3 Data size: 30 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: id is not null (type: boolean)
+ Statistics: Num rows: 3 Data size: 30 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: id (type: int), val2 (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 3 Data size: 30 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: int)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: int)
+ Statistics: Num rows: 3 Data size: 30 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: string)
+ Reduce Operator Tree:
+ Join Operator
+ condition map:
+ Inner Join 0 to 1
+ keys:
+ 0 _col0 (type: int)
+ 1 _col0 (type: int)
+ outputColumnNames: _col0, _col4
+ Statistics: Num rows: 5 Data size: 121 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: _col0 (type: int), 't1val01' (type: string), _col4 (type: string)
+ outputColumnNames: _col0, _col1, _col2
+ Statistics: Num rows: 5 Data size: 121 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 5 Data size: 121 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: select table1.id, table1.val, table2.val2 from table1 left semi join table3 on table1.dimid = table3.id inner join table2 on table1.val = 't1val01' and table1.id = table2.id
+PREHOOK: type: QUERY
+PREHOOK: Input: default@table1
+PREHOOK: Input: default@table2
+PREHOOK: Input: default@table3
+#### A masked pattern was here ####
+POSTHOOK: query: select table1.id, table1.val, table2.val2 from table1 left semi join table3 on table1.dimid = table3.id inner join table2 on table1.val = 't1val01' and table1.id = table2.id
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@table1
+POSTHOOK: Input: default@table2
+POSTHOOK: Input: default@table3
+#### A masked pattern was here ####
+1 t1val01 t2val201
+3 t1val01 t2val203
+3 t1val01 t2val203
+PREHOOK: query: explain select table1.id, table1.val, table1.val1 from table1 left semi join table3 on table1.dimid = table3.id and table3.id = 100 where table1.dimid <> 100
+PREHOOK: type: QUERY
+POSTHOOK: query: explain select table1.id, table1.val, table1.val1 from table1 left semi join table3 on table1.dimid = table3.id and table3.id = 100 where table1.dimid <> 100
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ alias: table1
+ Statistics: Num rows: 10 Data size: 200 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: ((((dimid = 100) = true) and (dimid <> 100)) and (dimid = 100) is not null) (type: boolean)
+ Statistics: Num rows: 5 Data size: 100 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: id (type: int), val (type: string), val1 (type: string), dimid (type: int)
+ outputColumnNames: _col0, _col1, _col2, _col3
+ Statistics: Num rows: 5 Data size: 100 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col3 (type: int), true (type: boolean)
+ sort order: ++
+ Map-reduce partition columns: _col3 (type: int), true (type: boolean)
+ Statistics: Num rows: 5 Data size: 100 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string)
+ TableScan
+ alias: table3
+ Statistics: Num rows: 5 Data size: 15 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: ((((id = 100) = true) and (id <> 100)) and (id = 100) is not null) (type: boolean)
+ Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: id (type: int), (id = 100) (type: boolean)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE
+ Group By Operator
+ keys: _col0 (type: int), _col1 (type: boolean)
+ mode: hash
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: int), _col1 (type: boolean)
+ sort order: ++
+ Map-reduce partition columns: _col0 (type: int), _col1 (type: boolean)
+ Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE
+ Reduce Operator Tree:
+ Join Operator
+ condition map:
+ Left Semi Join 0 to 1
+ keys:
+ 0 _col3 (type: int), true (type: boolean)
+ 1 _col0 (type: int), _col1 (type: boolean)
+ outputColumnNames: _col0, _col1, _col2
+ Statistics: Num rows: 5 Data size: 110 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 5 Data size: 110 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: select table1.id, table1.val, table1.val1 from table1 left semi join table3 on table1.dimid = table3.id and table3.id = 100 where table1.dimid <> 100
+PREHOOK: type: QUERY
+PREHOOK: Input: default@table1
+PREHOOK: Input: default@table3
+#### A masked pattern was here ####
+POSTHOOK: query: select table1.id, table1.val, table1.val1 from table1 left semi join table3 on table1.dimid = table3.id and table3.id = 100 where table1.dimid <> 100
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@table1
+POSTHOOK: Input: default@table3
+#### A masked pattern was here ####
+PREHOOK: query: explain select table1.id, table1.val, table1.val1 from table1 left semi join table3 on table1.dimid = table3.id and table3.id = 100 where table1.dimid IN (100,200)
+PREHOOK: type: QUERY
+POSTHOOK: query: explain select table1.id, table1.val, table1.val1 from table1 left semi join table3 on table1.dimid = table3.id and table3.id = 100 where table1.dimid IN (100,200)
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ alias: table1
+ Statistics: Num rows: 10 Data size: 200 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: ((((dimid = 100) = true) and (dimid) IN (100, 200)) and (dimid = 100) is not null) (type: boolean)
+ Statistics: Num rows: 2 Data size: 40 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: id (type: int), val (type: string), val1 (type: string), dimid (type: int)
+ outputColumnNames: _col0, _col1, _col2, _col3
+ Statistics: Num rows: 2 Data size: 40 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col3 (type: int), true (type: boolean)
+ sort order: ++
+ Map-reduce partition columns: _col3 (type: int), true (type: boolean)
+ Statistics: Num rows: 2 Data size: 40 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string)
+ TableScan
+ alias: table3
+ Statistics: Num rows: 5 Data size: 15 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: ((((id = 100) = true) and (id) IN (100, 200)) and (id = 100) is not null) (type: boolean)
+ Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: id (type: int), (id = 100) (type: boolean)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE
+ Group By Operator
+ keys: _col0 (type: int), _col1 (type: boolean)
+ mode: hash
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: int), _col1 (type: boolean)
+ sort order: ++
+ Map-reduce partition columns: _col0 (type: int), _col1 (type: boolean)
+ Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE
+ Reduce Operator Tree:
+ Join Operator
+ condition map:
+ Left Semi Join 0 to 1
+ keys:
+ 0 _col3 (type: int), true (type: boolean)
+ 1 _col0 (type: int), _col1 (type: boolean)
+ outputColumnNames: _col0, _col1, _col2
+ Statistics: Num rows: 2 Data size: 44 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 2 Data size: 44 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: select table1.id, table1.val, table1.val1 from table1 left semi join table3 on table1.dimid = table3.id and table3.id = 100 where table1.dimid IN (100,200)
+PREHOOK: type: QUERY
+PREHOOK: Input: default@table1
+PREHOOK: Input: default@table3
+#### A masked pattern was here ####
+POSTHOOK: query: select table1.id, table1.val, table1.val1 from table1 left semi join table3 on table1.dimid = table3.id and table3.id = 100 where table1.dimid IN (100,200)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@table1
+POSTHOOK: Input: default@table3
+#### A masked pattern was here ####
+3 t1val01 val104
+1 t1val01 val101
+PREHOOK: query: explain select table1.id, table1.val, table1.val1 from table1 left semi join table3 on table1.dimid = table3.id and table3.id = 100 where table1.dimid = 200
+PREHOOK: type: QUERY
+POSTHOOK: query: explain select table1.id, table1.val, table1.val1 from table1 left semi join table3 on table1.dimid = table3.id and table3.id = 100 where table1.dimid = 200
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ alias: table1
+ Statistics: Num rows: 10 Data size: 200 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: ((((dimid = 100) = true) and (dimid = 200)) and (dimid = 100) is not null) (type: boolean)
+ Statistics: Num rows: 2 Data size: 40 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: id (type: int), val (type: string), val1 (type: string)
+ outputColumnNames: _col0, _col1, _col2
+ Statistics: Num rows: 2 Data size: 40 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: 200 (type: int), true (type: boolean)
+ sort order: ++
+ Map-reduce partition columns: 200 (type: int), true (type: boolean)
+ Statistics: Num rows: 2 Data size: 40 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string)
+ TableScan
+ alias: table3
+ Statistics: Num rows: 5 Data size: 15 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: ((((id = 100) = true) and (id = 200)) and (id = 100) is not null) (type: boolean)
+ Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE
+ Group By Operator
+ keys: 200 (type: int), false (type: boolean)
+ mode: hash
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: 200 (type: int), false (type: boolean)
+ sort order: ++
+ Map-reduce partition columns: 200 (type: int), false (type: boolean)
+ Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE
+ Reduce Operator Tree:
+ Join Operator
+ condition map:
+ Left Semi Join 0 to 1
+ keys:
+ 0 _col3 (type: int), true (type: boolean)
+ 1 _col0 (type: int), _col1 (type: boolean)
+ outputColumnNames: _col0, _col1, _col2
+ Statistics: Num rows: 2 Data size: 44 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 2 Data size: 44 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: select table1.id, table1.val, table1.val1 from table1 left semi join table3 on table1.dimid = table3.id and table3.id = 100 where table1.dimid = 200
+PREHOOK: type: QUERY
+PREHOOK: Input: default@table1
+PREHOOK: Input: default@table3
+#### A masked pattern was here ####
+POSTHOOK: query: select table1.id, table1.val, table1.val1 from table1 left semi join table3 on table1.dimid = table3.id and table3.id = 100 where table1.dimid = 200
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@table1
+POSTHOOK: Input: default@table3
+#### A masked pattern was here ####
+PREHOOK: query: explain select table1.id, table1.val, table1.val1 from table1 left semi join table3 on table1.dimid = table3.id and table3.id = 100 where table1.dimid = 100
+PREHOOK: type: QUERY
+POSTHOOK: query: explain select table1.id, table1.val, table1.val1 from table1 left semi join table3 on table1.dimid = table3.id and table3.id = 100 where table1.dimid = 100
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ alias: table1
+ Statistics: Num rows: 10 Data size: 200 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: ((((dimid = 100) = true) and (dimid = 100)) and (dimid = 100) is not null) (type: boolean)
+ Statistics: Num rows: 2 Data size: 40 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: id (type: int), val (type: string), val1 (type: string)
+ outputColumnNames: _col0, _col1, _col2
+ Statistics: Num rows: 2 Data size: 40 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: 100 (type: int), true (type: boolean)
+ sort order: ++
+ Map-reduce partition columns: 100 (type: int), true (type: boolean)
+ Statistics: Num rows: 2 Data size: 40 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string)
+ TableScan
+ alias: table3
+ Statistics: Num rows: 5 Data size: 15 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: ((((id = 100) = true) and (id = 100)) and (id = 100) is not null) (type: boolean)
+ Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE
+ Group By Operator
+ keys: 100 (type: int), true (type: boolean)
+ mode: hash
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: 100 (type: int), true (type: boolean)
+ sort order: ++
+ Map-reduce partition columns: 100 (type: int), true (type: boolean)
+ Statistics: Num rows: 1 Data size: 3 Basic stats: COMPLETE Column stats: NONE
+ Reduce Operator Tree:
+ Join Operator
+ condition map:
+ Left Semi Join 0 to 1
+ keys:
+ 0 _col3 (type: int), true (type: boolean)
+ 1 _col0 (type: int), _col1 (type: boolean)
+ outputColumnNames: _col0, _col1, _col2
+ Statistics: Num rows: 2 Data size: 44 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 2 Data size: 44 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: select table1.id, table1.val, table1.val1 from table1 left semi join table3 on table1.dimid = table3.id and table3.id = 100 where table1.dimid = 100
+PREHOOK: type: QUERY
+PREHOOK: Input: default@table1
+PREHOOK: Input: default@table3
+#### A masked pattern was here ####
+POSTHOOK: query: select table1.id, table1.val, table1.val1 from table1 left semi join table3 on table1.dimid = table3.id and table3.id = 100 where table1.dimid = 100
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@table1
+POSTHOOK: Input: default@table3
+#### A masked pattern was here ####
+3 t1val01 val104
+1 t1val01 val101
+PREHOOK: query: explain select table1.id, table1.val, table1.val1 from table1 left semi join table3 on table1.dimid = table3.id and table3.id = 100
+PREHOOK: type: QUERY
+POSTHOOK: query: explain select table1.id, table1.val, table1.val1 from table1 left semi join table3 on table1.dimid = table3.id and table3.id = 100
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ alias: table1
+ Statistics: Num rows: 10 Data size: 200 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: ((((dimid = 100) = true) and dimid is not null) and (dimid = 100) is not null) (type: boolean)
+ Statistics: Num rows: 5 Data size: 100 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: id (type: int), val (type: string), val1 (type: string), dimid (type: int)
+ outputColumnNames: _col0, _col1, _col2, _col3
+ Statistics: Num rows: 5 Data size: 100 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col3 (type: int), true (type: boolean)
+ sort order: ++
+ Map-reduce partition columns: _col3 (type: int), true (type: boolean)
+ Statistics: Num rows: 5 Data size: 100 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string)
+ TableScan
+ alias: table3
+ Statistics: Num rows: 5 Data size: 15 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: ((((id = 100) = true) and id is not null) and (id = 100) is not null) (type: boolean)
+ Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: id (type: int), (id = 100) (type: boolean)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE
+ Group By Operator
+ keys: _col0 (type: int), _col1 (type: boolean)
+ mode: hash
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: int), _col1 (type: boolean)
+ sort order: ++
+ Map-reduce partition columns: _col0 (type: int), _col1 (type: boolean)
+ Statistics: Num rows: 2 Data size: 6 Basic stats: COMPLETE Column stats: NONE
+ Reduce Operator Tree:
+ Join Operator
+ condition map:
+ Left Semi Join 0 to 1
+ keys:
+ 0 _col3 (type: int), true (type: boolean)
+ 1 _col0 (type: int), _col1 (type: boolean)
+ outputColumnNames: _col0, _col1, _col2
+ Statistics: Num rows: 5 Data size: 110 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 5 Data size: 110 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: select table1.id, table1.val, table1.val1 from table1 left semi join table3 on table1.dimid = table3.id and table3.id = 100
+PREHOOK: type: QUERY
+PREHOOK: Input: default@table1
+PREHOOK: Input: default@table3
+#### A masked pattern was here ####
+POSTHOOK: query: select table1.id, table1.val, table1.val1 from table1 left semi join table3 on table1.dimid = table3.id and table3.id = 100
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@table1
+POSTHOOK: Input: default@table3
+#### A masked pattern was here ####
+3 t1val01 val104
+1 t1val01 val101

http://git-wip-us.apache.org/repos/asf/hive/blob/c659b202/ql/src/test/results/clientpositive/semijoin2.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/semijoin2.q.out b/ql/src/test/results/clientpositive/semijoin2.q.out
index 449dc3a..3fe0a89 100644
--- a/ql/src/test/results/clientpositive/semijoin2.q.out
+++ b/ql/src/test/results/clientpositive/semijoin2.q.out
@@ -110,9 +110,9 @@ STAGE PLANS:
                value expressions: _col16 (type: smallint), _col21 (type: double), _col98 (type: int)
            TableScan
              Reduce Output Operator
- key expressions: _col0 (type: int), _col1 (type: timestamp)
+ key expressions: -92 (type: int), _col1 (type: timestamp)
                sort order: ++
- Map-reduce partition columns: _col0 (type: int), _col1 (type: timestamp)
+ Map-reduce partition columns: -92 (type: int), _col1 (type: timestamp)
                Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
        Reduce Operator Tree:
          Join Operator

http://git-wip-us.apache.org/repos/asf/hive/blob/c659b202/ql/src/test/results/clientpositive/semijoin4.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/semijoin4.q.out b/ql/src/test/results/clientpositive/semijoin4.q.out
index e42668b..4dca08b 100644
--- a/ql/src/test/results/clientpositive/semijoin4.q.out
+++ b/ql/src/test/results/clientpositive/semijoin4.q.out
@@ -106,8 +106,8 @@ STAGE PLANS:
            outputColumnNames: _col1, _col3
            Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
            Select Operator
- expressions: _col1 (type: smallint), _col3 (type: double), -92 (type: tinyint), -92 (type: int)
- outputColumnNames: _col0, _col1, _col2, _col3
+ expressions: _col1 (type: smallint), _col3 (type: double)
+ outputColumnNames: _col0, _col1
              Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
              File Output Operator
                compressed: false
@@ -121,16 +121,16 @@ STAGE PLANS:
        Map Operator Tree:
            TableScan
              Reduce Output Operator
- key expressions: _col3 (type: int)
+ key expressions: -92 (type: int)
                sort order: +
- Map-reduce partition columns: _col3 (type: int)
+ Map-reduce partition columns: -92 (type: int)
                Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
- value expressions: _col0 (type: smallint), _col1 (type: double), _col2 (type: tinyint)
+ value expressions: _col0 (type: smallint), _col1 (type: double)
            TableScan
              Reduce Output Operator
- key expressions: _col0 (type: int)
+ key expressions: -92 (type: int)
                sort order: +
- Map-reduce partition columns: _col0 (type: int)
+ Map-reduce partition columns: -92 (type: int)
                Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
        Reduce Operator Tree:
          Join Operator
@@ -139,7 +139,7 @@ STAGE PLANS:
            keys:
              0 _col3 (type: int)
              1 _col0 (type: int)
- outputColumnNames: _col0, _col1, _col2
+ outputColumnNames: _col0, _col1
            Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
            File Output Operator
              compressed: false
@@ -153,14 +153,14 @@ STAGE PLANS:
        Map Operator Tree:
            TableScan
              Reduce Output Operator
- key expressions: (UDFToShort(_col2) + _col0) (type: smallint), floor(_col1) (type: bigint)
+ key expressions: (-92 + _col0) (type: smallint), floor(_col1) (type: bigint)
                sort order: +-
- Map-reduce partition columns: (UDFToShort(_col2) + _col0) (type: smallint)
+ Map-reduce partition columns: (-92 + _col0) (type: smallint)
                Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
- value expressions: _col0 (type: smallint), _col1 (type: double), _col2 (type: tinyint)
+ value expressions: _col0 (type: smallint), _col1 (type: double)
        Reduce Operator Tree:
          Select Operator
- expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: double), VALUE._col2 (type: tinyint)
+ expressions: VALUE._col0 (type: smallint), VALUE._col1 (type: double), -92 (type: tinyint)
            outputColumnNames: _col0, _col1, _col2
            Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
            PTF Operator

http://git-wip-us.apache.org/repos/asf/hive/blob/c659b202/ql/src/test/results/clientpositive/spark/constprog_partitioner.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/constprog_partitioner.q.out b/ql/src/test/results/clientpositive/spark/constprog_partitioner.q.out
index 6877ee7..367dab9 100644
--- a/ql/src/test/results/clientpositive/spark/constprog_partitioner.q.out
+++ b/ql/src/test/results/clientpositive/spark/constprog_partitioner.q.out
@@ -124,13 +124,13 @@ STAGE PLANS:
                      predicate: ((l_linenumber = 1) and l_orderkey is not null) (type: boolean)
                      Statistics: Num rows: 50 Data size: 5999 Basic stats: COMPLETE Column stats: NONE
                      Select Operator
- expressions: l_orderkey (type: int), l_partkey (type: int), l_suppkey (type: int), 1 (type: int)
- outputColumnNames: _col0, _col1, _col2, _col3
+ expressions: l_orderkey (type: int), l_partkey (type: int), l_suppkey (type: int)
+ outputColumnNames: _col0, _col1, _col2
                        Statistics: Num rows: 50 Data size: 5999 Basic stats: COMPLETE Column stats: NONE
                        Reduce Output Operator
- key expressions: _col0 (type: int), _col3 (type: int)
+ key expressions: _col0 (type: int), 1 (type: int)
                          sort order: ++
- Map-reduce partition columns: _col0 (type: int), _col3 (type: int)
+ Map-reduce partition columns: _col0 (type: int), 1 (type: int)
                          Statistics: Num rows: 50 Data size: 5999 Basic stats: COMPLETE Column stats: NONE
                          value expressions: _col1 (type: int), _col2 (type: int)
          Map 3
@@ -151,9 +151,9 @@ STAGE PLANS:
                          outputColumnNames: _col0, _col1
                          Statistics: Num rows: 25 Data size: 2999 Basic stats: COMPLETE Column stats: NONE
                          Reduce Output Operator
- key expressions: _col0 (type: int), _col1 (type: int)
+ key expressions: _col0 (type: int), 1 (type: int)
                            sort order: ++
- Map-reduce partition columns: _col0 (type: int), _col1 (type: int)
+ Map-reduce partition columns: _col0 (type: int), 1 (type: int)
                            Statistics: Num rows: 25 Data size: 2999 Basic stats: COMPLETE Column stats: NONE
          Reducer 2
              Reduce Operator Tree:

Search Discussions

Discussion Posts

Previous

Related Discussions

Discussion Navigation
viewthread | post
posts ‹ prev | 2 of 2 | next ›
Discussion Overview
groupcommits @
categorieshive, hadoop
postedFeb 22, '16 at 9:26p
activeFeb 22, '16 at 9:27p
posts2
users1
websitehive.apache.org

1 user in discussion

Ctang: 2 posts

People

Translate

site design / logo © 2021 Grokbase