FAQ
Author: szehon
Date: Mon Dec 22 19:49:05 2014
New Revision: 1647389

URL: http://svn.apache.org/r1647389
Log:
HIVE-9173 : Fix auto_join29.q, auto_join_without_localtask.q determinism (Jimmy Xiang via Szehon)

Modified:
     hive/trunk/ql/src/test/queries/clientpositive/auto_join29.q
     hive/trunk/ql/src/test/queries/clientpositive/auto_join_without_localtask.q
     hive/trunk/ql/src/test/results/clientpositive/auto_join29.q.out
     hive/trunk/ql/src/test/results/clientpositive/auto_join_without_localtask.q.out

Modified: hive/trunk/ql/src/test/queries/clientpositive/auto_join29.q
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientpositive/auto_join29.q?rev=1647389&r1=1647388&r2=1647389&view=diff
==============================================================================
--- hive/trunk/ql/src/test/queries/clientpositive/auto_join29.q (original)
+++ hive/trunk/ql/src/test/queries/clientpositive/auto_join29.q Mon Dec 22 19:49:05 2014
@@ -1,4 +1,7 @@
  set hive.auto.convert.join = true;
+
+-- SORT_QUERY_RESULTS
+
  explain
  SELECT * FROM src src1 LEFT OUTER JOIN src src2 ON (src1.key = src2.key AND src1.key < 10 AND src2.key > 10) RIGHT OUTER JOIN src src3 ON (src2.key = src3.key AND src3.key < 10) SORT BY src1.key, src1.value, src2.key, src2.value, src3.key, src3.value;


Modified: hive/trunk/ql/src/test/queries/clientpositive/auto_join_without_localtask.q
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientpositive/auto_join_without_localtask.q?rev=1647389&r1=1647388&r2=1647389&view=diff
==============================================================================
--- hive/trunk/ql/src/test/queries/clientpositive/auto_join_without_localtask.q (original)
+++ hive/trunk/ql/src/test/queries/clientpositive/auto_join_without_localtask.q Mon Dec 22 19:49:05 2014
@@ -17,13 +17,13 @@ select a.* from src a join src b on a.ke
  set hive.auto.convert.join.noconditionaltask.size=100;

  explain
-select a.* from src a join src b on a.key=b.key join src c on a.value=c.value where a.key>100 limit 40;
+select a.* from src a join src b on a.key=b.key join src c on a.value=c.value where a.key>100 order by a.key, a.value limit 40;

-select a.* from src a join src b on a.key=b.key join src c on a.value=c.value where a.key>100 limit 40;
+select a.* from src a join src b on a.key=b.key join src c on a.value=c.value where a.key>100 order by a.key, a.value limit 40;

  set hive.mapjoin.localtask.max.memory.usage = 0.0001;
  set hive.mapjoin.check.memory.rows = 2;

  -- fallback to common join
-select a.* from src a join src b on a.key=b.key join src c on a.value=c.value where a.key>100 limit 40;
+select a.* from src a join src b on a.key=b.key join src c on a.value=c.value where a.key>100 order by a.key, a.value limit 40;


Modified: hive/trunk/ql/src/test/results/clientpositive/auto_join29.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/auto_join29.q.out?rev=1647389&r1=1647388&r2=1647389&view=diff
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/auto_join29.q.out (original)
+++ hive/trunk/ql/src/test/results/clientpositive/auto_join29.q.out Mon Dec 22 19:49:05 2014
@@ -1,7 +1,11 @@
-PREHOOK: query: explain
+PREHOOK: query: -- SORT_QUERY_RESULTS
+
+explain
  SELECT * FROM src src1 LEFT OUTER JOIN src src2 ON (src1.key = src2.key AND src1.key < 10 AND src2.key > 10) RIGHT OUTER JOIN src src3 ON (src2.key = src3.key AND src3.key < 10) SORT BY src1.key, src1.value, src2.key, src2.value, src3.key, src3.value
  PREHOOK: type: QUERY
-POSTHOOK: query: explain
+POSTHOOK: query: -- SORT_QUERY_RESULTS
+
+explain
  SELECT * FROM src src1 LEFT OUTER JOIN src src2 ON (src1.key = src2.key AND src1.key < 10 AND src2.key > 10) RIGHT OUTER JOIN src src3 ON (src2.key = src3.key AND src3.key < 10) SORT BY src1.key, src1.value, src2.key, src2.value, src3.key, src3.value
  POSTHOOK: type: QUERY
  STAGE DEPENDENCIES:
@@ -1952,6 +1956,28 @@ POSTHOOK: query: SELECT * FROM src src1
  POSTHOOK: type: QUERY
  POSTHOOK: Input: default@src
  #### A masked pattern was here ####
+NULL NULL 0 val_0 0 val_0
+NULL NULL 0 val_0 0 val_0
+NULL NULL 0 val_0 0 val_0
+NULL NULL 0 val_0 0 val_0
+NULL NULL 0 val_0 0 val_0
+NULL NULL 0 val_0 0 val_0
+NULL NULL 0 val_0 0 val_0
+NULL NULL 0 val_0 0 val_0
+NULL NULL 0 val_0 0 val_0
+NULL NULL 2 val_2 2 val_2
+NULL NULL 4 val_4 4 val_4
+NULL NULL 5 val_5 5 val_5
+NULL NULL 5 val_5 5 val_5
+NULL NULL 5 val_5 5 val_5
+NULL NULL 5 val_5 5 val_5
+NULL NULL 5 val_5 5 val_5
+NULL NULL 5 val_5 5 val_5
+NULL NULL 5 val_5 5 val_5
+NULL NULL 5 val_5 5 val_5
+NULL NULL 5 val_5 5 val_5
+NULL NULL 8 val_8 8 val_8
+NULL NULL 9 val_9 9 val_9
  NULL NULL NULL NULL 10 val_10
  NULL NULL NULL NULL 100 val_100
  NULL NULL NULL NULL 100 val_100
@@ -2442,28 +2468,6 @@ NULL NULL NULL NULL 97 val_97
  NULL NULL NULL NULL 97 val_97
  NULL NULL NULL NULL 98 val_98
  NULL NULL NULL NULL 98 val_98
-NULL NULL 0 val_0 0 val_0
-NULL NULL 0 val_0 0 val_0
-NULL NULL 0 val_0 0 val_0
-NULL NULL 0 val_0 0 val_0
-NULL NULL 0 val_0 0 val_0
-NULL NULL 0 val_0 0 val_0
-NULL NULL 0 val_0 0 val_0
-NULL NULL 0 val_0 0 val_0
-NULL NULL 0 val_0 0 val_0
-NULL NULL 2 val_2 2 val_2
-NULL NULL 4 val_4 4 val_4
-NULL NULL 5 val_5 5 val_5
-NULL NULL 5 val_5 5 val_5
-NULL NULL 5 val_5 5 val_5
-NULL NULL 5 val_5 5 val_5
-NULL NULL 5 val_5 5 val_5
-NULL NULL 5 val_5 5 val_5
-NULL NULL 5 val_5 5 val_5
-NULL NULL 5 val_5 5 val_5
-NULL NULL 5 val_5 5 val_5
-NULL NULL 8 val_8 8 val_8
-NULL NULL 9 val_9 9 val_9
  PREHOOK: query: explain
  SELECT * FROM src src1 JOIN src src2 ON (src1.key = src2.key AND src1.key < 10 AND src2.key > 10) LEFT OUTER JOIN src src3 ON (src2.key = src3.key AND src3.key < 10) SORT BY src1.key, src1.value, src2.key, src2.value, src3.key, src3.value
  PREHOOK: type: QUERY

Modified: hive/trunk/ql/src/test/results/clientpositive/auto_join_without_localtask.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/auto_join_without_localtask.q.out?rev=1647389&r1=1647388&r2=1647389&view=diff
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/auto_join_without_localtask.q.out (original)
+++ hive/trunk/ql/src/test/results/clientpositive/auto_join_without_localtask.q.out Mon Dec 22 19:49:05 2014
@@ -612,30 +612,31 @@ RUN: Stage-3:MAPRED
  103 val_103
  103 val_103
  PREHOOK: query: explain
-select a.* from src a join src b on a.key=b.key join src c on a.value=c.value where a.key>100 limit 40
+select a.* from src a join src b on a.key=b.key join src c on a.value=c.value where a.key>100 order by a.key, a.value limit 40
  PREHOOK: type: QUERY
  POSTHOOK: query: explain
-select a.* from src a join src b on a.key=b.key join src c on a.value=c.value where a.key>100 limit 40
+select a.* from src a join src b on a.key=b.key join src c on a.value=c.value where a.key>100 order by a.key, a.value limit 40
  POSTHOOK: type: QUERY
  STAGE DEPENDENCIES:
- Stage-9 is a root stage , consists of Stage-12, Stage-13, Stage-2
- Stage-12 has a backup stage: Stage-2
- Stage-7 depends on stages: Stage-12
- Stage-6 depends on stages: Stage-2, Stage-7, Stage-8 , consists of Stage-4, Stage-11, Stage-1
- Stage-4 has a backup stage: Stage-1
- Stage-11 has a backup stage: Stage-1
- Stage-5 depends on stages: Stage-11
- Stage-1
- Stage-13 has a backup stage: Stage-2
+ Stage-10 is a root stage , consists of Stage-13, Stage-14, Stage-3
+ Stage-13 has a backup stage: Stage-3
    Stage-8 depends on stages: Stage-13
- Stage-2
- Stage-0 depends on stages: Stage-4, Stage-5, Stage-1
+ Stage-7 depends on stages: Stage-3, Stage-8, Stage-9 , consists of Stage-5, Stage-12, Stage-1
+ Stage-5 has a backup stage: Stage-1
+ Stage-2 depends on stages: Stage-1, Stage-5, Stage-6
+ Stage-12 has a backup stage: Stage-1
+ Stage-6 depends on stages: Stage-12
+ Stage-1
+ Stage-14 has a backup stage: Stage-3
+ Stage-9 depends on stages: Stage-14
+ Stage-3
+ Stage-0 depends on stages: Stage-2

  STAGE PLANS:
- Stage: Stage-9
+ Stage: Stage-10
      Conditional Operator

- Stage: Stage-12
+ Stage: Stage-13
      Map Reduce Local Work
        Alias -> Map Local Tables:
          $hdt$_0:$hdt$_1:$hdt$_2:a
@@ -655,7 +656,7 @@ STAGE PLANS:
                      0 _col0 (type: string)
                      1 _col0 (type: string)

- Stage: Stage-7
+ Stage: Stage-8
      Map Reduce
        Map Operator Tree:
            TableScan
@@ -681,10 +682,10 @@ STAGE PLANS:
        Local Work:
          Map Reduce Local Work

- Stage: Stage-6
+ Stage: Stage-7
      Conditional Operator

- Stage: Stage-4
+ Stage: Stage-5
      Map Reduce
        Map Operator Tree:
            TableScan
@@ -704,14 +705,12 @@ STAGE PLANS:
                    Select Operator
                      expressions: _col1 (type: string), _col2 (type: string)
                      outputColumnNames: _col0, _col1
- Limit
- Number of rows: 40
- File Output Operator
- compressed: false
- table:
- input format: org.apache.hadoop.mapred.TextInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ File Output Operator
+ compressed: false
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
        Local Work:
          Map Reduce Local Work
            Alias -> Map Local Tables:
@@ -722,7 +721,31 @@ STAGE PLANS:
              $hdt$_0:$INTNAME
                TableScan

- Stage: Stage-11
+ Stage: Stage-2
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ Reduce Output Operator
+ key expressions: _col0 (type: string), _col1 (type: string)
+ sort order: ++
+ Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE
+ Reduce Operator Tree:
+ Select Operator
+ expressions: KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE
+ Limit
+ Number of rows: 40
+ Statistics: Num rows: 40 Data size: 400 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 40 Data size: 400 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-12
      Map Reduce Local Work
        Alias -> Map Local Tables:
          $hdt$_0:$hdt$_0:a
@@ -742,7 +765,7 @@ STAGE PLANS:
                      0 _col0 (type: string)
                      1 _col1 (type: string)

- Stage: Stage-5
+ Stage: Stage-6
      Map Reduce
        Map Operator Tree:
            TableScan
@@ -756,14 +779,12 @@ STAGE PLANS:
                Select Operator
                  expressions: _col1 (type: string), _col2 (type: string)
                  outputColumnNames: _col0, _col1
- Limit
- Number of rows: 40
- File Output Operator
- compressed: false
- table:
- input format: org.apache.hadoop.mapred.TextInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ File Output Operator
+ compressed: false
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
        Local Work:
          Map Reduce Local Work

@@ -805,18 +826,14 @@ STAGE PLANS:
              expressions: _col1 (type: string), _col2 (type: string)
              outputColumnNames: _col0, _col1
              Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE
- Limit
- Number of rows: 40
- Statistics: Num rows: 40 Data size: 400 Basic stats: COMPLETE Column stats: NONE
- File Output Operator
- compressed: false
- Statistics: Num rows: 40 Data size: 400 Basic stats: COMPLETE Column stats: NONE
- table:
- input format: org.apache.hadoop.mapred.TextInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ File Output Operator
+ compressed: false
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe

- Stage: Stage-13
+ Stage: Stage-14
      Map Reduce Local Work
        Alias -> Map Local Tables:
          $hdt$_0:$hdt$_1:$hdt$_1:a
@@ -836,7 +853,7 @@ STAGE PLANS:
                      0 _col0 (type: string)
                      1 _col0 (type: string)

- Stage: Stage-8
+ Stage: Stage-9
      Map Reduce
        Map Operator Tree:
            TableScan
@@ -862,7 +879,7 @@ STAGE PLANS:
        Local Work:
          Map Reduce Local Work

- Stage: Stage-2
+ Stage: Stage-3
      Map Reduce
        Map Operator Tree:
            TableScan
@@ -918,62 +935,63 @@ STAGE PLANS:
        Processor Tree:
          ListSink

-PREHOOK: query: select a.* from src a join src b on a.key=b.key join src c on a.value=c.value where a.key>100 limit 40
+PREHOOK: query: select a.* from src a join src b on a.key=b.key join src c on a.value=c.value where a.key>100 order by a.key, a.value limit 40
  PREHOOK: type: QUERY
  PREHOOK: Input: default@src
  #### A masked pattern was here ####
-POSTHOOK: query: select a.* from src a join src b on a.key=b.key join src c on a.value=c.value where a.key>100 limit 40
+POSTHOOK: query: select a.* from src a join src b on a.key=b.key join src c on a.value=c.value where a.key>100 order by a.key, a.value limit 40
  POSTHOOK: type: QUERY
  POSTHOOK: Input: default@src
  #### A masked pattern was here ####
-RUN: Stage-9:CONDITIONAL
+RUN: Stage-10:CONDITIONAL
+RUN: Stage-14:MAPREDLOCAL
+RUN: Stage-9:MAPRED
+RUN: Stage-7:CONDITIONAL
  RUN: Stage-12:MAPREDLOCAL
-RUN: Stage-7:MAPRED
-RUN: Stage-6:CONDITIONAL
-RUN: Stage-11:MAPREDLOCAL
-RUN: Stage-5:MAPRED
-238 val_238
-238 val_238
-238 val_238
-238 val_238
-311 val_311
-311 val_311
-311 val_311
-311 val_311
-311 val_311
-311 val_311
-311 val_311
-311 val_311
-311 val_311
-165 val_165
-165 val_165
-165 val_165
-165 val_165
-409 val_409
-409 val_409
-409 val_409
-409 val_409
-409 val_409
-409 val_409
-409 val_409
-409 val_409
-409 val_409
-255 val_255
-255 val_255
-255 val_255
-255 val_255
-278 val_278
-278 val_278
-278 val_278
-278 val_278
-484 val_484
-265 val_265
-265 val_265
-265 val_265
-265 val_265
-193 val_193
+RUN: Stage-6:MAPRED
+RUN: Stage-2:MAPRED
+103 val_103
+103 val_103
+103 val_103
+103 val_103
+103 val_103
+103 val_103
+103 val_103
+103 val_103
+104 val_104
+104 val_104
+104 val_104
+104 val_104
+104 val_104
+104 val_104
+104 val_104
+104 val_104
+105 val_105
+111 val_111
+113 val_113
+113 val_113
+113 val_113
+113 val_113
+113 val_113
+113 val_113
+113 val_113
+113 val_113
+114 val_114
+116 val_116
+118 val_118
+118 val_118
+118 val_118
+118 val_118
+118 val_118
+118 val_118
+118 val_118
+118 val_118
+119 val_119
+119 val_119
+119 val_119
+119 val_119
  PREHOOK: query: -- fallback to common join
-select a.* from src a join src b on a.key=b.key join src c on a.value=c.value where a.key>100 limit 40
+select a.* from src a join src b on a.key=b.key join src c on a.value=c.value where a.key>100 order by a.key, a.value limit 40
  PREHOOK: type: QUERY
  PREHOOK: Input: default@src
  #### A masked pattern was here ####
@@ -982,16 +1000,17 @@ ATTEMPT: Execute BackupTask: org.apache.
  FAILED: Execution Error, return code 3 from org.apache.hadoop.hive.ql.exec.mr.MapredLocalTask
  ATTEMPT: Execute BackupTask: org.apache.hadoop.hive.ql.exec.mr.MapRedTask
  POSTHOOK: query: -- fallback to common join
-select a.* from src a join src b on a.key=b.key join src c on a.value=c.value where a.key>100 limit 40
+select a.* from src a join src b on a.key=b.key join src c on a.value=c.value where a.key>100 order by a.key, a.value limit 40
  POSTHOOK: type: QUERY
  POSTHOOK: Input: default@src
  #### A masked pattern was here ####
-RUN: Stage-9:CONDITIONAL
+RUN: Stage-10:CONDITIONAL
+RUN: Stage-14:MAPREDLOCAL
+RUN: Stage-3:MAPRED
+RUN: Stage-7:CONDITIONAL
  RUN: Stage-12:MAPREDLOCAL
-RUN: Stage-2:MAPRED
-RUN: Stage-6:CONDITIONAL
-RUN: Stage-11:MAPREDLOCAL
  RUN: Stage-1:MAPRED
+RUN: Stage-2:MAPRED
  103 val_103
  103 val_103
  103 val_103

Search Discussions

Related Discussions

Discussion Navigation
viewthread | post
Discussion Overview
groupcommits @
categorieshive, hadoop
postedDec 22, '14 at 7:49p
activeDec 22, '14 at 7:49p
posts1
users1
websitehive.apache.org

1 user in discussion

Szehon: 1 post

People

Translate

site design / logo © 2021 Grokbase