FAQ
Author: brock
Date: Thu Dec 25 18:06:30 2014
New Revision: 1647912

URL: http://svn.apache.org/r1647912
Log:
HIVE-9206 - Fix Desc Formatted related Java 8 ordering differences (Mohit Sabharwal via Brock)

Added:
     hive/trunk/ql/src/test/results/clientpositive/list_bucket_dml_11.q.java1.7.out
     hive/trunk/ql/src/test/results/clientpositive/list_bucket_dml_11.q.java1.8.out
     hive/trunk/ql/src/test/results/clientpositive/list_bucket_dml_12.q.java1.7.out
     hive/trunk/ql/src/test/results/clientpositive/list_bucket_dml_12.q.java1.8.out
     hive/trunk/ql/src/test/results/clientpositive/list_bucket_dml_13.q.java1.7.out
     hive/trunk/ql/src/test/results/clientpositive/list_bucket_dml_13.q.java1.8.out
     hive/trunk/ql/src/test/results/clientpositive/list_bucket_dml_2.q.java1.7.out
     hive/trunk/ql/src/test/results/clientpositive/list_bucket_dml_2.q.java1.8.out
     hive/trunk/ql/src/test/results/clientpositive/list_bucket_dml_4.q.java1.7.out
     hive/trunk/ql/src/test/results/clientpositive/list_bucket_dml_4.q.java1.8.out
     hive/trunk/ql/src/test/results/clientpositive/list_bucket_dml_5.q.java1.7.out
     hive/trunk/ql/src/test/results/clientpositive/list_bucket_dml_5.q.java1.8.out
     hive/trunk/ql/src/test/results/clientpositive/list_bucket_dml_8.q.java1.7.out
     hive/trunk/ql/src/test/results/clientpositive/list_bucket_dml_8.q.java1.8.out
     hive/trunk/ql/src/test/results/clientpositive/list_bucket_dml_9.q.java1.7.out
     hive/trunk/ql/src/test/results/clientpositive/list_bucket_dml_9.q.java1.8.out
     hive/trunk/ql/src/test/results/clientpositive/stats_list_bucket.q.java1.7.out
     hive/trunk/ql/src/test/results/clientpositive/stats_list_bucket.q.java1.8.out
Removed:
     hive/trunk/ql/src/test/results/clientpositive/list_bucket_dml_11.q.out
     hive/trunk/ql/src/test/results/clientpositive/list_bucket_dml_12.q.out
     hive/trunk/ql/src/test/results/clientpositive/list_bucket_dml_13.q.out
     hive/trunk/ql/src/test/results/clientpositive/list_bucket_dml_2.q.out
     hive/trunk/ql/src/test/results/clientpositive/list_bucket_dml_4.q.out
     hive/trunk/ql/src/test/results/clientpositive/list_bucket_dml_5.q.out
     hive/trunk/ql/src/test/results/clientpositive/list_bucket_dml_8.q.out
     hive/trunk/ql/src/test/results/clientpositive/list_bucket_dml_9.q.out
     hive/trunk/ql/src/test/results/clientpositive/stats_list_bucket.q.out
Modified:
     hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/metadata/formatting/MapBuilder.java
     hive/trunk/ql/src/test/queries/clientpositive/list_bucket_dml_10.q
     hive/trunk/ql/src/test/queries/clientpositive/list_bucket_dml_11.q
     hive/trunk/ql/src/test/queries/clientpositive/list_bucket_dml_12.q
     hive/trunk/ql/src/test/queries/clientpositive/list_bucket_dml_13.q
     hive/trunk/ql/src/test/queries/clientpositive/list_bucket_dml_2.q
     hive/trunk/ql/src/test/queries/clientpositive/list_bucket_dml_4.q
     hive/trunk/ql/src/test/queries/clientpositive/list_bucket_dml_5.q
     hive/trunk/ql/src/test/queries/clientpositive/list_bucket_dml_8.q
     hive/trunk/ql/src/test/queries/clientpositive/list_bucket_dml_9.q
     hive/trunk/ql/src/test/queries/clientpositive/stats_list_bucket.q
     hive/trunk/ql/src/test/results/clientpositive/partitions_json.q.out

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/metadata/formatting/MapBuilder.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/metadata/formatting/MapBuilder.java?rev=1647912&r1=1647911&r2=1647912&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/metadata/formatting/MapBuilder.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/metadata/formatting/MapBuilder.java Thu Dec 25 18:06:30 2014
@@ -18,6 +18,7 @@
  package org.apache.hadoop.hive.ql.metadata.formatting;

  import java.util.HashMap;
+import java.util.LinkedHashMap;
  import java.util.Map;

  /**
@@ -25,7 +26,7 @@ import java.util.Map;
   * add non-null entries to the Map.
   */
  public class MapBuilder {
- private Map<String, Object> map = new HashMap<String, Object>();
+ private Map<String, Object> map = new LinkedHashMap<String, Object>();

      private MapBuilder() {}


Modified: hive/trunk/ql/src/test/queries/clientpositive/list_bucket_dml_10.q
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientpositive/list_bucket_dml_10.q?rev=1647912&r1=1647911&r2=1647912&view=diff
==============================================================================
--- hive/trunk/ql/src/test/queries/clientpositive/list_bucket_dml_10.q (original)
+++ hive/trunk/ql/src/test/queries/clientpositive/list_bucket_dml_10.q Thu Dec 25 18:06:30 2014
@@ -3,6 +3,7 @@ set hive.mapred.supports.subdirectories=
  set mapred.input.dir.recursive=true;

  -- run this test case in minimr to ensure it works in cluster
+-- JAVA_VERSION_SPECIFIC_OUTPUT

  -- list bucketing DML: static partition. multiple skewed columns.
  -- ds=2008-04-08/hr=11/HIVE_DEFAULT_LIST_BUCKETING_DIR_NAME:

Modified: hive/trunk/ql/src/test/queries/clientpositive/list_bucket_dml_11.q
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientpositive/list_bucket_dml_11.q?rev=1647912&r1=1647911&r2=1647912&view=diff
==============================================================================
--- hive/trunk/ql/src/test/queries/clientpositive/list_bucket_dml_11.q (original)
+++ hive/trunk/ql/src/test/queries/clientpositive/list_bucket_dml_11.q Thu Dec 25 18:06:30 2014
@@ -6,6 +6,7 @@ set hive.merge.mapredfiles=false;
  -- Ensure it works if skewed column is not the first column in the table columns

  -- INCLUDE_HADOOP_MAJOR_VERSIONS(0.23)
+-- JAVA_VERSION_SPECIFIC_OUTPUT

  -- list bucketing DML: static partition. multiple skewed columns.


Modified: hive/trunk/ql/src/test/queries/clientpositive/list_bucket_dml_12.q
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientpositive/list_bucket_dml_12.q?rev=1647912&r1=1647911&r2=1647912&view=diff
==============================================================================
--- hive/trunk/ql/src/test/queries/clientpositive/list_bucket_dml_12.q (original)
+++ hive/trunk/ql/src/test/queries/clientpositive/list_bucket_dml_12.q Thu Dec 25 18:06:30 2014
@@ -7,6 +7,7 @@ set hive.merge.mapredfiles=false;

  -- INCLUDE_HADOOP_MAJOR_VERSIONS(0.23)
  -- SORT_QUERY_RESULTS
+-- JAVA_VERSION_SPECIFIC_OUTPUT

  -- test where the skewed values are more than 1 say columns no. 2 and 4 in a table with 5 columns
  create table list_bucketing_mul_col (col1 String, col2 String, col3 String, col4 String, col5 string)

Modified: hive/trunk/ql/src/test/queries/clientpositive/list_bucket_dml_13.q
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientpositive/list_bucket_dml_13.q?rev=1647912&r1=1647911&r2=1647912&view=diff
==============================================================================
--- hive/trunk/ql/src/test/queries/clientpositive/list_bucket_dml_13.q (original)
+++ hive/trunk/ql/src/test/queries/clientpositive/list_bucket_dml_13.q Thu Dec 25 18:06:30 2014
@@ -7,6 +7,7 @@ set hive.merge.mapredfiles=false;

  -- INCLUDE_HADOOP_MAJOR_VERSIONS(0.23)
  -- SORT_QUERY_RESULTS
+-- JAVA_VERSION_SPECIFIC_OUTPUT

  -- test where the skewed values are more than 1 say columns no. 2 and 4 in a table with 5 columns
  create table list_bucketing_mul_col (col1 String, col2 String, col3 String, col4 String, col5 string)

Modified: hive/trunk/ql/src/test/queries/clientpositive/list_bucket_dml_2.q
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientpositive/list_bucket_dml_2.q?rev=1647912&r1=1647911&r2=1647912&view=diff
==============================================================================
--- hive/trunk/ql/src/test/queries/clientpositive/list_bucket_dml_2.q (original)
+++ hive/trunk/ql/src/test/queries/clientpositive/list_bucket_dml_2.q Thu Dec 25 18:06:30 2014
@@ -10,6 +10,7 @@ set hive.stats.reliable=true;

  -- INCLUDE_HADOOP_MAJOR_VERSIONS(0.23)
  -- SORT_QUERY_RESULTS
+-- JAVA_VERSION_SPECIFIC_OUTPUT

  -- list bucketing DML: static partition. multiple skewed columns.
  -- ds=2008-04-08/hr=11/HIVE_DEFAULT_LIST_BUCKETING_DIR_NAME:

Modified: hive/trunk/ql/src/test/queries/clientpositive/list_bucket_dml_4.q
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientpositive/list_bucket_dml_4.q?rev=1647912&r1=1647911&r2=1647912&view=diff
==============================================================================
--- hive/trunk/ql/src/test/queries/clientpositive/list_bucket_dml_4.q (original)
+++ hive/trunk/ql/src/test/queries/clientpositive/list_bucket_dml_4.q Thu Dec 25 18:06:30 2014
@@ -9,6 +9,7 @@ set hive.merge.mapredfiles=false;

  -- INCLUDE_HADOOP_MAJOR_VERSIONS(0.23)
  -- SORT_QUERY_RESULTS
+-- JAVA_VERSION_SPECIFIC_OUTPUT

  -- list bucketing DML: static partition. multiple skewed columns. merge.
  -- ds=2008-04-08/hr=11/HIVE_DEFAULT_LIST_BUCKETING_DIR_NAME:

Modified: hive/trunk/ql/src/test/queries/clientpositive/list_bucket_dml_5.q
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientpositive/list_bucket_dml_5.q?rev=1647912&r1=1647911&r2=1647912&view=diff
==============================================================================
--- hive/trunk/ql/src/test/queries/clientpositive/list_bucket_dml_5.q (original)
+++ hive/trunk/ql/src/test/queries/clientpositive/list_bucket_dml_5.q Thu Dec 25 18:06:30 2014
@@ -10,6 +10,7 @@ set mapred.input.dir.recursive=true;

  -- INCLUDE_HADOOP_MAJOR_VERSIONS(0.23)
  -- SORT_QUERY_RESULTS
+-- JAVA_VERSION_SPECIFIC_OUTPUT

  -- create a skewed table
  create table list_bucketing_dynamic_part (key String, value String)

Modified: hive/trunk/ql/src/test/queries/clientpositive/list_bucket_dml_8.q
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientpositive/list_bucket_dml_8.q?rev=1647912&r1=1647911&r2=1647912&view=diff
==============================================================================
--- hive/trunk/ql/src/test/queries/clientpositive/list_bucket_dml_8.q (original)
+++ hive/trunk/ql/src/test/queries/clientpositive/list_bucket_dml_8.q Thu Dec 25 18:06:30 2014
@@ -48,6 +48,7 @@ set hive.merge.mapredfiles=false;
  -- 118 000002_0

  -- INCLUDE_HADOOP_MAJOR_VERSIONS(0.23)
+-- JAVA_VERSION_SPECIFIC_OUTPUT

  -- create a skewed table
  create table list_bucketing_dynamic_part (key String, value String)

Modified: hive/trunk/ql/src/test/queries/clientpositive/list_bucket_dml_9.q
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientpositive/list_bucket_dml_9.q?rev=1647912&r1=1647911&r2=1647912&view=diff
==============================================================================
--- hive/trunk/ql/src/test/queries/clientpositive/list_bucket_dml_9.q (original)
+++ hive/trunk/ql/src/test/queries/clientpositive/list_bucket_dml_9.q Thu Dec 25 18:06:30 2014
@@ -9,6 +9,7 @@ set hive.merge.mapredfiles=false;

  -- INCLUDE_HADOOP_MAJOR_VERSIONS(0.23)
  -- SORT_QUERY_RESULTS
+-- JAVA_VERSION_SPECIFIC_OUTPUT

  -- list bucketing DML: static partition. multiple skewed columns. merge.
  -- ds=2008-04-08/hr=11/HIVE_DEFAULT_LIST_BUCKETING_DIR_NAME:

Modified: hive/trunk/ql/src/test/queries/clientpositive/stats_list_bucket.q
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientpositive/stats_list_bucket.q?rev=1647912&r1=1647911&r2=1647912&view=diff
==============================================================================
--- hive/trunk/ql/src/test/queries/clientpositive/stats_list_bucket.q (original)
+++ hive/trunk/ql/src/test/queries/clientpositive/stats_list_bucket.q Thu Dec 25 18:06:30 2014
@@ -2,6 +2,7 @@
  set hive.mapred.supports.subdirectories=true;

  -- INCLUDE_HADOOP_MAJOR_VERSIONS(0.23)
+-- JAVA_VERSION_SPECIFIC_OUTPUT

  drop table stats_list_bucket;
  drop table stats_list_bucket_1;

Added: hive/trunk/ql/src/test/results/clientpositive/list_bucket_dml_11.q.java1.7.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/list_bucket_dml_11.q.java1.7.out?rev=1647912&view=auto
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/list_bucket_dml_11.q.java1.7.out (added)
+++ hive/trunk/ql/src/test/results/clientpositive/list_bucket_dml_11.q.java1.7.out Thu Dec 25 18:06:30 2014
@@ -0,0 +1,426 @@
+PREHOOK: query: -- Ensure it works if skewed column is not the first column in the table columns
+
+-- INCLUDE_HADOOP_MAJOR_VERSIONS(0.23)
+-- JAVA_VERSION_SPECIFIC_OUTPUT
+
+-- list bucketing DML: static partition. multiple skewed columns.
+
+-- create a skewed table
+create table list_bucketing_static_part (key String, value String)
+ partitioned by (ds String, hr String)
+ skewed by (value) on ('val_466','val_287','val_82')
+ stored as DIRECTORIES
+ STORED AS RCFILE
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@list_bucketing_static_part
+POSTHOOK: query: -- Ensure it works if skewed column is not the first column in the table columns
+
+-- INCLUDE_HADOOP_MAJOR_VERSIONS(0.23)
+-- JAVA_VERSION_SPECIFIC_OUTPUT
+
+-- list bucketing DML: static partition. multiple skewed columns.
+
+-- create a skewed table
+create table list_bucketing_static_part (key String, value String)
+ partitioned by (ds String, hr String)
+ skewed by (value) on ('val_466','val_287','val_82')
+ stored as DIRECTORIES
+ STORED AS RCFILE
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@list_bucketing_static_part
+PREHOOK: query: -- list bucketing DML without merge. use bucketize to generate a few small files.
+explain extended
+insert overwrite table list_bucketing_static_part partition (ds = '2008-04-08', hr = '11')
+select key, value from src
+PREHOOK: type: QUERY
+POSTHOOK: query: -- list bucketing DML without merge. use bucketize to generate a few small files.
+explain extended
+insert overwrite table list_bucketing_static_part partition (ds = '2008-04-08', hr = '11')
+select key, value from src
+POSTHOOK: type: QUERY
+ABSTRACT SYNTAX TREE:
+
+TOK_QUERY
+ TOK_FROM
+ TOK_TABREF
+ TOK_TABNAME
+ src
+ TOK_INSERT
+ TOK_DESTINATION
+ TOK_TAB
+ TOK_TABNAME
+ list_bucketing_static_part
+ TOK_PARTSPEC
+ TOK_PARTVAL
+ ds
+ '2008-04-08'
+ TOK_PARTVAL
+ hr
+ '11'
+ TOK_SELECT
+ TOK_SELEXPR
+ TOK_TABLE_OR_COL
+ key
+ TOK_SELEXPR
+ TOK_TABLE_OR_COL
+ value
+
+
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+ Stage-2 depends on stages: Stage-0
+
+STAGE PLANS:
+ Stage: Stage-1
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ alias: src
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ GatherStats: false
+ Select Operator
+ expressions: key (type: string), value (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ GlobalTableId: 1
+#### A masked pattern was here ####
+ NumFilesPerFileSink: 1
+ Static Partition Specification: ds=2008-04-08/hr=11/
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+#### A masked pattern was here ####
+ table:
+ input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat
+ properties:
+ bucket_count -1
+ columns key,value
+ columns.comments
+ columns.types string:string
+#### A masked pattern was here ####
+ name default.list_bucketing_static_part
+ partition_columns ds/hr
+ partition_columns.types string:string
+ serialization.ddl struct list_bucketing_static_part { string key, string value}
+ serialization.format 1
+ serialization.lib org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
+#### A masked pattern was here ####
+ serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
+ name: default.list_bucketing_static_part
+ TotalFiles: 1
+ GatherStats: true
+ MultiFileSpray: false
+ Path -> Alias:
+#### A masked pattern was here ####
+ Path -> Partition:
+#### A masked pattern was here ####
+ Partition
+ base file name: src
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ properties:
+ COLUMN_STATS_ACCURATE true
+ bucket_count -1
+ columns key,value
+ columns.comments 'default','default'
+ columns.types string:string
+#### A masked pattern was here ####
+ name default.src
+ numFiles 1
+ numRows 500
+ rawDataSize 5312
+ serialization.ddl struct src { string key, string value}
+ serialization.format 1
+ serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ totalSize 5812
+#### A masked pattern was here ####
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ properties:
+ COLUMN_STATS_ACCURATE true
+ bucket_count -1
+ columns key,value
+ columns.comments 'default','default'
+ columns.types string:string
+#### A masked pattern was here ####
+ name default.src
+ numFiles 1
+ numRows 500
+ rawDataSize 5312
+ serialization.ddl struct src { string key, string value}
+ serialization.format 1
+ serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ totalSize 5812
+#### A masked pattern was here ####
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.src
+ name: default.src
+ Truncated Path -> Alias:
+ /src [src]
+
+ Stage: Stage-0
+ Move Operator
+ tables:
+ partition:
+ ds 2008-04-08
+ hr 11
+ replace: true
+#### A masked pattern was here ####
+ table:
+ input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat
+ properties:
+ bucket_count -1
+ columns key,value
+ columns.comments
+ columns.types string:string
+#### A masked pattern was here ####
+ name default.list_bucketing_static_part
+ partition_columns ds/hr
+ partition_columns.types string:string
+ serialization.ddl struct list_bucketing_static_part { string key, string value}
+ serialization.format 1
+ serialization.lib org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
+#### A masked pattern was here ####
+ serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
+ name: default.list_bucketing_static_part
+
+ Stage: Stage-2
+ Stats-Aggr Operator
+#### A masked pattern was here ####
+
+PREHOOK: query: insert overwrite table list_bucketing_static_part partition (ds = '2008-04-08', hr = '11')
+select key, value from src
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Output: default@list_bucketing_static_part@ds=2008-04-08/hr=11
+POSTHOOK: query: insert overwrite table list_bucketing_static_part partition (ds = '2008-04-08', hr = '11')
+select key, value from src
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Output: default@list_bucketing_static_part@ds=2008-04-08/hr=11
+POSTHOOK: Lineage: list_bucketing_static_part PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: list_bucketing_static_part PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+PREHOOK: query: -- check DML result
+show partitions list_bucketing_static_part
+PREHOOK: type: SHOWPARTITIONS
+PREHOOK: Input: default@list_bucketing_static_part
+POSTHOOK: query: -- check DML result
+show partitions list_bucketing_static_part
+POSTHOOK: type: SHOWPARTITIONS
+POSTHOOK: Input: default@list_bucketing_static_part
+ds=2008-04-08/hr=11
+PREHOOK: query: desc formatted list_bucketing_static_part partition (ds='2008-04-08', hr='11')
+PREHOOK: type: DESCTABLE
+PREHOOK: Input: default@list_bucketing_static_part
+POSTHOOK: query: desc formatted list_bucketing_static_part partition (ds='2008-04-08', hr='11')
+POSTHOOK: type: DESCTABLE
+POSTHOOK: Input: default@list_bucketing_static_part
+# col_name data_type comment
+
+key string
+value string
+
+# Partition Information
+# col_name data_type comment
+
+ds string
+hr string
+
+# Detailed Partition Information
+Partition Value: [2008-04-08, 11]
+Database: default
+Table: list_bucketing_static_part
+#### A masked pattern was here ####
+Protect Mode: None
+#### A masked pattern was here ####
+Partition Parameters:
+ COLUMN_STATS_ACCURATE true
+ numFiles 4
+ numRows 500
+ rawDataSize 4812
+ totalSize 5522
+#### A masked pattern was here ####
+
+# Storage Information
+SerDe Library: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
+InputFormat: org.apache.hadoop.hive.ql.io.RCFileInputFormat
+OutputFormat: org.apache.hadoop.hive.ql.io.RCFileOutputFormat
+Compressed: No
+Num Buckets: -1
+Bucket Columns: []
+Sort Columns: []
+Stored As SubDirectories: Yes
+Skewed Columns: [value]
+Skewed Values: [[val_466], [val_287], [val_82]]
+#### A masked pattern was here ####
+Skewed Value to Truncated Path: {[val_82]=/list_bucketing_static_part/ds=2008-04-08/hr=11/value=val_82, [val_287]=/list_bucketing_static_part/ds=2008-04-08/hr=11/value=val_287, [val_466]=/list_bucketing_static_part/ds=2008-04-08/hr=11/value=val_466}
+Storage Desc Params:
+ serialization.format 1
+PREHOOK: query: explain extended
+select key, value from list_bucketing_static_part where ds='2008-04-08' and hr='11' and value = "val_466"
+PREHOOK: type: QUERY
+POSTHOOK: query: explain extended
+select key, value from list_bucketing_static_part where ds='2008-04-08' and hr='11' and value = "val_466"
+POSTHOOK: type: QUERY
+ABSTRACT SYNTAX TREE:
+
+TOK_QUERY
+ TOK_FROM
+ TOK_TABREF
+ TOK_TABNAME
+ list_bucketing_static_part
+ TOK_INSERT
+ TOK_DESTINATION
+ TOK_DIR
+ TOK_TMP_FILE
+ TOK_SELECT
+ TOK_SELEXPR
+ TOK_TABLE_OR_COL
+ key
+ TOK_SELEXPR
+ TOK_TABLE_OR_COL
+ value
+ TOK_WHERE
+ and
+ and
+ =
+ TOK_TABLE_OR_COL
+ ds
+ '2008-04-08'
+ =
+ TOK_TABLE_OR_COL
+ hr
+ '11'
+ =
+ TOK_TABLE_OR_COL
+ value
+ "val_466"
+
+
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ alias: list_bucketing_static_part
+ Statistics: Num rows: 500 Data size: 4812 Basic stats: COMPLETE Column stats: NONE
+ GatherStats: false
+ Filter Operator
+ isSamplingPred: false
+ predicate: (value = 'val_466') (type: boolean)
+ Statistics: Num rows: 250 Data size: 2406 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: key (type: string), 'val_466' (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 250 Data size: 2406 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ GlobalTableId: 0
+#### A masked pattern was here ####
+ NumFilesPerFileSink: 1
+ Statistics: Num rows: 250 Data size: 2406 Basic stats: COMPLETE Column stats: NONE
+#### A masked pattern was here ####
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ properties:
+ columns _col0,_col1
+ columns.types string:string
+ escape.delim \
+ hive.serialization.extend.nesting.levels true
+ serialization.format 1
+ serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ TotalFiles: 1
+ GatherStats: false
+ MultiFileSpray: false
+ Path -> Alias:
+#### A masked pattern was here ####
+ Path -> Partition:
+#### A masked pattern was here ####
+ Partition
+ base file name: value=val_466
+ input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat
+ partition values:
+ ds 2008-04-08
+ hr 11
+ properties:
+ COLUMN_STATS_ACCURATE true
+ bucket_count -1
+ columns key,value
+ columns.comments
+ columns.types string:string
+#### A masked pattern was here ####
+ name default.list_bucketing_static_part
+ numFiles 4
+ numRows 500
+ partition_columns ds/hr
+ partition_columns.types string:string
+ rawDataSize 4812
+ serialization.ddl struct list_bucketing_static_part { string key, string value}
+ serialization.format 1
+ serialization.lib org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
+ totalSize 5522
+#### A masked pattern was here ####
+ serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
+
+ input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat
+ properties:
+ bucket_count -1
+ columns key,value
+ columns.comments
+ columns.types string:string
+#### A masked pattern was here ####
+ name default.list_bucketing_static_part
+ partition_columns ds/hr
+ partition_columns.types string:string
+ serialization.ddl struct list_bucketing_static_part { string key, string value}
+ serialization.format 1
+ serialization.lib org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
+#### A masked pattern was here ####
+ serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
+ name: default.list_bucketing_static_part
+ name: default.list_bucketing_static_part
+ Truncated Path -> Alias:
+ /list_bucketing_static_part/ds=2008-04-08/hr=11/value=val_466 [$hdt$_0:list_bucketing_static_part]
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: select key, value from list_bucketing_static_part where ds='2008-04-08' and hr='11' and value = "val_466"
+PREHOOK: type: QUERY
+PREHOOK: Input: default@list_bucketing_static_part
+PREHOOK: Input: default@list_bucketing_static_part@ds=2008-04-08/hr=11
+#### A masked pattern was here ####
+POSTHOOK: query: select key, value from list_bucketing_static_part where ds='2008-04-08' and hr='11' and value = "val_466"
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@list_bucketing_static_part
+POSTHOOK: Input: default@list_bucketing_static_part@ds=2008-04-08/hr=11
+#### A masked pattern was here ####
+466 val_466
+466 val_466
+466 val_466
+PREHOOK: query: drop table list_bucketing_static_part
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@list_bucketing_static_part
+PREHOOK: Output: default@list_bucketing_static_part
+POSTHOOK: query: drop table list_bucketing_static_part
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@list_bucketing_static_part
+POSTHOOK: Output: default@list_bucketing_static_part

Added: hive/trunk/ql/src/test/results/clientpositive/list_bucket_dml_11.q.java1.8.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/list_bucket_dml_11.q.java1.8.out?rev=1647912&view=auto
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/list_bucket_dml_11.q.java1.8.out (added)
+++ hive/trunk/ql/src/test/results/clientpositive/list_bucket_dml_11.q.java1.8.out Thu Dec 25 18:06:30 2014
@@ -0,0 +1,426 @@
+PREHOOK: query: -- Ensure it works if skewed column is not the first column in the table columns
+
+-- INCLUDE_HADOOP_MAJOR_VERSIONS(0.23)
+-- JAVA_VERSION_SPECIFIC_OUTPUT
+
+-- list bucketing DML: static partition. multiple skewed columns.
+
+-- create a skewed table
+create table list_bucketing_static_part (key String, value String)
+ partitioned by (ds String, hr String)
+ skewed by (value) on ('val_466','val_287','val_82')
+ stored as DIRECTORIES
+ STORED AS RCFILE
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@list_bucketing_static_part
+POSTHOOK: query: -- Ensure it works if skewed column is not the first column in the table columns
+
+-- INCLUDE_HADOOP_MAJOR_VERSIONS(0.23)
+-- JAVA_VERSION_SPECIFIC_OUTPUT
+
+-- list bucketing DML: static partition. multiple skewed columns.
+
+-- create a skewed table
+create table list_bucketing_static_part (key String, value String)
+ partitioned by (ds String, hr String)
+ skewed by (value) on ('val_466','val_287','val_82')
+ stored as DIRECTORIES
+ STORED AS RCFILE
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@list_bucketing_static_part
+PREHOOK: query: -- list bucketing DML without merge. use bucketize to generate a few small files.
+explain extended
+insert overwrite table list_bucketing_static_part partition (ds = '2008-04-08', hr = '11')
+select key, value from src
+PREHOOK: type: QUERY
+POSTHOOK: query: -- list bucketing DML without merge. use bucketize to generate a few small files.
+explain extended
+insert overwrite table list_bucketing_static_part partition (ds = '2008-04-08', hr = '11')
+select key, value from src
+POSTHOOK: type: QUERY
+ABSTRACT SYNTAX TREE:
+
+TOK_QUERY
+ TOK_FROM
+ TOK_TABREF
+ TOK_TABNAME
+ src
+ TOK_INSERT
+ TOK_DESTINATION
+ TOK_TAB
+ TOK_TABNAME
+ list_bucketing_static_part
+ TOK_PARTSPEC
+ TOK_PARTVAL
+ ds
+ '2008-04-08'
+ TOK_PARTVAL
+ hr
+ '11'
+ TOK_SELECT
+ TOK_SELEXPR
+ TOK_TABLE_OR_COL
+ key
+ TOK_SELEXPR
+ TOK_TABLE_OR_COL
+ value
+
+
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+ Stage-2 depends on stages: Stage-0
+
+STAGE PLANS:
+ Stage: Stage-1
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ alias: src
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ GatherStats: false
+ Select Operator
+ expressions: key (type: string), value (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ GlobalTableId: 1
+#### A masked pattern was here ####
+ NumFilesPerFileSink: 1
+ Static Partition Specification: ds=2008-04-08/hr=11/
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+#### A masked pattern was here ####
+ table:
+ input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat
+ properties:
+ bucket_count -1
+ columns key,value
+ columns.comments
+ columns.types string:string
+#### A masked pattern was here ####
+ name default.list_bucketing_static_part
+ partition_columns ds/hr
+ partition_columns.types string:string
+ serialization.ddl struct list_bucketing_static_part { string key, string value}
+ serialization.format 1
+ serialization.lib org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
+#### A masked pattern was here ####
+ serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
+ name: default.list_bucketing_static_part
+ TotalFiles: 1
+ GatherStats: true
+ MultiFileSpray: false
+ Path -> Alias:
+#### A masked pattern was here ####
+ Path -> Partition:
+#### A masked pattern was here ####
+ Partition
+ base file name: src
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ properties:
+ COLUMN_STATS_ACCURATE true
+ bucket_count -1
+ columns key,value
+ columns.comments 'default','default'
+ columns.types string:string
+#### A masked pattern was here ####
+ name default.src
+ numFiles 1
+ numRows 500
+ rawDataSize 5312
+ serialization.ddl struct src { string key, string value}
+ serialization.format 1
+ serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ totalSize 5812
+#### A masked pattern was here ####
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ properties:
+ COLUMN_STATS_ACCURATE true
+ bucket_count -1
+ columns key,value
+ columns.comments 'default','default'
+ columns.types string:string
+#### A masked pattern was here ####
+ name default.src
+ numFiles 1
+ numRows 500
+ rawDataSize 5312
+ serialization.ddl struct src { string key, string value}
+ serialization.format 1
+ serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ totalSize 5812
+#### A masked pattern was here ####
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.src
+ name: default.src
+ Truncated Path -> Alias:
+ /src [src]
+
+ Stage: Stage-0
+ Move Operator
+ tables:
+ partition:
+ ds 2008-04-08
+ hr 11
+ replace: true
+#### A masked pattern was here ####
+ table:
+ input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat
+ properties:
+ bucket_count -1
+ columns key,value
+ columns.comments
+ columns.types string:string
+#### A masked pattern was here ####
+ name default.list_bucketing_static_part
+ partition_columns ds/hr
+ partition_columns.types string:string
+ serialization.ddl struct list_bucketing_static_part { string key, string value}
+ serialization.format 1
+ serialization.lib org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
+#### A masked pattern was here ####
+ serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
+ name: default.list_bucketing_static_part
+
+ Stage: Stage-2
+ Stats-Aggr Operator
+#### A masked pattern was here ####
+
+PREHOOK: query: insert overwrite table list_bucketing_static_part partition (ds = '2008-04-08', hr = '11')
+select key, value from src
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Output: default@list_bucketing_static_part@ds=2008-04-08/hr=11
+POSTHOOK: query: insert overwrite table list_bucketing_static_part partition (ds = '2008-04-08', hr = '11')
+select key, value from src
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Output: default@list_bucketing_static_part@ds=2008-04-08/hr=11
+POSTHOOK: Lineage: list_bucketing_static_part PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: list_bucketing_static_part PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+PREHOOK: query: -- check DML result
+show partitions list_bucketing_static_part
+PREHOOK: type: SHOWPARTITIONS
+PREHOOK: Input: default@list_bucketing_static_part
+POSTHOOK: query: -- check DML result
+show partitions list_bucketing_static_part
+POSTHOOK: type: SHOWPARTITIONS
+POSTHOOK: Input: default@list_bucketing_static_part
+ds=2008-04-08/hr=11
+PREHOOK: query: desc formatted list_bucketing_static_part partition (ds='2008-04-08', hr='11')
+PREHOOK: type: DESCTABLE
+PREHOOK: Input: default@list_bucketing_static_part
+POSTHOOK: query: desc formatted list_bucketing_static_part partition (ds='2008-04-08', hr='11')
+POSTHOOK: type: DESCTABLE
+POSTHOOK: Input: default@list_bucketing_static_part
+# col_name data_type comment
+
+key string
+value string
+
+# Partition Information
+# col_name data_type comment
+
+ds string
+hr string
+
+# Detailed Partition Information
+Partition Value: [2008-04-08, 11]
+Database: default
+Table: list_bucketing_static_part
+#### A masked pattern was here ####
+Protect Mode: None
+#### A masked pattern was here ####
+Partition Parameters:
+ COLUMN_STATS_ACCURATE true
+ numFiles 4
+ numRows 500
+ rawDataSize 4812
+ totalSize 5522
+#### A masked pattern was here ####
+
+# Storage Information
+SerDe Library: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
+InputFormat: org.apache.hadoop.hive.ql.io.RCFileInputFormat
+OutputFormat: org.apache.hadoop.hive.ql.io.RCFileOutputFormat
+Compressed: No
+Num Buckets: -1
+Bucket Columns: []
+Sort Columns: []
+Stored As SubDirectories: Yes
+Skewed Columns: [value]
+Skewed Values: [[val_466], [val_287], [val_82]]
+#### A masked pattern was here ####
+Skewed Value to Truncated Path: {[val_287]=/list_bucketing_static_part/ds=2008-04-08/hr=11/value=val_287, [val_82]=/list_bucketing_static_part/ds=2008-04-08/hr=11/value=val_82, [val_466]=/list_bucketing_static_part/ds=2008-04-08/hr=11/value=val_466}
+Storage Desc Params:
+ serialization.format 1
+PREHOOK: query: explain extended
+select key, value from list_bucketing_static_part where ds='2008-04-08' and hr='11' and value = "val_466"
+PREHOOK: type: QUERY
+POSTHOOK: query: explain extended
+select key, value from list_bucketing_static_part where ds='2008-04-08' and hr='11' and value = "val_466"
+POSTHOOK: type: QUERY
+ABSTRACT SYNTAX TREE:
+
+TOK_QUERY
+ TOK_FROM
+ TOK_TABREF
+ TOK_TABNAME
+ list_bucketing_static_part
+ TOK_INSERT
+ TOK_DESTINATION
+ TOK_DIR
+ TOK_TMP_FILE
+ TOK_SELECT
+ TOK_SELEXPR
+ TOK_TABLE_OR_COL
+ key
+ TOK_SELEXPR
+ TOK_TABLE_OR_COL
+ value
+ TOK_WHERE
+ and
+ and
+ =
+ TOK_TABLE_OR_COL
+ ds
+ '2008-04-08'
+ =
+ TOK_TABLE_OR_COL
+ hr
+ '11'
+ =
+ TOK_TABLE_OR_COL
+ value
+ "val_466"
+
+
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ alias: list_bucketing_static_part
+ Statistics: Num rows: 500 Data size: 4812 Basic stats: COMPLETE Column stats: NONE
+ GatherStats: false
+ Filter Operator
+ isSamplingPred: false
+ predicate: (value = 'val_466') (type: boolean)
+ Statistics: Num rows: 250 Data size: 2406 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: key (type: string), 'val_466' (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 250 Data size: 2406 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ GlobalTableId: 0
+#### A masked pattern was here ####
+ NumFilesPerFileSink: 1
+ Statistics: Num rows: 250 Data size: 2406 Basic stats: COMPLETE Column stats: NONE
+#### A masked pattern was here ####
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ properties:
+ columns _col0,_col1
+ columns.types string:string
+ escape.delim \
+ hive.serialization.extend.nesting.levels true
+ serialization.format 1
+ serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ TotalFiles: 1
+ GatherStats: false
+ MultiFileSpray: false
+ Path -> Alias:
+#### A masked pattern was here ####
+ Path -> Partition:
+#### A masked pattern was here ####
+ Partition
+ base file name: value=val_466
+ input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat
+ partition values:
+ ds 2008-04-08
+ hr 11
+ properties:
+ COLUMN_STATS_ACCURATE true
+ bucket_count -1
+ columns key,value
+ columns.comments
+ columns.types string:string
+#### A masked pattern was here ####
+ name default.list_bucketing_static_part
+ numFiles 4
+ numRows 500
+ partition_columns ds/hr
+ partition_columns.types string:string
+ rawDataSize 4812
+ serialization.ddl struct list_bucketing_static_part { string key, string value}
+ serialization.format 1
+ serialization.lib org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
+ totalSize 5522
+#### A masked pattern was here ####
+ serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
+
+ input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat
+ properties:
+ bucket_count -1
+ columns key,value
+ columns.comments
+ columns.types string:string
+#### A masked pattern was here ####
+ name default.list_bucketing_static_part
+ partition_columns ds/hr
+ partition_columns.types string:string
+ serialization.ddl struct list_bucketing_static_part { string key, string value}
+ serialization.format 1
+ serialization.lib org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
+#### A masked pattern was here ####
+ serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
+ name: default.list_bucketing_static_part
+ name: default.list_bucketing_static_part
+ Truncated Path -> Alias:
+ /list_bucketing_static_part/ds=2008-04-08/hr=11/value=val_466 [$hdt$_0:list_bucketing_static_part]
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: select key, value from list_bucketing_static_part where ds='2008-04-08' and hr='11' and value = "val_466"
+PREHOOK: type: QUERY
+PREHOOK: Input: default@list_bucketing_static_part
+PREHOOK: Input: default@list_bucketing_static_part@ds=2008-04-08/hr=11
+#### A masked pattern was here ####
+POSTHOOK: query: select key, value from list_bucketing_static_part where ds='2008-04-08' and hr='11' and value = "val_466"
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@list_bucketing_static_part
+POSTHOOK: Input: default@list_bucketing_static_part@ds=2008-04-08/hr=11
+#### A masked pattern was here ####
+466 val_466
+466 val_466
+466 val_466
+PREHOOK: query: drop table list_bucketing_static_part
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@list_bucketing_static_part
+PREHOOK: Output: default@list_bucketing_static_part
+POSTHOOK: query: drop table list_bucketing_static_part
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@list_bucketing_static_part
+POSTHOOK: Output: default@list_bucketing_static_part

Added: hive/trunk/ql/src/test/results/clientpositive/list_bucket_dml_12.q.java1.7.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/list_bucket_dml_12.q.java1.7.out?rev=1647912&view=auto
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/list_bucket_dml_12.q.java1.7.out (added)
+++ hive/trunk/ql/src/test/results/clientpositive/list_bucket_dml_12.q.java1.7.out Thu Dec 25 18:06:30 2014
@@ -0,0 +1,598 @@
+PREHOOK: query: -- Ensure it works if skewed column is not the first column in the table columns
+
+-- INCLUDE_HADOOP_MAJOR_VERSIONS(0.23)
+-- SORT_QUERY_RESULTS
+-- JAVA_VERSION_SPECIFIC_OUTPUT
+
+-- test where the skewed values are more than 1 say columns no. 2 and 4 in a table with 5 columns
+create table list_bucketing_mul_col (col1 String, col2 String, col3 String, col4 String, col5 string)
+ partitioned by (ds String, hr String)
+ skewed by (col2, col4) on (('466','val_466'),('287','val_287'),('82','val_82'))
+ stored as DIRECTORIES
+ STORED AS RCFILE
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@list_bucketing_mul_col
+POSTHOOK: query: -- Ensure it works if skewed column is not the first column in the table columns
+
+-- INCLUDE_HADOOP_MAJOR_VERSIONS(0.23)
+-- SORT_QUERY_RESULTS
+-- JAVA_VERSION_SPECIFIC_OUTPUT
+
+-- test where the skewed values are more than 1 say columns no. 2 and 4 in a table with 5 columns
+create table list_bucketing_mul_col (col1 String, col2 String, col3 String, col4 String, col5 string)
+ partitioned by (ds String, hr String)
+ skewed by (col2, col4) on (('466','val_466'),('287','val_287'),('82','val_82'))
+ stored as DIRECTORIES
+ STORED AS RCFILE
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@list_bucketing_mul_col
+PREHOOK: query: -- list bucketing DML
+explain extended
+insert overwrite table list_bucketing_mul_col partition (ds = '2008-04-08', hr = '11')
+select 1, key, 1, value, 1 from src
+PREHOOK: type: QUERY
+POSTHOOK: query: -- list bucketing DML
+explain extended
+insert overwrite table list_bucketing_mul_col partition (ds = '2008-04-08', hr = '11')
+select 1, key, 1, value, 1 from src
+POSTHOOK: type: QUERY
+ABSTRACT SYNTAX TREE:
+
+TOK_QUERY
+ TOK_FROM
+ TOK_TABREF
+ TOK_TABNAME
+ src
+ TOK_INSERT
+ TOK_DESTINATION
+ TOK_TAB
+ TOK_TABNAME
+ list_bucketing_mul_col
+ TOK_PARTSPEC
+ TOK_PARTVAL
+ ds
+ '2008-04-08'
+ TOK_PARTVAL
+ hr
+ '11'
+ TOK_SELECT
+ TOK_SELEXPR
+ 1
+ TOK_SELEXPR
+ TOK_TABLE_OR_COL
+ key
+ TOK_SELEXPR
+ 1
+ TOK_SELEXPR
+ TOK_TABLE_OR_COL
+ value
+ TOK_SELEXPR
+ 1
+
+
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+ Stage-2 depends on stages: Stage-0
+
+STAGE PLANS:
+ Stage: Stage-1
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ alias: src
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ GatherStats: false
+ Select Operator
+ expressions: UDFToString(1) (type: string), key (type: string), UDFToString(1) (type: string), value (type: string), UDFToString(1) (type: string)
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ GlobalTableId: 1
+#### A masked pattern was here ####
+ NumFilesPerFileSink: 1
+ Static Partition Specification: ds=2008-04-08/hr=11/
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+#### A masked pattern was here ####
+ table:
+ input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat
+ properties:
+ bucket_count -1
+ columns col1,col2,col3,col4,col5
+ columns.comments
+ columns.types string:string:string:string:string
+#### A masked pattern was here ####
+ name default.list_bucketing_mul_col
+ partition_columns ds/hr
+ partition_columns.types string:string
+ serialization.ddl struct list_bucketing_mul_col { string col1, string col2, string col3, string col4, string col5}
+ serialization.format 1
+ serialization.lib org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
+#### A masked pattern was here ####
+ serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
+ name: default.list_bucketing_mul_col
+ TotalFiles: 1
+ GatherStats: true
+ MultiFileSpray: false
+ Path -> Alias:
+#### A masked pattern was here ####
+ Path -> Partition:
+#### A masked pattern was here ####
+ Partition
+ base file name: src
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ properties:
+ COLUMN_STATS_ACCURATE true
+ bucket_count -1
+ columns key,value
+ columns.comments 'default','default'
+ columns.types string:string
+#### A masked pattern was here ####
+ name default.src
+ numFiles 1
+ numRows 500
+ rawDataSize 5312
+ serialization.ddl struct src { string key, string value}
+ serialization.format 1
+ serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ totalSize 5812
+#### A masked pattern was here ####
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ properties:
+ COLUMN_STATS_ACCURATE true
+ bucket_count -1
+ columns key,value
+ columns.comments 'default','default'
+ columns.types string:string
+#### A masked pattern was here ####
+ name default.src
+ numFiles 1
+ numRows 500
+ rawDataSize 5312
+ serialization.ddl struct src { string key, string value}
+ serialization.format 1
+ serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ totalSize 5812
+#### A masked pattern was here ####
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.src
+ name: default.src
+ Truncated Path -> Alias:
+ /src [$hdt$_0:src]
+
+ Stage: Stage-0
+ Move Operator
+ tables:
+ partition:
+ ds 2008-04-08
+ hr 11
+ replace: true
+#### A masked pattern was here ####
+ table:
+ input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat
+ properties:
+ bucket_count -1
+ columns col1,col2,col3,col4,col5
+ columns.comments
+ columns.types string:string:string:string:string
+#### A masked pattern was here ####
+ name default.list_bucketing_mul_col
+ partition_columns ds/hr
+ partition_columns.types string:string
+ serialization.ddl struct list_bucketing_mul_col { string col1, string col2, string col3, string col4, string col5}
+ serialization.format 1
+ serialization.lib org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
+#### A masked pattern was here ####
+ serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
+ name: default.list_bucketing_mul_col
+
+ Stage: Stage-2
+ Stats-Aggr Operator
+#### A masked pattern was here ####
+
+PREHOOK: query: insert overwrite table list_bucketing_mul_col partition (ds = '2008-04-08', hr = '11')
+select 1, key, 1, value, 1 from src
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Output: default@list_bucketing_mul_col@ds=2008-04-08/hr=11
+POSTHOOK: query: insert overwrite table list_bucketing_mul_col partition (ds = '2008-04-08', hr = '11')
+select 1, key, 1, value, 1 from src
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Output: default@list_bucketing_mul_col@ds=2008-04-08/hr=11
+POSTHOOK: Lineage: list_bucketing_mul_col PARTITION(ds=2008-04-08,hr=11).col1 EXPRESSION []
+POSTHOOK: Lineage: list_bucketing_mul_col PARTITION(ds=2008-04-08,hr=11).col2 SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: list_bucketing_mul_col PARTITION(ds=2008-04-08,hr=11).col3 EXPRESSION []
+POSTHOOK: Lineage: list_bucketing_mul_col PARTITION(ds=2008-04-08,hr=11).col4 SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: list_bucketing_mul_col PARTITION(ds=2008-04-08,hr=11).col5 EXPRESSION []
+PREHOOK: query: -- check DML result
+show partitions list_bucketing_mul_col
+PREHOOK: type: SHOWPARTITIONS
+PREHOOK: Input: default@list_bucketing_mul_col
+POSTHOOK: query: -- check DML result
+show partitions list_bucketing_mul_col
+POSTHOOK: type: SHOWPARTITIONS
+POSTHOOK: Input: default@list_bucketing_mul_col
+ds=2008-04-08/hr=11
+PREHOOK: query: desc formatted list_bucketing_mul_col partition (ds='2008-04-08', hr='11')
+PREHOOK: type: DESCTABLE
+PREHOOK: Input: default@list_bucketing_mul_col
+POSTHOOK: query: desc formatted list_bucketing_mul_col partition (ds='2008-04-08', hr='11')
+POSTHOOK: type: DESCTABLE
+POSTHOOK: Input: default@list_bucketing_mul_col
+# col_name data_type comment
+
+col1 string
+col2 string
+col3 string
+col4 string
+col5 string
+
+# Partition Information
+# col_name data_type comment
+
+ds string
+hr string
+
+# Detailed Partition Information
+Partition Value: [2008-04-08, 11]
+Database: default
+Table: list_bucketing_mul_col
+#### A masked pattern was here ####
+Protect Mode: None
+#### A masked pattern was here ####
+Partition Parameters:
+ COLUMN_STATS_ACCURATE true
+ numFiles 4
+ numRows 500
+ rawDataSize 6312
+ totalSize 7094
+#### A masked pattern was here ####
+
+# Storage Information
+SerDe Library: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
+InputFormat: org.apache.hadoop.hive.ql.io.RCFileInputFormat
+OutputFormat: org.apache.hadoop.hive.ql.io.RCFileOutputFormat
+Compressed: No
+Num Buckets: -1
+Bucket Columns: []
+Sort Columns: []
+Stored As SubDirectories: Yes
+Skewed Columns: [col2, col4]
+Skewed Values: [[466, val_466], [287, val_287], [82, val_82]]
+#### A masked pattern was here ####
+Skewed Value to Truncated Path: {[82, val_82]=/list_bucketing_mul_col/ds=2008-04-08/hr=11/col2=82/col4=val_82, [466, val_466]=/list_bucketing_mul_col/ds=2008-04-08/hr=11/col2=466/col4=val_466, [287, val_287]=/list_bucketing_mul_col/ds=2008-04-08/hr=11/col2=287/col4=val_287}
+Storage Desc Params:
+ serialization.format 1
+PREHOOK: query: explain extended
+select * from list_bucketing_mul_col
+where ds='2008-04-08' and hr='11' and col2 = "466" and col4 = "val_466"
+PREHOOK: type: QUERY
+POSTHOOK: query: explain extended
+select * from list_bucketing_mul_col
+where ds='2008-04-08' and hr='11' and col2 = "466" and col4 = "val_466"
+POSTHOOK: type: QUERY
+ABSTRACT SYNTAX TREE:
+
+TOK_QUERY
+ TOK_FROM
+ TOK_TABREF
+ TOK_TABNAME
+ list_bucketing_mul_col
+ TOK_INSERT
+ TOK_DESTINATION
+ TOK_DIR
+ TOK_TMP_FILE
+ TOK_SELECT
+ TOK_SELEXPR
+ TOK_ALLCOLREF
+ TOK_WHERE
+ and
+ and
+ and
+ =
+ TOK_TABLE_OR_COL
+ ds
+ '2008-04-08'
+ =
+ TOK_TABLE_OR_COL
+ hr
+ '11'
+ =
+ TOK_TABLE_OR_COL
+ col2
+ "466"
+ =
+ TOK_TABLE_OR_COL
+ col4
+ "val_466"
+
+
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ alias: list_bucketing_mul_col
+ Statistics: Num rows: 500 Data size: 6312 Basic stats: COMPLETE Column stats: NONE
+ GatherStats: false
+ Filter Operator
+ isSamplingPred: false
+ predicate: ((col2 = '466') and (col4 = 'val_466')) (type: boolean)
+ Statistics: Num rows: 125 Data size: 1578 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: col1 (type: string), '466' (type: string), col3 (type: string), 'val_466' (type: string), col5 (type: string), '2008-04-08' (type: string), '11' (type: string)
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6
+ Statistics: Num rows: 125 Data size: 1578 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ GlobalTableId: 0
+#### A masked pattern was here ####
+ NumFilesPerFileSink: 1
+ Statistics: Num rows: 125 Data size: 1578 Basic stats: COMPLETE Column stats: NONE
+#### A masked pattern was here ####
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ properties:
+ columns _col0,_col1,_col2,_col3,_col4,_col5,_col6
+ columns.types string:string:string:string:string:string:string
+ escape.delim \
+ hive.serialization.extend.nesting.levels true
+ serialization.format 1
+ serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ TotalFiles: 1
+ GatherStats: false
+ MultiFileSpray: false
+ Path -> Alias:
+#### A masked pattern was here ####
+ Path -> Partition:
+#### A masked pattern was here ####
+ Partition
+ base file name: col4=val_466
+ input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat
+ partition values:
+ ds 2008-04-08
+ hr 11
+ properties:
+ COLUMN_STATS_ACCURATE true
+ bucket_count -1
+ columns col1,col2,col3,col4,col5
+ columns.comments
+ columns.types string:string:string:string:string
+#### A masked pattern was here ####
+ name default.list_bucketing_mul_col
+ numFiles 4
+ numRows 500
+ partition_columns ds/hr
+ partition_columns.types string:string
+ rawDataSize 6312
+ serialization.ddl struct list_bucketing_mul_col { string col1, string col2, string col3, string col4, string col5}
+ serialization.format 1
+ serialization.lib org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
+ totalSize 7094
+#### A masked pattern was here ####
+ serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
+
+ input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat
+ properties:
+ bucket_count -1
+ columns col1,col2,col3,col4,col5
+ columns.comments
+ columns.types string:string:string:string:string
+#### A masked pattern was here ####
+ name default.list_bucketing_mul_col
+ partition_columns ds/hr
+ partition_columns.types string:string
+ serialization.ddl struct list_bucketing_mul_col { string col1, string col2, string col3, string col4, string col5}
+ serialization.format 1
+ serialization.lib org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
+#### A masked pattern was here ####
+ serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
+ name: default.list_bucketing_mul_col
+ name: default.list_bucketing_mul_col
+ Truncated Path -> Alias:
+ /list_bucketing_mul_col/ds=2008-04-08/hr=11/col2=466/col4=val_466 [list_bucketing_mul_col]
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: select * from list_bucketing_mul_col
+where ds='2008-04-08' and hr='11' and col2 = "466" and col4 = "val_466"
+PREHOOK: type: QUERY
+PREHOOK: Input: default@list_bucketing_mul_col
+PREHOOK: Input: default@list_bucketing_mul_col@ds=2008-04-08/hr=11
+#### A masked pattern was here ####
+POSTHOOK: query: select * from list_bucketing_mul_col
+where ds='2008-04-08' and hr='11' and col2 = "466" and col4 = "val_466"
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@list_bucketing_mul_col
+POSTHOOK: Input: default@list_bucketing_mul_col@ds=2008-04-08/hr=11
+#### A masked pattern was here ####
+1 466 1 val_466 1 2008-04-08 11
+1 466 1 val_466 1 2008-04-08 11
+1 466 1 val_466 1 2008-04-08 11
+PREHOOK: query: explain extended
+select * from list_bucketing_mul_col
+where ds='2008-04-08' and hr='11' and col2 = "382" and col4 = "val_382"
+PREHOOK: type: QUERY
+POSTHOOK: query: explain extended
+select * from list_bucketing_mul_col
+where ds='2008-04-08' and hr='11' and col2 = "382" and col4 = "val_382"
+POSTHOOK: type: QUERY
+ABSTRACT SYNTAX TREE:
+
+TOK_QUERY
+ TOK_FROM
+ TOK_TABREF
+ TOK_TABNAME
+ list_bucketing_mul_col
+ TOK_INSERT
+ TOK_DESTINATION
+ TOK_DIR
+ TOK_TMP_FILE
+ TOK_SELECT
+ TOK_SELEXPR
+ TOK_ALLCOLREF
+ TOK_WHERE
+ and
+ and
+ and
+ =
+ TOK_TABLE_OR_COL
+ ds
+ '2008-04-08'
+ =
+ TOK_TABLE_OR_COL
+ hr
+ '11'
+ =
+ TOK_TABLE_OR_COL
+ col2
+ "382"
+ =
+ TOK_TABLE_OR_COL
+ col4
+ "val_382"
+
+
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ alias: list_bucketing_mul_col
+ Statistics: Num rows: 500 Data size: 6312 Basic stats: COMPLETE Column stats: NONE
+ GatherStats: false
+ Filter Operator
+ isSamplingPred: false
+ predicate: ((col2 = '382') and (col4 = 'val_382')) (type: boolean)
+ Statistics: Num rows: 125 Data size: 1578 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: col1 (type: string), '382' (type: string), col3 (type: string), 'val_382' (type: string), col5 (type: string), '2008-04-08' (type: string), '11' (type: string)
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6
+ Statistics: Num rows: 125 Data size: 1578 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ GlobalTableId: 0
+#### A masked pattern was here ####
+ NumFilesPerFileSink: 1
+ Statistics: Num rows: 125 Data size: 1578 Basic stats: COMPLETE Column stats: NONE
+#### A masked pattern was here ####
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ properties:
+ columns _col0,_col1,_col2,_col3,_col4,_col5,_col6
+ columns.types string:string:string:string:string:string:string
+ escape.delim \
+ hive.serialization.extend.nesting.levels true
+ serialization.format 1
+ serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ TotalFiles: 1
+ GatherStats: false
+ MultiFileSpray: false
+ Path -> Alias:
+#### A masked pattern was here ####
+ Path -> Partition:
+#### A masked pattern was here ####
+ Partition
+ base file name: HIVE_DEFAULT_LIST_BUCKETING_DIR_NAME
+ input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat
+ partition values:
+ ds 2008-04-08
+ hr 11
+ properties:
+ COLUMN_STATS_ACCURATE true
+ bucket_count -1
+ columns col1,col2,col3,col4,col5
+ columns.comments
+ columns.types string:string:string:string:string
+#### A masked pattern was here ####
+ name default.list_bucketing_mul_col
+ numFiles 4
+ numRows 500
+ partition_columns ds/hr
+ partition_columns.types string:string
+ rawDataSize 6312
+ serialization.ddl struct list_bucketing_mul_col { string col1, string col2, string col3, string col4, string col5}
+ serialization.format 1
+ serialization.lib org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
+ totalSize 7094
+#### A masked pattern was here ####
+ serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
+
+ input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat
+ properties:
+ bucket_count -1
+ columns col1,col2,col3,col4,col5
+ columns.comments
+ columns.types string:string:string:string:string
+#### A masked pattern was here ####
+ name default.list_bucketing_mul_col
+ partition_columns ds/hr
+ partition_columns.types string:string
+ serialization.ddl struct list_bucketing_mul_col { string col1, string col2, string col3, string col4, string col5}
+ serialization.format 1
+ serialization.lib org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
+#### A masked pattern was here ####
+ serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
+ name: default.list_bucketing_mul_col
+ name: default.list_bucketing_mul_col
+ Truncated Path -> Alias:
+ /list_bucketing_mul_col/ds=2008-04-08/hr=11/HIVE_DEFAULT_LIST_BUCKETING_DIR_NAME/HIVE_DEFAULT_LIST_BUCKETING_DIR_NAME [list_bucketing_mul_col]
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: select * from list_bucketing_mul_col
+where ds='2008-04-08' and hr='11' and col2 = "382" and col4 = "val_382"
+PREHOOK: type: QUERY
+PREHOOK: Input: default@list_bucketing_mul_col
+PREHOOK: Input: default@list_bucketing_mul_col@ds=2008-04-08/hr=11
+#### A masked pattern was here ####
+POSTHOOK: query: select * from list_bucketing_mul_col
+where ds='2008-04-08' and hr='11' and col2 = "382" and col4 = "val_382"
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@list_bucketing_mul_col
+POSTHOOK: Input: default@list_bucketing_mul_col@ds=2008-04-08/hr=11
+#### A masked pattern was here ####
+1 382 1 val_382 1 2008-04-08 11
+1 382 1 val_382 1 2008-04-08 11
+PREHOOK: query: drop table list_bucketing_mul_col
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@list_bucketing_mul_col
+PREHOOK: Output: default@list_bucketing_mul_col
+POSTHOOK: query: drop table list_bucketing_mul_col
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@list_bucketing_mul_col
+POSTHOOK: Output: default@list_bucketing_mul_col

Search Discussions

  • Brock at Dec 25, 2014 at 6:06 pm
    Added: hive/trunk/ql/src/test/results/clientpositive/list_bucket_dml_12.q.java1.8.out
    URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/list_bucket_dml_12.q.java1.8.out?rev=1647912&view=auto
    ==============================================================================
    --- hive/trunk/ql/src/test/results/clientpositive/list_bucket_dml_12.q.java1.8.out (added)
    +++ hive/trunk/ql/src/test/results/clientpositive/list_bucket_dml_12.q.java1.8.out Thu Dec 25 18:06:30 2014
    @@ -0,0 +1,598 @@
    +PREHOOK: query: -- Ensure it works if skewed column is not the first column in the table columns
    +
    +-- INCLUDE_HADOOP_MAJOR_VERSIONS(0.23)
    +-- SORT_QUERY_RESULTS
    +-- JAVA_VERSION_SPECIFIC_OUTPUT
    +
    +-- test where the skewed values are more than 1 say columns no. 2 and 4 in a table with 5 columns
    +create table list_bucketing_mul_col (col1 String, col2 String, col3 String, col4 String, col5 string)
    + partitioned by (ds String, hr String)
    + skewed by (col2, col4) on (('466','val_466'),('287','val_287'),('82','val_82'))
    + stored as DIRECTORIES
    + STORED AS RCFILE
    +PREHOOK: type: CREATETABLE
    +PREHOOK: Output: database:default
    +PREHOOK: Output: default@list_bucketing_mul_col
    +POSTHOOK: query: -- Ensure it works if skewed column is not the first column in the table columns
    +
    +-- INCLUDE_HADOOP_MAJOR_VERSIONS(0.23)
    +-- SORT_QUERY_RESULTS
    +-- JAVA_VERSION_SPECIFIC_OUTPUT
    +
    +-- test where the skewed values are more than 1 say columns no. 2 and 4 in a table with 5 columns
    +create table list_bucketing_mul_col (col1 String, col2 String, col3 String, col4 String, col5 string)
    + partitioned by (ds String, hr String)
    + skewed by (col2, col4) on (('466','val_466'),('287','val_287'),('82','val_82'))
    + stored as DIRECTORIES
    + STORED AS RCFILE
    +POSTHOOK: type: CREATETABLE
    +POSTHOOK: Output: database:default
    +POSTHOOK: Output: default@list_bucketing_mul_col
    +PREHOOK: query: -- list bucketing DML
    +explain extended
    +insert overwrite table list_bucketing_mul_col partition (ds = '2008-04-08', hr = '11')
    +select 1, key, 1, value, 1 from src
    +PREHOOK: type: QUERY
    +POSTHOOK: query: -- list bucketing DML
    +explain extended
    +insert overwrite table list_bucketing_mul_col partition (ds = '2008-04-08', hr = '11')
    +select 1, key, 1, value, 1 from src
    +POSTHOOK: type: QUERY
    +ABSTRACT SYNTAX TREE:
    +
    +TOK_QUERY
    + TOK_FROM
    + TOK_TABREF
    + TOK_TABNAME
    + src
    + TOK_INSERT
    + TOK_DESTINATION
    + TOK_TAB
    + TOK_TABNAME
    + list_bucketing_mul_col
    + TOK_PARTSPEC
    + TOK_PARTVAL
    + ds
    + '2008-04-08'
    + TOK_PARTVAL
    + hr
    + '11'
    + TOK_SELECT
    + TOK_SELEXPR
    + 1
    + TOK_SELEXPR
    + TOK_TABLE_OR_COL
    + key
    + TOK_SELEXPR
    + 1
    + TOK_SELEXPR
    + TOK_TABLE_OR_COL
    + value
    + TOK_SELEXPR
    + 1
    +
    +
    +STAGE DEPENDENCIES:
    + Stage-1 is a root stage
    + Stage-0 depends on stages: Stage-1
    + Stage-2 depends on stages: Stage-0
    +
    +STAGE PLANS:
    + Stage: Stage-1
    + Map Reduce
    + Map Operator Tree:
    + TableScan
    + alias: src
    + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
    + GatherStats: false
    + Select Operator
    + expressions: UDFToString(1) (type: string), key (type: string), UDFToString(1) (type: string), value (type: string), UDFToString(1) (type: string)
    + outputColumnNames: _col0, _col1, _col2, _col3, _col4
    + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
    + File Output Operator
    + compressed: false
    + GlobalTableId: 1
    +#### A masked pattern was here ####
    + NumFilesPerFileSink: 1
    + Static Partition Specification: ds=2008-04-08/hr=11/
    + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
    +#### A masked pattern was here ####
    + table:
    + input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat
    + output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat
    + properties:
    + bucket_count -1
    + columns col1,col2,col3,col4,col5
    + columns.comments
    + columns.types string:string:string:string:string
    +#### A masked pattern was here ####
    + name default.list_bucketing_mul_col
    + partition_columns ds/hr
    + partition_columns.types string:string
    + serialization.ddl struct list_bucketing_mul_col { string col1, string col2, string col3, string col4, string col5}
    + serialization.format 1
    + serialization.lib org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
    +#### A masked pattern was here ####
    + serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
    + name: default.list_bucketing_mul_col
    + TotalFiles: 1
    + GatherStats: true
    + MultiFileSpray: false
    + Path -> Alias:
    +#### A masked pattern was here ####
    + Path -> Partition:
    +#### A masked pattern was here ####
    + Partition
    + base file name: src
    + input format: org.apache.hadoop.mapred.TextInputFormat
    + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
    + properties:
    + COLUMN_STATS_ACCURATE true
    + bucket_count -1
    + columns key,value
    + columns.comments 'default','default'
    + columns.types string:string
    +#### A masked pattern was here ####
    + name default.src
    + numFiles 1
    + numRows 500
    + rawDataSize 5312
    + serialization.ddl struct src { string key, string value}
    + serialization.format 1
    + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
    + totalSize 5812
    +#### A masked pattern was here ####
    + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
    +
    + input format: org.apache.hadoop.mapred.TextInputFormat
    + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
    + properties:
    + COLUMN_STATS_ACCURATE true
    + bucket_count -1
    + columns key,value
    + columns.comments 'default','default'
    + columns.types string:string
    +#### A masked pattern was here ####
    + name default.src
    + numFiles 1
    + numRows 500
    + rawDataSize 5312
    + serialization.ddl struct src { string key, string value}
    + serialization.format 1
    + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
    + totalSize 5812
    +#### A masked pattern was here ####
    + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
    + name: default.src
    + name: default.src
    + Truncated Path -> Alias:
    + /src [$hdt$_0:src]
    +
    + Stage: Stage-0
    + Move Operator
    + tables:
    + partition:
    + ds 2008-04-08
    + hr 11
    + replace: true
    +#### A masked pattern was here ####
    + table:
    + input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat
    + output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat
    + properties:
    + bucket_count -1
    + columns col1,col2,col3,col4,col5
    + columns.comments
    + columns.types string:string:string:string:string
    +#### A masked pattern was here ####
    + name default.list_bucketing_mul_col
    + partition_columns ds/hr
    + partition_columns.types string:string
    + serialization.ddl struct list_bucketing_mul_col { string col1, string col2, string col3, string col4, string col5}
    + serialization.format 1
    + serialization.lib org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
    +#### A masked pattern was here ####
    + serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
    + name: default.list_bucketing_mul_col
    +
    + Stage: Stage-2
    + Stats-Aggr Operator
    +#### A masked pattern was here ####
    +
    +PREHOOK: query: insert overwrite table list_bucketing_mul_col partition (ds = '2008-04-08', hr = '11')
    +select 1, key, 1, value, 1 from src
    +PREHOOK: type: QUERY
    +PREHOOK: Input: default@src
    +PREHOOK: Output: default@list_bucketing_mul_col@ds=2008-04-08/hr=11
    +POSTHOOK: query: insert overwrite table list_bucketing_mul_col partition (ds = '2008-04-08', hr = '11')
    +select 1, key, 1, value, 1 from src
    +POSTHOOK: type: QUERY
    +POSTHOOK: Input: default@src
    +POSTHOOK: Output: default@list_bucketing_mul_col@ds=2008-04-08/hr=11
    +POSTHOOK: Lineage: list_bucketing_mul_col PARTITION(ds=2008-04-08,hr=11).col1 EXPRESSION []
    +POSTHOOK: Lineage: list_bucketing_mul_col PARTITION(ds=2008-04-08,hr=11).col2 SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ]
    +POSTHOOK: Lineage: list_bucketing_mul_col PARTITION(ds=2008-04-08,hr=11).col3 EXPRESSION []
    +POSTHOOK: Lineage: list_bucketing_mul_col PARTITION(ds=2008-04-08,hr=11).col4 SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
    +POSTHOOK: Lineage: list_bucketing_mul_col PARTITION(ds=2008-04-08,hr=11).col5 EXPRESSION []
    +PREHOOK: query: -- check DML result
    +show partitions list_bucketing_mul_col
    +PREHOOK: type: SHOWPARTITIONS
    +PREHOOK: Input: default@list_bucketing_mul_col
    +POSTHOOK: query: -- check DML result
    +show partitions list_bucketing_mul_col
    +POSTHOOK: type: SHOWPARTITIONS
    +POSTHOOK: Input: default@list_bucketing_mul_col
    +ds=2008-04-08/hr=11
    +PREHOOK: query: desc formatted list_bucketing_mul_col partition (ds='2008-04-08', hr='11')
    +PREHOOK: type: DESCTABLE
    +PREHOOK: Input: default@list_bucketing_mul_col
    +POSTHOOK: query: desc formatted list_bucketing_mul_col partition (ds='2008-04-08', hr='11')
    +POSTHOOK: type: DESCTABLE
    +POSTHOOK: Input: default@list_bucketing_mul_col
    +# col_name data_type comment
    +
    +col1 string
    +col2 string
    +col3 string
    +col4 string
    +col5 string
    +
    +# Partition Information
    +# col_name data_type comment
    +
    +ds string
    +hr string
    +
    +# Detailed Partition Information
    +Partition Value: [2008-04-08, 11]
    +Database: default
    +Table: list_bucketing_mul_col
    +#### A masked pattern was here ####
    +Protect Mode: None
    +#### A masked pattern was here ####
    +Partition Parameters:
    + COLUMN_STATS_ACCURATE true
    + numFiles 4
    + numRows 500
    + rawDataSize 6312
    + totalSize 7094
    +#### A masked pattern was here ####
    +
    +# Storage Information
    +SerDe Library: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
    +InputFormat: org.apache.hadoop.hive.ql.io.RCFileInputFormat
    +OutputFormat: org.apache.hadoop.hive.ql.io.RCFileOutputFormat
    +Compressed: No
    +Num Buckets: -1
    +Bucket Columns: []
    +Sort Columns: []
    +Stored As SubDirectories: Yes
    +Skewed Columns: [col2, col4]
    +Skewed Values: [[466, val_466], [287, val_287], [82, val_82]]
    +#### A masked pattern was here ####
    +Skewed Value to Truncated Path: {[466, val_466]=/list_bucketing_mul_col/ds=2008-04-08/hr=11/col2=466/col4=val_466, [82, val_82]=/list_bucketing_mul_col/ds=2008-04-08/hr=11/col2=82/col4=val_82, [287, val_287]=/list_bucketing_mul_col/ds=2008-04-08/hr=11/col2=287/col4=val_287}
    +Storage Desc Params:
    + serialization.format 1
    +PREHOOK: query: explain extended
    +select * from list_bucketing_mul_col
    +where ds='2008-04-08' and hr='11' and col2 = "466" and col4 = "val_466"
    +PREHOOK: type: QUERY
    +POSTHOOK: query: explain extended
    +select * from list_bucketing_mul_col
    +where ds='2008-04-08' and hr='11' and col2 = "466" and col4 = "val_466"
    +POSTHOOK: type: QUERY
    +ABSTRACT SYNTAX TREE:
    +
    +TOK_QUERY
    + TOK_FROM
    + TOK_TABREF
    + TOK_TABNAME
    + list_bucketing_mul_col
    + TOK_INSERT
    + TOK_DESTINATION
    + TOK_DIR
    + TOK_TMP_FILE
    + TOK_SELECT
    + TOK_SELEXPR
    + TOK_ALLCOLREF
    + TOK_WHERE
    + and
    + and
    + and
    + =
    + TOK_TABLE_OR_COL
    + ds
    + '2008-04-08'
    + =
    + TOK_TABLE_OR_COL
    + hr
    + '11'
    + =
    + TOK_TABLE_OR_COL
    + col2
    + "466"
    + =
    + TOK_TABLE_OR_COL
    + col4
    + "val_466"
    +
    +
    +STAGE DEPENDENCIES:
    + Stage-1 is a root stage
    + Stage-0 depends on stages: Stage-1
    +
    +STAGE PLANS:
    + Stage: Stage-1
    + Map Reduce
    + Map Operator Tree:
    + TableScan
    + alias: list_bucketing_mul_col
    + Statistics: Num rows: 500 Data size: 6312 Basic stats: COMPLETE Column stats: NONE
    + GatherStats: false
    + Filter Operator
    + isSamplingPred: false
    + predicate: ((col2 = '466') and (col4 = 'val_466')) (type: boolean)
    + Statistics: Num rows: 125 Data size: 1578 Basic stats: COMPLETE Column stats: NONE
    + Select Operator
    + expressions: col1 (type: string), '466' (type: string), col3 (type: string), 'val_466' (type: string), col5 (type: string), '2008-04-08' (type: string), '11' (type: string)
    + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6
    + Statistics: Num rows: 125 Data size: 1578 Basic stats: COMPLETE Column stats: NONE
    + File Output Operator
    + compressed: false
    + GlobalTableId: 0
    +#### A masked pattern was here ####
    + NumFilesPerFileSink: 1
    + Statistics: Num rows: 125 Data size: 1578 Basic stats: COMPLETE Column stats: NONE
    +#### A masked pattern was here ####
    + table:
    + input format: org.apache.hadoop.mapred.TextInputFormat
    + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
    + properties:
    + columns _col0,_col1,_col2,_col3,_col4,_col5,_col6
    + columns.types string:string:string:string:string:string:string
    + escape.delim \
    + hive.serialization.extend.nesting.levels true
    + serialization.format 1
    + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
    + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
    + TotalFiles: 1
    + GatherStats: false
    + MultiFileSpray: false
    + Path -> Alias:
    +#### A masked pattern was here ####
    + Path -> Partition:
    +#### A masked pattern was here ####
    + Partition
    + base file name: col4=val_466
    + input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat
    + output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat
    + partition values:
    + ds 2008-04-08
    + hr 11
    + properties:
    + COLUMN_STATS_ACCURATE true
    + bucket_count -1
    + columns col1,col2,col3,col4,col5
    + columns.comments
    + columns.types string:string:string:string:string
    +#### A masked pattern was here ####
    + name default.list_bucketing_mul_col
    + numFiles 4
    + numRows 500
    + partition_columns ds/hr
    + partition_columns.types string:string
    + rawDataSize 6312
    + serialization.ddl struct list_bucketing_mul_col { string col1, string col2, string col3, string col4, string col5}
    + serialization.format 1
    + serialization.lib org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
    + totalSize 7094
    +#### A masked pattern was here ####
    + serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
    +
    + input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat
    + output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat
    + properties:
    + bucket_count -1
    + columns col1,col2,col3,col4,col5
    + columns.comments
    + columns.types string:string:string:string:string
    +#### A masked pattern was here ####
    + name default.list_bucketing_mul_col
    + partition_columns ds/hr
    + partition_columns.types string:string
    + serialization.ddl struct list_bucketing_mul_col { string col1, string col2, string col3, string col4, string col5}
    + serialization.format 1
    + serialization.lib org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
    +#### A masked pattern was here ####
    + serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
    + name: default.list_bucketing_mul_col
    + name: default.list_bucketing_mul_col
    + Truncated Path -> Alias:
    + /list_bucketing_mul_col/ds=2008-04-08/hr=11/col2=466/col4=val_466 [list_bucketing_mul_col]
    +
    + Stage: Stage-0
    + Fetch Operator
    + limit: -1
    + Processor Tree:
    + ListSink
    +
    +PREHOOK: query: select * from list_bucketing_mul_col
    +where ds='2008-04-08' and hr='11' and col2 = "466" and col4 = "val_466"
    +PREHOOK: type: QUERY
    +PREHOOK: Input: default@list_bucketing_mul_col
    +PREHOOK: Input: default@list_bucketing_mul_col@ds=2008-04-08/hr=11
    +#### A masked pattern was here ####
    +POSTHOOK: query: select * from list_bucketing_mul_col
    +where ds='2008-04-08' and hr='11' and col2 = "466" and col4 = "val_466"
    +POSTHOOK: type: QUERY
    +POSTHOOK: Input: default@list_bucketing_mul_col
    +POSTHOOK: Input: default@list_bucketing_mul_col@ds=2008-04-08/hr=11
    +#### A masked pattern was here ####
    +1 466 1 val_466 1 2008-04-08 11
    +1 466 1 val_466 1 2008-04-08 11
    +1 466 1 val_466 1 2008-04-08 11
    +PREHOOK: query: explain extended
    +select * from list_bucketing_mul_col
    +where ds='2008-04-08' and hr='11' and col2 = "382" and col4 = "val_382"
    +PREHOOK: type: QUERY
    +POSTHOOK: query: explain extended
    +select * from list_bucketing_mul_col
    +where ds='2008-04-08' and hr='11' and col2 = "382" and col4 = "val_382"
    +POSTHOOK: type: QUERY
    +ABSTRACT SYNTAX TREE:
    +
    +TOK_QUERY
    + TOK_FROM
    + TOK_TABREF
    + TOK_TABNAME
    + list_bucketing_mul_col
    + TOK_INSERT
    + TOK_DESTINATION
    + TOK_DIR
    + TOK_TMP_FILE
    + TOK_SELECT
    + TOK_SELEXPR
    + TOK_ALLCOLREF
    + TOK_WHERE
    + and
    + and
    + and
    + =
    + TOK_TABLE_OR_COL
    + ds
    + '2008-04-08'
    + =
    + TOK_TABLE_OR_COL
    + hr
    + '11'
    + =
    + TOK_TABLE_OR_COL
    + col2
    + "382"
    + =
    + TOK_TABLE_OR_COL
    + col4
    + "val_382"
    +
    +
    +STAGE DEPENDENCIES:
    + Stage-1 is a root stage
    + Stage-0 depends on stages: Stage-1
    +
    +STAGE PLANS:
    + Stage: Stage-1
    + Map Reduce
    + Map Operator Tree:
    + TableScan
    + alias: list_bucketing_mul_col
    + Statistics: Num rows: 500 Data size: 6312 Basic stats: COMPLETE Column stats: NONE
    + GatherStats: false
    + Filter Operator
    + isSamplingPred: false
    + predicate: ((col2 = '382') and (col4 = 'val_382')) (type: boolean)
    + Statistics: Num rows: 125 Data size: 1578 Basic stats: COMPLETE Column stats: NONE
    + Select Operator
    + expressions: col1 (type: string), '382' (type: string), col3 (type: string), 'val_382' (type: string), col5 (type: string), '2008-04-08' (type: string), '11' (type: string)
    + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6
    + Statistics: Num rows: 125 Data size: 1578 Basic stats: COMPLETE Column stats: NONE
    + File Output Operator
    + compressed: false
    + GlobalTableId: 0
    +#### A masked pattern was here ####
    + NumFilesPerFileSink: 1
    + Statistics: Num rows: 125 Data size: 1578 Basic stats: COMPLETE Column stats: NONE
    +#### A masked pattern was here ####
    + table:
    + input format: org.apache.hadoop.mapred.TextInputFormat
    + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
    + properties:
    + columns _col0,_col1,_col2,_col3,_col4,_col5,_col6
    + columns.types string:string:string:string:string:string:string
    + escape.delim \
    + hive.serialization.extend.nesting.levels true
    + serialization.format 1
    + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
    + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
    + TotalFiles: 1
    + GatherStats: false
    + MultiFileSpray: false
    + Path -> Alias:
    +#### A masked pattern was here ####
    + Path -> Partition:
    +#### A masked pattern was here ####
    + Partition
    + base file name: HIVE_DEFAULT_LIST_BUCKETING_DIR_NAME
    + input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat
    + output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat
    + partition values:
    + ds 2008-04-08
    + hr 11
    + properties:
    + COLUMN_STATS_ACCURATE true
    + bucket_count -1
    + columns col1,col2,col3,col4,col5
    + columns.comments
    + columns.types string:string:string:string:string
    +#### A masked pattern was here ####
    + name default.list_bucketing_mul_col
    + numFiles 4
    + numRows 500
    + partition_columns ds/hr
    + partition_columns.types string:string
    + rawDataSize 6312
    + serialization.ddl struct list_bucketing_mul_col { string col1, string col2, string col3, string col4, string col5}
    + serialization.format 1
    + serialization.lib org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
    + totalSize 7094
    +#### A masked pattern was here ####
    + serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
    +
    + input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat
    + output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat
    + properties:
    + bucket_count -1
    + columns col1,col2,col3,col4,col5
    + columns.comments
    + columns.types string:string:string:string:string
    +#### A masked pattern was here ####
    + name default.list_bucketing_mul_col
    + partition_columns ds/hr
    + partition_columns.types string:string
    + serialization.ddl struct list_bucketing_mul_col { string col1, string col2, string col3, string col4, string col5}
    + serialization.format 1
    + serialization.lib org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
    +#### A masked pattern was here ####
    + serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
    + name: default.list_bucketing_mul_col
    + name: default.list_bucketing_mul_col
    + Truncated Path -> Alias:
    + /list_bucketing_mul_col/ds=2008-04-08/hr=11/HIVE_DEFAULT_LIST_BUCKETING_DIR_NAME/HIVE_DEFAULT_LIST_BUCKETING_DIR_NAME [list_bucketing_mul_col]
    +
    + Stage: Stage-0
    + Fetch Operator
    + limit: -1
    + Processor Tree:
    + ListSink
    +
    +PREHOOK: query: select * from list_bucketing_mul_col
    +where ds='2008-04-08' and hr='11' and col2 = "382" and col4 = "val_382"
    +PREHOOK: type: QUERY
    +PREHOOK: Input: default@list_bucketing_mul_col
    +PREHOOK: Input: default@list_bucketing_mul_col@ds=2008-04-08/hr=11
    +#### A masked pattern was here ####
    +POSTHOOK: query: select * from list_bucketing_mul_col
    +where ds='2008-04-08' and hr='11' and col2 = "382" and col4 = "val_382"
    +POSTHOOK: type: QUERY
    +POSTHOOK: Input: default@list_bucketing_mul_col
    +POSTHOOK: Input: default@list_bucketing_mul_col@ds=2008-04-08/hr=11
    +#### A masked pattern was here ####
    +1 382 1 val_382 1 2008-04-08 11
    +1 382 1 val_382 1 2008-04-08 11
    +PREHOOK: query: drop table list_bucketing_mul_col
    +PREHOOK: type: DROPTABLE
    +PREHOOK: Input: default@list_bucketing_mul_col
    +PREHOOK: Output: default@list_bucketing_mul_col
    +POSTHOOK: query: drop table list_bucketing_mul_col
    +POSTHOOK: type: DROPTABLE
    +POSTHOOK: Input: default@list_bucketing_mul_col
    +POSTHOOK: Output: default@list_bucketing_mul_col

    Added: hive/trunk/ql/src/test/results/clientpositive/list_bucket_dml_13.q.java1.7.out
    URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/list_bucket_dml_13.q.java1.7.out?rev=1647912&view=auto
    ==============================================================================
    --- hive/trunk/ql/src/test/results/clientpositive/list_bucket_dml_13.q.java1.7.out (added)
    +++ hive/trunk/ql/src/test/results/clientpositive/list_bucket_dml_13.q.java1.7.out Thu Dec 25 18:06:30 2014
    @@ -0,0 +1,441 @@
    +PREHOOK: query: -- Ensure skewed value map has escaped directory name
    +
    +-- INCLUDE_HADOOP_MAJOR_VERSIONS(0.23)
    +-- SORT_QUERY_RESULTS
    +-- JAVA_VERSION_SPECIFIC_OUTPUT
    +
    +-- test where the skewed values are more than 1 say columns no. 2 and 4 in a table with 5 columns
    +create table list_bucketing_mul_col (col1 String, col2 String, col3 String, col4 String, col5 string)
    + partitioned by (ds String, hr String)
    + skewed by (col2, col4) on (('466','val_466'),('287','val_287'),('82','val_82'))
    + stored as DIRECTORIES
    + STORED AS RCFILE
    +PREHOOK: type: CREATETABLE
    +PREHOOK: Output: database:default
    +PREHOOK: Output: default@list_bucketing_mul_col
    +POSTHOOK: query: -- Ensure skewed value map has escaped directory name
    +
    +-- INCLUDE_HADOOP_MAJOR_VERSIONS(0.23)
    +-- SORT_QUERY_RESULTS
    +-- JAVA_VERSION_SPECIFIC_OUTPUT
    +
    +-- test where the skewed values are more than 1 say columns no. 2 and 4 in a table with 5 columns
    +create table list_bucketing_mul_col (col1 String, col2 String, col3 String, col4 String, col5 string)
    + partitioned by (ds String, hr String)
    + skewed by (col2, col4) on (('466','val_466'),('287','val_287'),('82','val_82'))
    + stored as DIRECTORIES
    + STORED AS RCFILE
    +POSTHOOK: type: CREATETABLE
    +POSTHOOK: Output: database:default
    +POSTHOOK: Output: default@list_bucketing_mul_col
    +PREHOOK: query: -- list bucketing DML
    +explain extended
    +insert overwrite table list_bucketing_mul_col partition (ds = '2008-04-08', hr = '2013-01-23+18:00:99')
    +select 1, key, 1, value, 1 from src
    +PREHOOK: type: QUERY
    +POSTHOOK: query: -- list bucketing DML
    +explain extended
    +insert overwrite table list_bucketing_mul_col partition (ds = '2008-04-08', hr = '2013-01-23+18:00:99')
    +select 1, key, 1, value, 1 from src
    +POSTHOOK: type: QUERY
    +ABSTRACT SYNTAX TREE:
    +
    +TOK_QUERY
    + TOK_FROM
    + TOK_TABREF
    + TOK_TABNAME
    + src
    + TOK_INSERT
    + TOK_DESTINATION
    + TOK_TAB
    + TOK_TABNAME
    + list_bucketing_mul_col
    + TOK_PARTSPEC
    + TOK_PARTVAL
    + ds
    + '2008-04-08'
    + TOK_PARTVAL
    + hr
    + '2013-01-23+18:00:99'
    + TOK_SELECT
    + TOK_SELEXPR
    + 1
    + TOK_SELEXPR
    + TOK_TABLE_OR_COL
    + key
    + TOK_SELEXPR
    + 1
    + TOK_SELEXPR
    + TOK_TABLE_OR_COL
    + value
    + TOK_SELEXPR
    + 1
    +
    +
    +STAGE DEPENDENCIES:
    + Stage-1 is a root stage
    + Stage-0 depends on stages: Stage-1
    + Stage-2 depends on stages: Stage-0
    +
    +STAGE PLANS:
    + Stage: Stage-1
    + Map Reduce
    + Map Operator Tree:
    + TableScan
    + alias: src
    + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
    + GatherStats: false
    + Select Operator
    + expressions: UDFToString(1) (type: string), key (type: string), UDFToString(1) (type: string), value (type: string), UDFToString(1) (type: string)
    + outputColumnNames: _col0, _col1, _col2, _col3, _col4
    + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
    + File Output Operator
    + compressed: false
    + GlobalTableId: 1
    +#### A masked pattern was here ####
    + NumFilesPerFileSink: 1
    + Static Partition Specification: ds=2008-04-08/hr=2013-01-23+18%3A00%3A99/
    + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
    +#### A masked pattern was here ####
    + table:
    + input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat
    + output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat
    + properties:
    + bucket_count -1
    + columns col1,col2,col3,col4,col5
    + columns.comments
    + columns.types string:string:string:string:string
    +#### A masked pattern was here ####
    + name default.list_bucketing_mul_col
    + partition_columns ds/hr
    + partition_columns.types string:string
    + serialization.ddl struct list_bucketing_mul_col { string col1, string col2, string col3, string col4, string col5}
    + serialization.format 1
    + serialization.lib org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
    +#### A masked pattern was here ####
    + serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
    + name: default.list_bucketing_mul_col
    + TotalFiles: 1
    + GatherStats: true
    + MultiFileSpray: false
    + Path -> Alias:
    +#### A masked pattern was here ####
    + Path -> Partition:
    +#### A masked pattern was here ####
    + Partition
    + base file name: src
    + input format: org.apache.hadoop.mapred.TextInputFormat
    + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
    + properties:
    + COLUMN_STATS_ACCURATE true
    + bucket_count -1
    + columns key,value
    + columns.comments 'default','default'
    + columns.types string:string
    +#### A masked pattern was here ####
    + name default.src
    + numFiles 1
    + numRows 500
    + rawDataSize 5312
    + serialization.ddl struct src { string key, string value}
    + serialization.format 1
    + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
    + totalSize 5812
    +#### A masked pattern was here ####
    + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
    +
    + input format: org.apache.hadoop.mapred.TextInputFormat
    + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
    + properties:
    + COLUMN_STATS_ACCURATE true
    + bucket_count -1
    + columns key,value
    + columns.comments 'default','default'
    + columns.types string:string
    +#### A masked pattern was here ####
    + name default.src
    + numFiles 1
    + numRows 500
    + rawDataSize 5312
    + serialization.ddl struct src { string key, string value}
    + serialization.format 1
    + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
    + totalSize 5812
    +#### A masked pattern was here ####
    + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
    + name: default.src
    + name: default.src
    + Truncated Path -> Alias:
    + /src [$hdt$_0:src]
    +
    + Stage: Stage-0
    + Move Operator
    + tables:
    + partition:
    + ds 2008-04-08
    + hr 2013-01-23+18:00:99
    + replace: true
    +#### A masked pattern was here ####
    + table:
    + input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat
    + output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat
    + properties:
    + bucket_count -1
    + columns col1,col2,col3,col4,col5
    + columns.comments
    + columns.types string:string:string:string:string
    +#### A masked pattern was here ####
    + name default.list_bucketing_mul_col
    + partition_columns ds/hr
    + partition_columns.types string:string
    + serialization.ddl struct list_bucketing_mul_col { string col1, string col2, string col3, string col4, string col5}
    + serialization.format 1
    + serialization.lib org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
    +#### A masked pattern was here ####
    + serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
    + name: default.list_bucketing_mul_col
    +
    + Stage: Stage-2
    + Stats-Aggr Operator
    +#### A masked pattern was here ####
    +
    +PREHOOK: query: insert overwrite table list_bucketing_mul_col partition (ds = '2008-04-08', hr = '2013-01-23+18:00:99')
    +select 1, key, 1, value, 1 from src
    +PREHOOK: type: QUERY
    +PREHOOK: Input: default@src
    +PREHOOK: Output: default@list_bucketing_mul_col@ds=2008-04-08/hr=2013-01-23+18%3A00%3A99
    +POSTHOOK: query: insert overwrite table list_bucketing_mul_col partition (ds = '2008-04-08', hr = '2013-01-23+18:00:99')
    +select 1, key, 1, value, 1 from src
    +POSTHOOK: type: QUERY
    +POSTHOOK: Input: default@src
    +POSTHOOK: Output: default@list_bucketing_mul_col@ds=2008-04-08/hr=2013-01-23+18%3A00%3A99
    +POSTHOOK: Lineage: list_bucketing_mul_col PARTITION(ds=2008-04-08,hr=2013-01-23+18:00:99).col1 EXPRESSION []
    +POSTHOOK: Lineage: list_bucketing_mul_col PARTITION(ds=2008-04-08,hr=2013-01-23+18:00:99).col2 SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ]
    +POSTHOOK: Lineage: list_bucketing_mul_col PARTITION(ds=2008-04-08,hr=2013-01-23+18:00:99).col3 EXPRESSION []
    +POSTHOOK: Lineage: list_bucketing_mul_col PARTITION(ds=2008-04-08,hr=2013-01-23+18:00:99).col4 SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
    +POSTHOOK: Lineage: list_bucketing_mul_col PARTITION(ds=2008-04-08,hr=2013-01-23+18:00:99).col5 EXPRESSION []
    +PREHOOK: query: -- check DML result
    +show partitions list_bucketing_mul_col
    +PREHOOK: type: SHOWPARTITIONS
    +PREHOOK: Input: default@list_bucketing_mul_col
    +POSTHOOK: query: -- check DML result
    +show partitions list_bucketing_mul_col
    +POSTHOOK: type: SHOWPARTITIONS
    +POSTHOOK: Input: default@list_bucketing_mul_col
    +ds=2008-04-08/hr=2013-01-23+18%3A00%3A99
    +PREHOOK: query: desc formatted list_bucketing_mul_col partition (ds='2008-04-08', hr='2013-01-23+18:00:99')
    +PREHOOK: type: DESCTABLE
    +PREHOOK: Input: default@list_bucketing_mul_col
    +POSTHOOK: query: desc formatted list_bucketing_mul_col partition (ds='2008-04-08', hr='2013-01-23+18:00:99')
    +POSTHOOK: type: DESCTABLE
    +POSTHOOK: Input: default@list_bucketing_mul_col
    +# col_name data_type comment
    +
    +col1 string
    +col2 string
    +col3 string
    +col4 string
    +col5 string
    +
    +# Partition Information
    +# col_name data_type comment
    +
    +ds string
    +hr string
    +
    +# Detailed Partition Information
    +Partition Value: [2008-04-08, 2013-01-23+18:00:99]
    +Database: default
    +Table: list_bucketing_mul_col
    +#### A masked pattern was here ####
    +Protect Mode: None
    +#### A masked pattern was here ####
    +Partition Parameters:
    + COLUMN_STATS_ACCURATE true
    + numFiles 4
    + numRows 500
    + rawDataSize 6312
    + totalSize 7094
    +#### A masked pattern was here ####
    +
    +# Storage Information
    +SerDe Library: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
    +InputFormat: org.apache.hadoop.hive.ql.io.RCFileInputFormat
    +OutputFormat: org.apache.hadoop.hive.ql.io.RCFileOutputFormat
    +Compressed: No
    +Num Buckets: -1
    +Bucket Columns: []
    +Sort Columns: []
    +Stored As SubDirectories: Yes
    +Skewed Columns: [col2, col4]
    +Skewed Values: [[466, val_466], [287, val_287], [82, val_82]]
    +#### A masked pattern was here ####
    +Skewed Value to Truncated Path: {[82, val_82]=/list_bucketing_mul_col/ds=2008-04-08/hr=2013-01-23+18%3A00%3A99/col2=82/col4=val_82, [466, val_466]=/list_bucketing_mul_col/ds=2008-04-08/hr=2013-01-23+18%3A00%3A99/col2=466/col4=val_466, [287, val_287]=/list_bucketing_mul_col/ds=2008-04-08/hr=2013-01-23+18%3A00%3A99/col2=287/col4=val_287}
    +Storage Desc Params:
    + serialization.format 1
    +PREHOOK: query: explain extended
    +select * from list_bucketing_mul_col
    +where ds='2008-04-08' and hr='2013-01-23+18:00:99' and col2 = "466" and col4 = "val_466"
    +PREHOOK: type: QUERY
    +POSTHOOK: query: explain extended
    +select * from list_bucketing_mul_col
    +where ds='2008-04-08' and hr='2013-01-23+18:00:99' and col2 = "466" and col4 = "val_466"
    +POSTHOOK: type: QUERY
    +ABSTRACT SYNTAX TREE:
    +
    +TOK_QUERY
    + TOK_FROM
    + TOK_TABREF
    + TOK_TABNAME
    + list_bucketing_mul_col
    + TOK_INSERT
    + TOK_DESTINATION
    + TOK_DIR
    + TOK_TMP_FILE
    + TOK_SELECT
    + TOK_SELEXPR
    + TOK_ALLCOLREF
    + TOK_WHERE
    + and
    + and
    + and
    + =
    + TOK_TABLE_OR_COL
    + ds
    + '2008-04-08'
    + =
    + TOK_TABLE_OR_COL
    + hr
    + '2013-01-23+18:00:99'
    + =
    + TOK_TABLE_OR_COL
    + col2
    + "466"
    + =
    + TOK_TABLE_OR_COL
    + col4
    + "val_466"
    +
    +
    +STAGE DEPENDENCIES:
    + Stage-1 is a root stage
    + Stage-0 depends on stages: Stage-1
    +
    +STAGE PLANS:
    + Stage: Stage-1
    + Map Reduce
    + Map Operator Tree:
    + TableScan
    + alias: list_bucketing_mul_col
    + Statistics: Num rows: 500 Data size: 6312 Basic stats: COMPLETE Column stats: NONE
    + GatherStats: false
    + Filter Operator
    + isSamplingPred: false
    + predicate: ((col2 = '466') and (col4 = 'val_466')) (type: boolean)
    + Statistics: Num rows: 125 Data size: 1578 Basic stats: COMPLETE Column stats: NONE
    + Select Operator
    + expressions: col1 (type: string), '466' (type: string), col3 (type: string), 'val_466' (type: string), col5 (type: string), '2008-04-08' (type: string), '2013-01-23+18:00:99' (type: string)
    + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6
    + Statistics: Num rows: 125 Data size: 1578 Basic stats: COMPLETE Column stats: NONE
    + File Output Operator
    + compressed: false
    + GlobalTableId: 0
    +#### A masked pattern was here ####
    + NumFilesPerFileSink: 1
    + Statistics: Num rows: 125 Data size: 1578 Basic stats: COMPLETE Column stats: NONE
    +#### A masked pattern was here ####
    + table:
    + input format: org.apache.hadoop.mapred.TextInputFormat
    + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
    + properties:
    + columns _col0,_col1,_col2,_col3,_col4,_col5,_col6
    + columns.types string:string:string:string:string:string:string
    + escape.delim \
    + hive.serialization.extend.nesting.levels true
    + serialization.format 1
    + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
    + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
    + TotalFiles: 1
    + GatherStats: false
    + MultiFileSpray: false
    + Path -> Alias:
    +#### A masked pattern was here ####
    + Path -> Partition:
    +#### A masked pattern was here ####
    + Partition
    + base file name: col4=val_466
    + input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat
    + output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat
    + partition values:
    + ds 2008-04-08
    + hr 2013-01-23+18:00:99
    + properties:
    + COLUMN_STATS_ACCURATE true
    + bucket_count -1
    + columns col1,col2,col3,col4,col5
    + columns.comments
    + columns.types string:string:string:string:string
    +#### A masked pattern was here ####
    + name default.list_bucketing_mul_col
    + numFiles 4
    + numRows 500
    + partition_columns ds/hr
    + partition_columns.types string:string
    + rawDataSize 6312
    + serialization.ddl struct list_bucketing_mul_col { string col1, string col2, string col3, string col4, string col5}
    + serialization.format 1
    + serialization.lib org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
    + totalSize 7094
    +#### A masked pattern was here ####
    + serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
    +
    + input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat
    + output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat
    + properties:
    + bucket_count -1
    + columns col1,col2,col3,col4,col5
    + columns.comments
    + columns.types string:string:string:string:string
    +#### A masked pattern was here ####
    + name default.list_bucketing_mul_col
    + partition_columns ds/hr
    + partition_columns.types string:string
    + serialization.ddl struct list_bucketing_mul_col { string col1, string col2, string col3, string col4, string col5}
    + serialization.format 1
    + serialization.lib org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
    +#### A masked pattern was here ####
    + serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
    + name: default.list_bucketing_mul_col
    + name: default.list_bucketing_mul_col
    + Truncated Path -> Alias:
    + /list_bucketing_mul_col/ds=2008-04-08/hr=2013-01-23+18%3A00%3A99/col2=466/col4=val_466 [list_bucketing_mul_col]
    +
    + Stage: Stage-0
    + Fetch Operator
    + limit: -1
    + Processor Tree:
    + ListSink
    +
    +PREHOOK: query: select * from list_bucketing_mul_col
    +where ds='2008-04-08' and hr='2013-01-23+18:00:99' and col2 = "466" and col4 = "val_466"
    +PREHOOK: type: QUERY
    +PREHOOK: Input: default@list_bucketing_mul_col
    +PREHOOK: Input: default@list_bucketing_mul_col@ds=2008-04-08/hr=2013-01-23+18%3A00%3A99
    +#### A masked pattern was here ####
    +POSTHOOK: query: select * from list_bucketing_mul_col
    +where ds='2008-04-08' and hr='2013-01-23+18:00:99' and col2 = "466" and col4 = "val_466"
    +POSTHOOK: type: QUERY
    +POSTHOOK: Input: default@list_bucketing_mul_col
    +POSTHOOK: Input: default@list_bucketing_mul_col@ds=2008-04-08/hr=2013-01-23+18%3A00%3A99
    +#### A masked pattern was here ####
    +1 466 1 val_466 1 2008-04-08 2013-01-23+18:00:99
    +1 466 1 val_466 1 2008-04-08 2013-01-23+18:00:99
    +1 466 1 val_466 1 2008-04-08 2013-01-23+18:00:99
    +PREHOOK: query: drop table list_bucketing_mul_col
    +PREHOOK: type: DROPTABLE
    +PREHOOK: Input: default@list_bucketing_mul_col
    +PREHOOK: Output: default@list_bucketing_mul_col
    +POSTHOOK: query: drop table list_bucketing_mul_col
    +POSTHOOK: type: DROPTABLE
    +POSTHOOK: Input: default@list_bucketing_mul_col
    +POSTHOOK: Output: default@list_bucketing_mul_col

    Added: hive/trunk/ql/src/test/results/clientpositive/list_bucket_dml_13.q.java1.8.out
    URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/list_bucket_dml_13.q.java1.8.out?rev=1647912&view=auto
    ==============================================================================
    --- hive/trunk/ql/src/test/results/clientpositive/list_bucket_dml_13.q.java1.8.out (added)
    +++ hive/trunk/ql/src/test/results/clientpositive/list_bucket_dml_13.q.java1.8.out Thu Dec 25 18:06:30 2014
    @@ -0,0 +1,441 @@
    +PREHOOK: query: -- Ensure skewed value map has escaped directory name
    +
    +-- INCLUDE_HADOOP_MAJOR_VERSIONS(0.23)
    +-- SORT_QUERY_RESULTS
    +-- JAVA_VERSION_SPECIFIC_OUTPUT
    +
    +-- test where the skewed values are more than 1 say columns no. 2 and 4 in a table with 5 columns
    +create table list_bucketing_mul_col (col1 String, col2 String, col3 String, col4 String, col5 string)
    + partitioned by (ds String, hr String)
    + skewed by (col2, col4) on (('466','val_466'),('287','val_287'),('82','val_82'))
    + stored as DIRECTORIES
    + STORED AS RCFILE
    +PREHOOK: type: CREATETABLE
    +PREHOOK: Output: database:default
    +PREHOOK: Output: default@list_bucketing_mul_col
    +POSTHOOK: query: -- Ensure skewed value map has escaped directory name
    +
    +-- INCLUDE_HADOOP_MAJOR_VERSIONS(0.23)
    +-- SORT_QUERY_RESULTS
    +-- JAVA_VERSION_SPECIFIC_OUTPUT
    +
    +-- test where the skewed values are more than 1 say columns no. 2 and 4 in a table with 5 columns
    +create table list_bucketing_mul_col (col1 String, col2 String, col3 String, col4 String, col5 string)
    + partitioned by (ds String, hr String)
    + skewed by (col2, col4) on (('466','val_466'),('287','val_287'),('82','val_82'))
    + stored as DIRECTORIES
    + STORED AS RCFILE
    +POSTHOOK: type: CREATETABLE
    +POSTHOOK: Output: database:default
    +POSTHOOK: Output: default@list_bucketing_mul_col
    +PREHOOK: query: -- list bucketing DML
    +explain extended
    +insert overwrite table list_bucketing_mul_col partition (ds = '2008-04-08', hr = '2013-01-23+18:00:99')
    +select 1, key, 1, value, 1 from src
    +PREHOOK: type: QUERY
    +POSTHOOK: query: -- list bucketing DML
    +explain extended
    +insert overwrite table list_bucketing_mul_col partition (ds = '2008-04-08', hr = '2013-01-23+18:00:99')
    +select 1, key, 1, value, 1 from src
    +POSTHOOK: type: QUERY
    +ABSTRACT SYNTAX TREE:
    +
    +TOK_QUERY
    + TOK_FROM
    + TOK_TABREF
    + TOK_TABNAME
    + src
    + TOK_INSERT
    + TOK_DESTINATION
    + TOK_TAB
    + TOK_TABNAME
    + list_bucketing_mul_col
    + TOK_PARTSPEC
    + TOK_PARTVAL
    + ds
    + '2008-04-08'
    + TOK_PARTVAL
    + hr
    + '2013-01-23+18:00:99'
    + TOK_SELECT
    + TOK_SELEXPR
    + 1
    + TOK_SELEXPR
    + TOK_TABLE_OR_COL
    + key
    + TOK_SELEXPR
    + 1
    + TOK_SELEXPR
    + TOK_TABLE_OR_COL
    + value
    + TOK_SELEXPR
    + 1
    +
    +
    +STAGE DEPENDENCIES:
    + Stage-1 is a root stage
    + Stage-0 depends on stages: Stage-1
    + Stage-2 depends on stages: Stage-0
    +
    +STAGE PLANS:
    + Stage: Stage-1
    + Map Reduce
    + Map Operator Tree:
    + TableScan
    + alias: src
    + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
    + GatherStats: false
    + Select Operator
    + expressions: UDFToString(1) (type: string), key (type: string), UDFToString(1) (type: string), value (type: string), UDFToString(1) (type: string)
    + outputColumnNames: _col0, _col1, _col2, _col3, _col4
    + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
    + File Output Operator
    + compressed: false
    + GlobalTableId: 1
    +#### A masked pattern was here ####
    + NumFilesPerFileSink: 1
    + Static Partition Specification: ds=2008-04-08/hr=2013-01-23+18%3A00%3A99/
    + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
    +#### A masked pattern was here ####
    + table:
    + input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat
    + output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat
    + properties:
    + bucket_count -1
    + columns col1,col2,col3,col4,col5
    + columns.comments
    + columns.types string:string:string:string:string
    +#### A masked pattern was here ####
    + name default.list_bucketing_mul_col
    + partition_columns ds/hr
    + partition_columns.types string:string
    + serialization.ddl struct list_bucketing_mul_col { string col1, string col2, string col3, string col4, string col5}
    + serialization.format 1
    + serialization.lib org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
    +#### A masked pattern was here ####
    + serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
    + name: default.list_bucketing_mul_col
    + TotalFiles: 1
    + GatherStats: true
    + MultiFileSpray: false
    + Path -> Alias:
    +#### A masked pattern was here ####
    + Path -> Partition:
    +#### A masked pattern was here ####
    + Partition
    + base file name: src
    + input format: org.apache.hadoop.mapred.TextInputFormat
    + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
    + properties:
    + COLUMN_STATS_ACCURATE true
    + bucket_count -1
    + columns key,value
    + columns.comments 'default','default'
    + columns.types string:string
    +#### A masked pattern was here ####
    + name default.src
    + numFiles 1
    + numRows 500
    + rawDataSize 5312
    + serialization.ddl struct src { string key, string value}
    + serialization.format 1
    + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
    + totalSize 5812
    +#### A masked pattern was here ####
    + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
    +
    + input format: org.apache.hadoop.mapred.TextInputFormat
    + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
    + properties:
    + COLUMN_STATS_ACCURATE true
    + bucket_count -1
    + columns key,value
    + columns.comments 'default','default'
    + columns.types string:string
    +#### A masked pattern was here ####
    + name default.src
    + numFiles 1
    + numRows 500
    + rawDataSize 5312
    + serialization.ddl struct src { string key, string value}
    + serialization.format 1
    + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
    + totalSize 5812
    +#### A masked pattern was here ####
    + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
    + name: default.src
    + name: default.src
    + Truncated Path -> Alias:
    + /src [$hdt$_0:src]
    +
    + Stage: Stage-0
    + Move Operator
    + tables:
    + partition:
    + ds 2008-04-08
    + hr 2013-01-23+18:00:99
    + replace: true
    +#### A masked pattern was here ####
    + table:
    + input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat
    + output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat
    + properties:
    + bucket_count -1
    + columns col1,col2,col3,col4,col5
    + columns.comments
    + columns.types string:string:string:string:string
    +#### A masked pattern was here ####
    + name default.list_bucketing_mul_col
    + partition_columns ds/hr
    + partition_columns.types string:string
    + serialization.ddl struct list_bucketing_mul_col { string col1, string col2, string col3, string col4, string col5}
    + serialization.format 1
    + serialization.lib org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
    +#### A masked pattern was here ####
    + serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
    + name: default.list_bucketing_mul_col
    +
    + Stage: Stage-2
    + Stats-Aggr Operator
    +#### A masked pattern was here ####
    +
    +PREHOOK: query: insert overwrite table list_bucketing_mul_col partition (ds = '2008-04-08', hr = '2013-01-23+18:00:99')
    +select 1, key, 1, value, 1 from src
    +PREHOOK: type: QUERY
    +PREHOOK: Input: default@src
    +PREHOOK: Output: default@list_bucketing_mul_col@ds=2008-04-08/hr=2013-01-23+18%3A00%3A99
    +POSTHOOK: query: insert overwrite table list_bucketing_mul_col partition (ds = '2008-04-08', hr = '2013-01-23+18:00:99')
    +select 1, key, 1, value, 1 from src
    +POSTHOOK: type: QUERY
    +POSTHOOK: Input: default@src
    +POSTHOOK: Output: default@list_bucketing_mul_col@ds=2008-04-08/hr=2013-01-23+18%3A00%3A99
    +POSTHOOK: Lineage: list_bucketing_mul_col PARTITION(ds=2008-04-08,hr=2013-01-23+18:00:99).col1 EXPRESSION []
    +POSTHOOK: Lineage: list_bucketing_mul_col PARTITION(ds=2008-04-08,hr=2013-01-23+18:00:99).col2 SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ]
    +POSTHOOK: Lineage: list_bucketing_mul_col PARTITION(ds=2008-04-08,hr=2013-01-23+18:00:99).col3 EXPRESSION []
    +POSTHOOK: Lineage: list_bucketing_mul_col PARTITION(ds=2008-04-08,hr=2013-01-23+18:00:99).col4 SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
    +POSTHOOK: Lineage: list_bucketing_mul_col PARTITION(ds=2008-04-08,hr=2013-01-23+18:00:99).col5 EXPRESSION []
    +PREHOOK: query: -- check DML result
    +show partitions list_bucketing_mul_col
    +PREHOOK: type: SHOWPARTITIONS
    +PREHOOK: Input: default@list_bucketing_mul_col
    +POSTHOOK: query: -- check DML result
    +show partitions list_bucketing_mul_col
    +POSTHOOK: type: SHOWPARTITIONS
    +POSTHOOK: Input: default@list_bucketing_mul_col
    +ds=2008-04-08/hr=2013-01-23+18%3A00%3A99
    +PREHOOK: query: desc formatted list_bucketing_mul_col partition (ds='2008-04-08', hr='2013-01-23+18:00:99')
    +PREHOOK: type: DESCTABLE
    +PREHOOK: Input: default@list_bucketing_mul_col
    +POSTHOOK: query: desc formatted list_bucketing_mul_col partition (ds='2008-04-08', hr='2013-01-23+18:00:99')
    +POSTHOOK: type: DESCTABLE
    +POSTHOOK: Input: default@list_bucketing_mul_col
    +# col_name data_type comment
    +
    +col1 string
    +col2 string
    +col3 string
    +col4 string
    +col5 string
    +
    +# Partition Information
    +# col_name data_type comment
    +
    +ds string
    +hr string
    +
    +# Detailed Partition Information
    +Partition Value: [2008-04-08, 2013-01-23+18:00:99]
    +Database: default
    +Table: list_bucketing_mul_col
    +#### A masked pattern was here ####
    +Protect Mode: None
    +#### A masked pattern was here ####
    +Partition Parameters:
    + COLUMN_STATS_ACCURATE true
    + numFiles 4
    + numRows 500
    + rawDataSize 6312
    + totalSize 7094
    +#### A masked pattern was here ####
    +
    +# Storage Information
    +SerDe Library: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
    +InputFormat: org.apache.hadoop.hive.ql.io.RCFileInputFormat
    +OutputFormat: org.apache.hadoop.hive.ql.io.RCFileOutputFormat
    +Compressed: No
    +Num Buckets: -1
    +Bucket Columns: []
    +Sort Columns: []
    +Stored As SubDirectories: Yes
    +Skewed Columns: [col2, col4]
    +Skewed Values: [[466, val_466], [287, val_287], [82, val_82]]
    +#### A masked pattern was here ####
    +Skewed Value to Truncated Path: {[466, val_466]=/list_bucketing_mul_col/ds=2008-04-08/hr=2013-01-23+18%3A00%3A99/col2=466/col4=val_466, [287, val_287]=/list_bucketing_mul_col/ds=2008-04-08/hr=2013-01-23+18%3A00%3A99/col2=287/col4=val_287, [82, val_82]=/list_bucketing_mul_col/ds=2008-04-08/hr=2013-01-23+18%3A00%3A99/col2=82/col4=val_82}
    +Storage Desc Params:
    + serialization.format 1
    +PREHOOK: query: explain extended
    +select * from list_bucketing_mul_col
    +where ds='2008-04-08' and hr='2013-01-23+18:00:99' and col2 = "466" and col4 = "val_466"
    +PREHOOK: type: QUERY
    +POSTHOOK: query: explain extended
    +select * from list_bucketing_mul_col
    +where ds='2008-04-08' and hr='2013-01-23+18:00:99' and col2 = "466" and col4 = "val_466"
    +POSTHOOK: type: QUERY
    +ABSTRACT SYNTAX TREE:
    +
    +TOK_QUERY
    + TOK_FROM
    + TOK_TABREF
    + TOK_TABNAME
    + list_bucketing_mul_col
    + TOK_INSERT
    + TOK_DESTINATION
    + TOK_DIR
    + TOK_TMP_FILE
    + TOK_SELECT
    + TOK_SELEXPR
    + TOK_ALLCOLREF
    + TOK_WHERE
    + and
    + and
    + and
    + =
    + TOK_TABLE_OR_COL
    + ds
    + '2008-04-08'
    + =
    + TOK_TABLE_OR_COL
    + hr
    + '2013-01-23+18:00:99'
    + =
    + TOK_TABLE_OR_COL
    + col2
    + "466"
    + =
    + TOK_TABLE_OR_COL
    + col4
    + "val_466"
    +
    +
    +STAGE DEPENDENCIES:
    + Stage-1 is a root stage
    + Stage-0 depends on stages: Stage-1
    +
    +STAGE PLANS:
    + Stage: Stage-1
    + Map Reduce
    + Map Operator Tree:
    + TableScan
    + alias: list_bucketing_mul_col
    + Statistics: Num rows: 500 Data size: 6312 Basic stats: COMPLETE Column stats: NONE
    + GatherStats: false
    + Filter Operator
    + isSamplingPred: false
    + predicate: ((col2 = '466') and (col4 = 'val_466')) (type: boolean)
    + Statistics: Num rows: 125 Data size: 1578 Basic stats: COMPLETE Column stats: NONE
    + Select Operator
    + expressions: col1 (type: string), '466' (type: string), col3 (type: string), 'val_466' (type: string), col5 (type: string), '2008-04-08' (type: string), '2013-01-23+18:00:99' (type: string)
    + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6
    + Statistics: Num rows: 125 Data size: 1578 Basic stats: COMPLETE Column stats: NONE
    + File Output Operator
    + compressed: false
    + GlobalTableId: 0
    +#### A masked pattern was here ####
    + NumFilesPerFileSink: 1
    + Statistics: Num rows: 125 Data size: 1578 Basic stats: COMPLETE Column stats: NONE
    +#### A masked pattern was here ####
    + table:
    + input format: org.apache.hadoop.mapred.TextInputFormat
    + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
    + properties:
    + columns _col0,_col1,_col2,_col3,_col4,_col5,_col6
    + columns.types string:string:string:string:string:string:string
    + escape.delim \
    + hive.serialization.extend.nesting.levels true
    + serialization.format 1
    + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
    + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
    + TotalFiles: 1
    + GatherStats: false
    + MultiFileSpray: false
    + Path -> Alias:
    +#### A masked pattern was here ####
    + Path -> Partition:
    +#### A masked pattern was here ####
    + Partition
    + base file name: col4=val_466
    + input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat
    + output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat
    + partition values:
    + ds 2008-04-08
    + hr 2013-01-23+18:00:99
    + properties:
    + COLUMN_STATS_ACCURATE true
    + bucket_count -1
    + columns col1,col2,col3,col4,col5
    + columns.comments
    + columns.types string:string:string:string:string
    +#### A masked pattern was here ####
    + name default.list_bucketing_mul_col
    + numFiles 4
    + numRows 500
    + partition_columns ds/hr
    + partition_columns.types string:string
    + rawDataSize 6312
    + serialization.ddl struct list_bucketing_mul_col { string col1, string col2, string col3, string col4, string col5}
    + serialization.format 1
    + serialization.lib org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
    + totalSize 7094
    +#### A masked pattern was here ####
    + serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
    +
    + input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat
    + output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat
    + properties:
    + bucket_count -1
    + columns col1,col2,col3,col4,col5
    + columns.comments
    + columns.types string:string:string:string:string
    +#### A masked pattern was here ####
    + name default.list_bucketing_mul_col
    + partition_columns ds/hr
    + partition_columns.types string:string
    + serialization.ddl struct list_bucketing_mul_col { string col1, string col2, string col3, string col4, string col5}
    + serialization.format 1
    + serialization.lib org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
    +#### A masked pattern was here ####
    + serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
    + name: default.list_bucketing_mul_col
    + name: default.list_bucketing_mul_col
    + Truncated Path -> Alias:
    + /list_bucketing_mul_col/ds=2008-04-08/hr=2013-01-23+18%3A00%3A99/col2=466/col4=val_466 [list_bucketing_mul_col]
    +
    + Stage: Stage-0
    + Fetch Operator
    + limit: -1
    + Processor Tree:
    + ListSink
    +
    +PREHOOK: query: select * from list_bucketing_mul_col
    +where ds='2008-04-08' and hr='2013-01-23+18:00:99' and col2 = "466" and col4 = "val_466"
    +PREHOOK: type: QUERY
    +PREHOOK: Input: default@list_bucketing_mul_col
    +PREHOOK: Input: default@list_bucketing_mul_col@ds=2008-04-08/hr=2013-01-23+18%3A00%3A99
    +#### A masked pattern was here ####
    +POSTHOOK: query: select * from list_bucketing_mul_col
    +where ds='2008-04-08' and hr='2013-01-23+18:00:99' and col2 = "466" and col4 = "val_466"
    +POSTHOOK: type: QUERY
    +POSTHOOK: Input: default@list_bucketing_mul_col
    +POSTHOOK: Input: default@list_bucketing_mul_col@ds=2008-04-08/hr=2013-01-23+18%3A00%3A99
    +#### A masked pattern was here ####
    +1 466 1 val_466 1 2008-04-08 2013-01-23+18:00:99
    +1 466 1 val_466 1 2008-04-08 2013-01-23+18:00:99
    +1 466 1 val_466 1 2008-04-08 2013-01-23+18:00:99
    +PREHOOK: query: drop table list_bucketing_mul_col
    +PREHOOK: type: DROPTABLE
    +PREHOOK: Input: default@list_bucketing_mul_col
    +PREHOOK: Output: default@list_bucketing_mul_col
    +POSTHOOK: query: drop table list_bucketing_mul_col
    +POSTHOOK: type: DROPTABLE
    +POSTHOOK: Input: default@list_bucketing_mul_col
    +POSTHOOK: Output: default@list_bucketing_mul_col
  • Brock at Dec 25, 2014 at 6:06 pm
    Added: hive/trunk/ql/src/test/results/clientpositive/list_bucket_dml_5.q.java1.7.out
    URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/list_bucket_dml_5.q.java1.7.out?rev=1647912&view=auto
    ==============================================================================
    --- hive/trunk/ql/src/test/results/clientpositive/list_bucket_dml_5.q.java1.7.out (added)
    +++ hive/trunk/ql/src/test/results/clientpositive/list_bucket_dml_5.q.java1.7.out Thu Dec 25 18:06:30 2014
    @@ -0,0 +1,621 @@
    +PREHOOK: query: -- list bucketing DML: multiple skewed columns. 2 stages
    +
    +-- INCLUDE_HADOOP_MAJOR_VERSIONS(0.23)
    +-- SORT_QUERY_RESULTS
    +-- JAVA_VERSION_SPECIFIC_OUTPUT
    +
    +-- create a skewed table
    +create table list_bucketing_dynamic_part (key String, value String)
    +partitioned by (ds String, hr String)
    +skewed by (key, value) on (('484','val_484'),('51','val_14'),('103','val_103'))
    +stored as DIRECTORIES
    +PREHOOK: type: CREATETABLE
    +PREHOOK: Output: database:default
    +PREHOOK: Output: default@list_bucketing_dynamic_part
    +POSTHOOK: query: -- list bucketing DML: multiple skewed columns. 2 stages
    +
    +-- INCLUDE_HADOOP_MAJOR_VERSIONS(0.23)
    +-- SORT_QUERY_RESULTS
    +-- JAVA_VERSION_SPECIFIC_OUTPUT
    +
    +-- create a skewed table
    +create table list_bucketing_dynamic_part (key String, value String)
    +partitioned by (ds String, hr String)
    +skewed by (key, value) on (('484','val_484'),('51','val_14'),('103','val_103'))
    +stored as DIRECTORIES
    +POSTHOOK: type: CREATETABLE
    +POSTHOOK: Output: database:default
    +POSTHOOK: Output: default@list_bucketing_dynamic_part
    +PREHOOK: query: -- list bucketing DML
    +explain extended
    +insert overwrite table list_bucketing_dynamic_part partition (ds='2008-04-08', hr) select key, value, hr from srcpart where ds='2008-04-08'
    +PREHOOK: type: QUERY
    +POSTHOOK: query: -- list bucketing DML
    +explain extended
    +insert overwrite table list_bucketing_dynamic_part partition (ds='2008-04-08', hr) select key, value, hr from srcpart where ds='2008-04-08'
    +POSTHOOK: type: QUERY
    +ABSTRACT SYNTAX TREE:
    +
    +TOK_QUERY
    + TOK_FROM
    + TOK_TABREF
    + TOK_TABNAME
    + srcpart
    + TOK_INSERT
    + TOK_DESTINATION
    + TOK_TAB
    + TOK_TABNAME
    + list_bucketing_dynamic_part
    + TOK_PARTSPEC
    + TOK_PARTVAL
    + ds
    + '2008-04-08'
    + TOK_PARTVAL
    + hr
    + TOK_SELECT
    + TOK_SELEXPR
    + TOK_TABLE_OR_COL
    + key
    + TOK_SELEXPR
    + TOK_TABLE_OR_COL
    + value
    + TOK_SELEXPR
    + TOK_TABLE_OR_COL
    + hr
    + TOK_WHERE
    + =
    + TOK_TABLE_OR_COL
    + ds
    + '2008-04-08'
    +
    +
    +STAGE DEPENDENCIES:
    + Stage-1 is a root stage
    + Stage-0 depends on stages: Stage-1
    + Stage-2 depends on stages: Stage-0
    +
    +STAGE PLANS:
    + Stage: Stage-1
    + Map Reduce
    + Map Operator Tree:
    + TableScan
    + alias: srcpart
    + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE
    + GatherStats: false
    + Select Operator
    + expressions: key (type: string), value (type: string), hr (type: string)
    + outputColumnNames: _col0, _col1, _col2
    + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE
    + File Output Operator
    + compressed: false
    + GlobalTableId: 1
    +#### A masked pattern was here ####
    + NumFilesPerFileSink: 1
    + Static Partition Specification: ds=2008-04-08/
    + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE
    +#### A masked pattern was here ####
    + table:
    + input format: org.apache.hadoop.mapred.TextInputFormat
    + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
    + properties:
    + bucket_count -1
    + columns key,value
    + columns.comments
    + columns.types string:string
    +#### A masked pattern was here ####
    + name default.list_bucketing_dynamic_part
    + partition_columns ds/hr
    + partition_columns.types string:string
    + serialization.ddl struct list_bucketing_dynamic_part { string key, string value}
    + serialization.format 1
    + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
    +#### A masked pattern was here ####
    + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
    + name: default.list_bucketing_dynamic_part
    + TotalFiles: 1
    + GatherStats: true
    + MultiFileSpray: false
    + Path -> Alias:
    +#### A masked pattern was here ####
    + Path -> Partition:
    +#### A masked pattern was here ####
    + Partition
    + base file name: hr=11
    + input format: org.apache.hadoop.mapred.TextInputFormat
    + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
    + partition values:
    + ds 2008-04-08
    + hr 11
    + properties:
    + COLUMN_STATS_ACCURATE true
    + bucket_count -1
    + columns key,value
    + columns.comments 'default','default'
    + columns.types string:string
    +#### A masked pattern was here ####
    + name default.srcpart
    + numFiles 1
    + numRows 500
    + partition_columns ds/hr
    + partition_columns.types string:string
    + rawDataSize 5312
    + serialization.ddl struct srcpart { string key, string value}
    + serialization.format 1
    + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
    + totalSize 5812
    +#### A masked pattern was here ####
    + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
    +
    + input format: org.apache.hadoop.mapred.TextInputFormat
    + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
    + properties:
    + bucket_count -1
    + columns key,value
    + columns.comments 'default','default'
    + columns.types string:string
    +#### A masked pattern was here ####
    + name default.srcpart
    + partition_columns ds/hr
    + partition_columns.types string:string
    + serialization.ddl struct srcpart { string key, string value}
    + serialization.format 1
    + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
    +#### A masked pattern was here ####
    + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
    + name: default.srcpart
    + name: default.srcpart
    +#### A masked pattern was here ####
    + Partition
    + base file name: hr=12
    + input format: org.apache.hadoop.mapred.TextInputFormat
    + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
    + partition values:
    + ds 2008-04-08
    + hr 12
    + properties:
    + COLUMN_STATS_ACCURATE true
    + bucket_count -1
    + columns key,value
    + columns.comments 'default','default'
    + columns.types string:string
    +#### A masked pattern was here ####
    + name default.srcpart
    + numFiles 1
    + numRows 500
    + partition_columns ds/hr
    + partition_columns.types string:string
    + rawDataSize 5312
    + serialization.ddl struct srcpart { string key, string value}
    + serialization.format 1
    + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
    + totalSize 5812
    +#### A masked pattern was here ####
    + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
    +
    + input format: org.apache.hadoop.mapred.TextInputFormat
    + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
    + properties:
    + bucket_count -1
    + columns key,value
    + columns.comments 'default','default'
    + columns.types string:string
    +#### A masked pattern was here ####
    + name default.srcpart
    + partition_columns ds/hr
    + partition_columns.types string:string
    + serialization.ddl struct srcpart { string key, string value}
    + serialization.format 1
    + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
    +#### A masked pattern was here ####
    + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
    + name: default.srcpart
    + name: default.srcpart
    + Truncated Path -> Alias:
    + /srcpart/ds=2008-04-08/hr=11 [$hdt$_0:srcpart]
    + /srcpart/ds=2008-04-08/hr=12 [$hdt$_0:srcpart]
    +
    + Stage: Stage-0
    + Move Operator
    + tables:
    + partition:
    + ds 2008-04-08
    + hr
    + replace: true
    +#### A masked pattern was here ####
    + table:
    + input format: org.apache.hadoop.mapred.TextInputFormat
    + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
    + properties:
    + bucket_count -1
    + columns key,value
    + columns.comments
    + columns.types string:string
    +#### A masked pattern was here ####
    + name default.list_bucketing_dynamic_part
    + partition_columns ds/hr
    + partition_columns.types string:string
    + serialization.ddl struct list_bucketing_dynamic_part { string key, string value}
    + serialization.format 1
    + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
    +#### A masked pattern was here ####
    + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
    + name: default.list_bucketing_dynamic_part
    +
    + Stage: Stage-2
    + Stats-Aggr Operator
    +#### A masked pattern was here ####
    +
    +PREHOOK: query: insert overwrite table list_bucketing_dynamic_part partition (ds='2008-04-08', hr) select key, value, hr from srcpart where ds='2008-04-08'
    +PREHOOK: type: QUERY
    +PREHOOK: Input: default@srcpart
    +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11
    +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12
    +PREHOOK: Output: default@list_bucketing_dynamic_part@ds=2008-04-08
    +POSTHOOK: query: insert overwrite table list_bucketing_dynamic_part partition (ds='2008-04-08', hr) select key, value, hr from srcpart where ds='2008-04-08'
    +POSTHOOK: type: QUERY
    +POSTHOOK: Input: default@srcpart
    +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11
    +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12
    +POSTHOOK: Output: default@list_bucketing_dynamic_part@ds=2008-04-08/hr=11
    +POSTHOOK: Output: default@list_bucketing_dynamic_part@ds=2008-04-08/hr=12
    +POSTHOOK: Lineage: list_bucketing_dynamic_part PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ]
    +POSTHOOK: Lineage: list_bucketing_dynamic_part PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ]
    +POSTHOOK: Lineage: list_bucketing_dynamic_part PARTITION(ds=2008-04-08,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ]
    +POSTHOOK: Lineage: list_bucketing_dynamic_part PARTITION(ds=2008-04-08,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ]
    +PREHOOK: query: -- check DML result
    +desc formatted list_bucketing_dynamic_part partition (ds='2008-04-08', hr='11')
    +PREHOOK: type: DESCTABLE
    +PREHOOK: Input: default@list_bucketing_dynamic_part
    +POSTHOOK: query: -- check DML result
    +desc formatted list_bucketing_dynamic_part partition (ds='2008-04-08', hr='11')
    +POSTHOOK: type: DESCTABLE
    +POSTHOOK: Input: default@list_bucketing_dynamic_part
    +# col_name data_type comment
    +
    +key string
    +value string
    +
    +# Partition Information
    +# col_name data_type comment
    +
    +ds string
    +hr string
    +
    +# Detailed Partition Information
    +Partition Value: [2008-04-08, 11]
    +Database: default
    +Table: list_bucketing_dynamic_part
    +#### A masked pattern was here ####
    +Protect Mode: None
    +#### A masked pattern was here ####
    +Partition Parameters:
    + COLUMN_STATS_ACCURATE true
    + numFiles 3
    + numRows 500
    + rawDataSize 5312
    + totalSize 5812
    +#### A masked pattern was here ####
    +
    +# Storage Information
    +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
    +InputFormat: org.apache.hadoop.mapred.TextInputFormat
    +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
    +Compressed: No
    +Num Buckets: -1
    +Bucket Columns: []
    +Sort Columns: []
    +Stored As SubDirectories: Yes
    +Skewed Columns: [key, value]
    +Skewed Values: [[484, val_484], [51, val_14], [103, val_103]]
    +#### A masked pattern was here ####
    +Skewed Value to Truncated Path: {[103, val_103]=/list_bucketing_dynamic_part/ds=2008-04-08/hr=11/key=103/value=val_103, [484, val_484]=/list_bucketing_dynamic_part/ds=2008-04-08/hr=11/key=484/value=val_484}
    +Storage Desc Params:
    + serialization.format 1
    +PREHOOK: query: desc formatted list_bucketing_dynamic_part partition (ds='2008-04-08', hr='12')
    +PREHOOK: type: DESCTABLE
    +PREHOOK: Input: default@list_bucketing_dynamic_part
    +POSTHOOK: query: desc formatted list_bucketing_dynamic_part partition (ds='2008-04-08', hr='12')
    +POSTHOOK: type: DESCTABLE
    +POSTHOOK: Input: default@list_bucketing_dynamic_part
    +# col_name data_type comment
    +
    +key string
    +value string
    +
    +# Partition Information
    +# col_name data_type comment
    +
    +ds string
    +hr string
    +
    +# Detailed Partition Information
    +Partition Value: [2008-04-08, 12]
    +Database: default
    +Table: list_bucketing_dynamic_part
    +#### A masked pattern was here ####
    +Protect Mode: None
    +#### A masked pattern was here ####
    +Partition Parameters:
    + COLUMN_STATS_ACCURATE true
    + numFiles 3
    + numRows 500
    + rawDataSize 5312
    + totalSize 5812
    +#### A masked pattern was here ####
    +
    +# Storage Information
    +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
    +InputFormat: org.apache.hadoop.mapred.TextInputFormat
    +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
    +Compressed: No
    +Num Buckets: -1
    +Bucket Columns: []
    +Sort Columns: []
    +Stored As SubDirectories: Yes
    +Skewed Columns: [key, value]
    +Skewed Values: [[484, val_484], [51, val_14], [103, val_103]]
    +#### A masked pattern was here ####
    +Skewed Value to Truncated Path: {[103, val_103]=/list_bucketing_dynamic_part/ds=2008-04-08/hr=12/key=103/value=val_103, [484, val_484]=/list_bucketing_dynamic_part/ds=2008-04-08/hr=12/key=484/value=val_484}
    +Storage Desc Params:
    + serialization.format 1
    +PREHOOK: query: select count(1) from srcpart where ds='2008-04-08'
    +PREHOOK: type: QUERY
    +PREHOOK: Input: default@srcpart
    +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11
    +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12
    +#### A masked pattern was here ####
    +POSTHOOK: query: select count(1) from srcpart where ds='2008-04-08'
    +POSTHOOK: type: QUERY
    +POSTHOOK: Input: default@srcpart
    +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11
    +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12
    +#### A masked pattern was here ####
    +1000
    +PREHOOK: query: select count(1) from list_bucketing_dynamic_part where ds='2008-04-08'
    +PREHOOK: type: QUERY
    +PREHOOK: Input: default@list_bucketing_dynamic_part
    +PREHOOK: Input: default@list_bucketing_dynamic_part@ds=2008-04-08/hr=11
    +PREHOOK: Input: default@list_bucketing_dynamic_part@ds=2008-04-08/hr=12
    +#### A masked pattern was here ####
    +POSTHOOK: query: select count(1) from list_bucketing_dynamic_part where ds='2008-04-08'
    +POSTHOOK: type: QUERY
    +POSTHOOK: Input: default@list_bucketing_dynamic_part
    +POSTHOOK: Input: default@list_bucketing_dynamic_part@ds=2008-04-08/hr=11
    +POSTHOOK: Input: default@list_bucketing_dynamic_part@ds=2008-04-08/hr=12
    +#### A masked pattern was here ####
    +1000
    +PREHOOK: query: select key, value from srcpart where ds='2008-04-08' and key = "103" and value ="val_103"
    +PREHOOK: type: QUERY
    +PREHOOK: Input: default@srcpart
    +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11
    +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12
    +#### A masked pattern was here ####
    +POSTHOOK: query: select key, value from srcpart where ds='2008-04-08' and key = "103" and value ="val_103"
    +POSTHOOK: type: QUERY
    +POSTHOOK: Input: default@srcpart
    +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11
    +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12
    +#### A masked pattern was here ####
    +103 val_103
    +103 val_103
    +103 val_103
    +103 val_103
    +PREHOOK: query: explain extended
    +select key, value, ds, hr from list_bucketing_dynamic_part where ds='2008-04-08' and key = "103" and value ="val_103"
    +PREHOOK: type: QUERY
    +POSTHOOK: query: explain extended
    +select key, value, ds, hr from list_bucketing_dynamic_part where ds='2008-04-08' and key = "103" and value ="val_103"
    +POSTHOOK: type: QUERY
    +ABSTRACT SYNTAX TREE:
    +
    +TOK_QUERY
    + TOK_FROM
    + TOK_TABREF
    + TOK_TABNAME
    + list_bucketing_dynamic_part
    + TOK_INSERT
    + TOK_DESTINATION
    + TOK_DIR
    + TOK_TMP_FILE
    + TOK_SELECT
    + TOK_SELEXPR
    + TOK_TABLE_OR_COL
    + key
    + TOK_SELEXPR
    + TOK_TABLE_OR_COL
    + value
    + TOK_SELEXPR
    + TOK_TABLE_OR_COL
    + ds
    + TOK_SELEXPR
    + TOK_TABLE_OR_COL
    + hr
    + TOK_WHERE
    + and
    + and
    + =
    + TOK_TABLE_OR_COL
    + ds
    + '2008-04-08'
    + =
    + TOK_TABLE_OR_COL
    + key
    + "103"
    + =
    + TOK_TABLE_OR_COL
    + value
    + "val_103"
    +
    +
    +STAGE DEPENDENCIES:
    + Stage-1 is a root stage
    + Stage-0 depends on stages: Stage-1
    +
    +STAGE PLANS:
    + Stage: Stage-1
    + Map Reduce
    + Map Operator Tree:
    + TableScan
    + alias: list_bucketing_dynamic_part
    + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE
    + GatherStats: false
    + Filter Operator
    + isSamplingPred: false
    + predicate: ((key = '103') and (value = 'val_103')) (type: boolean)
    + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
    + Select Operator
    + expressions: '103' (type: string), 'val_103' (type: string), '2008-04-08' (type: string), hr (type: string)
    + outputColumnNames: _col0, _col1, _col2, _col3
    + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
    + File Output Operator
    + compressed: false
    + GlobalTableId: 0
    +#### A masked pattern was here ####
    + NumFilesPerFileSink: 1
    + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
    +#### A masked pattern was here ####
    + table:
    + input format: org.apache.hadoop.mapred.TextInputFormat
    + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
    + properties:
    + columns _col0,_col1,_col2,_col3
    + columns.types string:string:string:string
    + escape.delim \
    + hive.serialization.extend.nesting.levels true
    + serialization.format 1
    + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
    + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
    + TotalFiles: 1
    + GatherStats: false
    + MultiFileSpray: false
    + Path -> Alias:
    +#### A masked pattern was here ####
    + Path -> Partition:
    +#### A masked pattern was here ####
    + Partition
    + base file name: value=val_103
    + input format: org.apache.hadoop.mapred.TextInputFormat
    + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
    + partition values:
    + ds 2008-04-08
    + hr 11
    + properties:
    + COLUMN_STATS_ACCURATE true
    + bucket_count -1
    + columns key,value
    + columns.comments
    + columns.types string:string
    +#### A masked pattern was here ####
    + name default.list_bucketing_dynamic_part
    + numFiles 3
    + numRows 500
    + partition_columns ds/hr
    + partition_columns.types string:string
    + rawDataSize 5312
    + serialization.ddl struct list_bucketing_dynamic_part { string key, string value}
    + serialization.format 1
    + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
    + totalSize 5812
    +#### A masked pattern was here ####
    + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
    +
    + input format: org.apache.hadoop.mapred.TextInputFormat
    + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
    + properties:
    + bucket_count -1
    + columns key,value
    + columns.comments
    + columns.types string:string
    +#### A masked pattern was here ####
    + name default.list_bucketing_dynamic_part
    + partition_columns ds/hr
    + partition_columns.types string:string
    + serialization.ddl struct list_bucketing_dynamic_part { string key, string value}
    + serialization.format 1
    + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
    +#### A masked pattern was here ####
    + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
    + name: default.list_bucketing_dynamic_part
    + name: default.list_bucketing_dynamic_part
    +#### A masked pattern was here ####
    + Partition
    + base file name: value=val_103
    + input format: org.apache.hadoop.mapred.TextInputFormat
    + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
    + partition values:
    + ds 2008-04-08
    + hr 12
    + properties:
    + COLUMN_STATS_ACCURATE true
    + bucket_count -1
    + columns key,value
    + columns.comments
    + columns.types string:string
    +#### A masked pattern was here ####
    + name default.list_bucketing_dynamic_part
    + numFiles 3
    + numRows 500
    + partition_columns ds/hr
    + partition_columns.types string:string
    + rawDataSize 5312
    + serialization.ddl struct list_bucketing_dynamic_part { string key, string value}
    + serialization.format 1
    + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
    + totalSize 5812
    +#### A masked pattern was here ####
    + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
    +
    + input format: org.apache.hadoop.mapred.TextInputFormat
    + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
    + properties:
    + bucket_count -1
    + columns key,value
    + columns.comments
    + columns.types string:string
    +#### A masked pattern was here ####
    + name default.list_bucketing_dynamic_part
    + partition_columns ds/hr
    + partition_columns.types string:string
    + serialization.ddl struct list_bucketing_dynamic_part { string key, string value}
    + serialization.format 1
    + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
    +#### A masked pattern was here ####
    + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
    + name: default.list_bucketing_dynamic_part
    + name: default.list_bucketing_dynamic_part
    + Truncated Path -> Alias:
    + /list_bucketing_dynamic_part/ds=2008-04-08/hr=11/key=103/value=val_103 [list_bucketing_dynamic_part]
    + /list_bucketing_dynamic_part/ds=2008-04-08/hr=12/key=103/value=val_103 [list_bucketing_dynamic_part]
    +
    + Stage: Stage-0
    + Fetch Operator
    + limit: -1
    + Processor Tree:
    + ListSink
    +
    +PREHOOK: query: select key, value, ds, hr from list_bucketing_dynamic_part where ds='2008-04-08' and key = "103" and value ="val_103"
    +PREHOOK: type: QUERY
    +PREHOOK: Input: default@list_bucketing_dynamic_part
    +PREHOOK: Input: default@list_bucketing_dynamic_part@ds=2008-04-08/hr=11
    +PREHOOK: Input: default@list_bucketing_dynamic_part@ds=2008-04-08/hr=12
    +#### A masked pattern was here ####
    +POSTHOOK: query: select key, value, ds, hr from list_bucketing_dynamic_part where ds='2008-04-08' and key = "103" and value ="val_103"
    +POSTHOOK: type: QUERY
    +POSTHOOK: Input: default@list_bucketing_dynamic_part
    +POSTHOOK: Input: default@list_bucketing_dynamic_part@ds=2008-04-08/hr=11
    +POSTHOOK: Input: default@list_bucketing_dynamic_part@ds=2008-04-08/hr=12
    +#### A masked pattern was here ####
    +103 val_103 2008-04-08 11
    +103 val_103 2008-04-08 11
    +103 val_103 2008-04-08 12
    +103 val_103 2008-04-08 12
    +PREHOOK: query: -- clean up resources
    +drop table list_bucketing_dynamic_part
    +PREHOOK: type: DROPTABLE
    +PREHOOK: Input: default@list_bucketing_dynamic_part
    +PREHOOK: Output: default@list_bucketing_dynamic_part
    +POSTHOOK: query: -- clean up resources
    +drop table list_bucketing_dynamic_part
    +POSTHOOK: type: DROPTABLE
    +POSTHOOK: Input: default@list_bucketing_dynamic_part
    +POSTHOOK: Output: default@list_bucketing_dynamic_part

    Added: hive/trunk/ql/src/test/results/clientpositive/list_bucket_dml_5.q.java1.8.out
    URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/list_bucket_dml_5.q.java1.8.out?rev=1647912&view=auto
    ==============================================================================
    --- hive/trunk/ql/src/test/results/clientpositive/list_bucket_dml_5.q.java1.8.out (added)
    +++ hive/trunk/ql/src/test/results/clientpositive/list_bucket_dml_5.q.java1.8.out Thu Dec 25 18:06:30 2014
    @@ -0,0 +1,621 @@
    +PREHOOK: query: -- list bucketing DML: multiple skewed columns. 2 stages
    +
    +-- INCLUDE_HADOOP_MAJOR_VERSIONS(0.23)
    +-- SORT_QUERY_RESULTS
    +-- JAVA_VERSION_SPECIFIC_OUTPUT
    +
    +-- create a skewed table
    +create table list_bucketing_dynamic_part (key String, value String)
    +partitioned by (ds String, hr String)
    +skewed by (key, value) on (('484','val_484'),('51','val_14'),('103','val_103'))
    +stored as DIRECTORIES
    +PREHOOK: type: CREATETABLE
    +PREHOOK: Output: database:default
    +PREHOOK: Output: default@list_bucketing_dynamic_part
    +POSTHOOK: query: -- list bucketing DML: multiple skewed columns. 2 stages
    +
    +-- INCLUDE_HADOOP_MAJOR_VERSIONS(0.23)
    +-- SORT_QUERY_RESULTS
    +-- JAVA_VERSION_SPECIFIC_OUTPUT
    +
    +-- create a skewed table
    +create table list_bucketing_dynamic_part (key String, value String)
    +partitioned by (ds String, hr String)
    +skewed by (key, value) on (('484','val_484'),('51','val_14'),('103','val_103'))
    +stored as DIRECTORIES
    +POSTHOOK: type: CREATETABLE
    +POSTHOOK: Output: database:default
    +POSTHOOK: Output: default@list_bucketing_dynamic_part
    +PREHOOK: query: -- list bucketing DML
    +explain extended
    +insert overwrite table list_bucketing_dynamic_part partition (ds='2008-04-08', hr) select key, value, hr from srcpart where ds='2008-04-08'
    +PREHOOK: type: QUERY
    +POSTHOOK: query: -- list bucketing DML
    +explain extended
    +insert overwrite table list_bucketing_dynamic_part partition (ds='2008-04-08', hr) select key, value, hr from srcpart where ds='2008-04-08'
    +POSTHOOK: type: QUERY
    +ABSTRACT SYNTAX TREE:
    +
    +TOK_QUERY
    + TOK_FROM
    + TOK_TABREF
    + TOK_TABNAME
    + srcpart
    + TOK_INSERT
    + TOK_DESTINATION
    + TOK_TAB
    + TOK_TABNAME
    + list_bucketing_dynamic_part
    + TOK_PARTSPEC
    + TOK_PARTVAL
    + ds
    + '2008-04-08'
    + TOK_PARTVAL
    + hr
    + TOK_SELECT
    + TOK_SELEXPR
    + TOK_TABLE_OR_COL
    + key
    + TOK_SELEXPR
    + TOK_TABLE_OR_COL
    + value
    + TOK_SELEXPR
    + TOK_TABLE_OR_COL
    + hr
    + TOK_WHERE
    + =
    + TOK_TABLE_OR_COL
    + ds
    + '2008-04-08'
    +
    +
    +STAGE DEPENDENCIES:
    + Stage-1 is a root stage
    + Stage-0 depends on stages: Stage-1
    + Stage-2 depends on stages: Stage-0
    +
    +STAGE PLANS:
    + Stage: Stage-1
    + Map Reduce
    + Map Operator Tree:
    + TableScan
    + alias: srcpart
    + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE
    + GatherStats: false
    + Select Operator
    + expressions: key (type: string), value (type: string), hr (type: string)
    + outputColumnNames: _col0, _col1, _col2
    + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE
    + File Output Operator
    + compressed: false
    + GlobalTableId: 1
    +#### A masked pattern was here ####
    + NumFilesPerFileSink: 1
    + Static Partition Specification: ds=2008-04-08/
    + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE
    +#### A masked pattern was here ####
    + table:
    + input format: org.apache.hadoop.mapred.TextInputFormat
    + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
    + properties:
    + bucket_count -1
    + columns key,value
    + columns.comments
    + columns.types string:string
    +#### A masked pattern was here ####
    + name default.list_bucketing_dynamic_part
    + partition_columns ds/hr
    + partition_columns.types string:string
    + serialization.ddl struct list_bucketing_dynamic_part { string key, string value}
    + serialization.format 1
    + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
    +#### A masked pattern was here ####
    + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
    + name: default.list_bucketing_dynamic_part
    + TotalFiles: 1
    + GatherStats: true
    + MultiFileSpray: false
    + Path -> Alias:
    +#### A masked pattern was here ####
    + Path -> Partition:
    +#### A masked pattern was here ####
    + Partition
    + base file name: hr=11
    + input format: org.apache.hadoop.mapred.TextInputFormat
    + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
    + partition values:
    + ds 2008-04-08
    + hr 11
    + properties:
    + COLUMN_STATS_ACCURATE true
    + bucket_count -1
    + columns key,value
    + columns.comments 'default','default'
    + columns.types string:string
    +#### A masked pattern was here ####
    + name default.srcpart
    + numFiles 1
    + numRows 500
    + partition_columns ds/hr
    + partition_columns.types string:string
    + rawDataSize 5312
    + serialization.ddl struct srcpart { string key, string value}
    + serialization.format 1
    + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
    + totalSize 5812
    +#### A masked pattern was here ####
    + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
    +
    + input format: org.apache.hadoop.mapred.TextInputFormat
    + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
    + properties:
    + bucket_count -1
    + columns key,value
    + columns.comments 'default','default'
    + columns.types string:string
    +#### A masked pattern was here ####
    + name default.srcpart
    + partition_columns ds/hr
    + partition_columns.types string:string
    + serialization.ddl struct srcpart { string key, string value}
    + serialization.format 1
    + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
    +#### A masked pattern was here ####
    + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
    + name: default.srcpart
    + name: default.srcpart
    +#### A masked pattern was here ####
    + Partition
    + base file name: hr=12
    + input format: org.apache.hadoop.mapred.TextInputFormat
    + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
    + partition values:
    + ds 2008-04-08
    + hr 12
    + properties:
    + COLUMN_STATS_ACCURATE true
    + bucket_count -1
    + columns key,value
    + columns.comments 'default','default'
    + columns.types string:string
    +#### A masked pattern was here ####
    + name default.srcpart
    + numFiles 1
    + numRows 500
    + partition_columns ds/hr
    + partition_columns.types string:string
    + rawDataSize 5312
    + serialization.ddl struct srcpart { string key, string value}
    + serialization.format 1
    + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
    + totalSize 5812
    +#### A masked pattern was here ####
    + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
    +
    + input format: org.apache.hadoop.mapred.TextInputFormat
    + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
    + properties:
    + bucket_count -1
    + columns key,value
    + columns.comments 'default','default'
    + columns.types string:string
    +#### A masked pattern was here ####
    + name default.srcpart
    + partition_columns ds/hr
    + partition_columns.types string:string
    + serialization.ddl struct srcpart { string key, string value}
    + serialization.format 1
    + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
    +#### A masked pattern was here ####
    + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
    + name: default.srcpart
    + name: default.srcpart
    + Truncated Path -> Alias:
    + /srcpart/ds=2008-04-08/hr=11 [$hdt$_0:srcpart]
    + /srcpart/ds=2008-04-08/hr=12 [$hdt$_0:srcpart]
    +
    + Stage: Stage-0
    + Move Operator
    + tables:
    + partition:
    + ds 2008-04-08
    + hr
    + replace: true
    +#### A masked pattern was here ####
    + table:
    + input format: org.apache.hadoop.mapred.TextInputFormat
    + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
    + properties:
    + bucket_count -1
    + columns key,value
    + columns.comments
    + columns.types string:string
    +#### A masked pattern was here ####
    + name default.list_bucketing_dynamic_part
    + partition_columns ds/hr
    + partition_columns.types string:string
    + serialization.ddl struct list_bucketing_dynamic_part { string key, string value}
    + serialization.format 1
    + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
    +#### A masked pattern was here ####
    + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
    + name: default.list_bucketing_dynamic_part
    +
    + Stage: Stage-2
    + Stats-Aggr Operator
    +#### A masked pattern was here ####
    +
    +PREHOOK: query: insert overwrite table list_bucketing_dynamic_part partition (ds='2008-04-08', hr) select key, value, hr from srcpart where ds='2008-04-08'
    +PREHOOK: type: QUERY
    +PREHOOK: Input: default@srcpart
    +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11
    +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12
    +PREHOOK: Output: default@list_bucketing_dynamic_part@ds=2008-04-08
    +POSTHOOK: query: insert overwrite table list_bucketing_dynamic_part partition (ds='2008-04-08', hr) select key, value, hr from srcpart where ds='2008-04-08'
    +POSTHOOK: type: QUERY
    +POSTHOOK: Input: default@srcpart
    +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11
    +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12
    +POSTHOOK: Output: default@list_bucketing_dynamic_part@ds=2008-04-08/hr=11
    +POSTHOOK: Output: default@list_bucketing_dynamic_part@ds=2008-04-08/hr=12
    +POSTHOOK: Lineage: list_bucketing_dynamic_part PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ]
    +POSTHOOK: Lineage: list_bucketing_dynamic_part PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ]
    +POSTHOOK: Lineage: list_bucketing_dynamic_part PARTITION(ds=2008-04-08,hr=12).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ]
    +POSTHOOK: Lineage: list_bucketing_dynamic_part PARTITION(ds=2008-04-08,hr=12).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ]
    +PREHOOK: query: -- check DML result
    +desc formatted list_bucketing_dynamic_part partition (ds='2008-04-08', hr='11')
    +PREHOOK: type: DESCTABLE
    +PREHOOK: Input: default@list_bucketing_dynamic_part
    +POSTHOOK: query: -- check DML result
    +desc formatted list_bucketing_dynamic_part partition (ds='2008-04-08', hr='11')
    +POSTHOOK: type: DESCTABLE
    +POSTHOOK: Input: default@list_bucketing_dynamic_part
    +# col_name data_type comment
    +
    +key string
    +value string
    +
    +# Partition Information
    +# col_name data_type comment
    +
    +ds string
    +hr string
    +
    +# Detailed Partition Information
    +Partition Value: [2008-04-08, 11]
    +Database: default
    +Table: list_bucketing_dynamic_part
    +#### A masked pattern was here ####
    +Protect Mode: None
    +#### A masked pattern was here ####
    +Partition Parameters:
    + COLUMN_STATS_ACCURATE true
    + numFiles 3
    + numRows 500
    + rawDataSize 5312
    + totalSize 5812
    +#### A masked pattern was here ####
    +
    +# Storage Information
    +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
    +InputFormat: org.apache.hadoop.mapred.TextInputFormat
    +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
    +Compressed: No
    +Num Buckets: -1
    +Bucket Columns: []
    +Sort Columns: []
    +Stored As SubDirectories: Yes
    +Skewed Columns: [key, value]
    +Skewed Values: [[484, val_484], [51, val_14], [103, val_103]]
    +#### A masked pattern was here ####
    +Skewed Value to Truncated Path: {[484, val_484]=/list_bucketing_dynamic_part/ds=2008-04-08/hr=11/key=484/value=val_484, [103, val_103]=/list_bucketing_dynamic_part/ds=2008-04-08/hr=11/key=103/value=val_103}
    +Storage Desc Params:
    + serialization.format 1
    +PREHOOK: query: desc formatted list_bucketing_dynamic_part partition (ds='2008-04-08', hr='12')
    +PREHOOK: type: DESCTABLE
    +PREHOOK: Input: default@list_bucketing_dynamic_part
    +POSTHOOK: query: desc formatted list_bucketing_dynamic_part partition (ds='2008-04-08', hr='12')
    +POSTHOOK: type: DESCTABLE
    +POSTHOOK: Input: default@list_bucketing_dynamic_part
    +# col_name data_type comment
    +
    +key string
    +value string
    +
    +# Partition Information
    +# col_name data_type comment
    +
    +ds string
    +hr string
    +
    +# Detailed Partition Information
    +Partition Value: [2008-04-08, 12]
    +Database: default
    +Table: list_bucketing_dynamic_part
    +#### A masked pattern was here ####
    +Protect Mode: None
    +#### A masked pattern was here ####
    +Partition Parameters:
    + COLUMN_STATS_ACCURATE true
    + numFiles 3
    + numRows 500
    + rawDataSize 5312
    + totalSize 5812
    +#### A masked pattern was here ####
    +
    +# Storage Information
    +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
    +InputFormat: org.apache.hadoop.mapred.TextInputFormat
    +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
    +Compressed: No
    +Num Buckets: -1
    +Bucket Columns: []
    +Sort Columns: []
    +Stored As SubDirectories: Yes
    +Skewed Columns: [key, value]
    +Skewed Values: [[484, val_484], [51, val_14], [103, val_103]]
    +#### A masked pattern was here ####
    +Skewed Value to Truncated Path: {[484, val_484]=/list_bucketing_dynamic_part/ds=2008-04-08/hr=12/key=484/value=val_484, [103, val_103]=/list_bucketing_dynamic_part/ds=2008-04-08/hr=12/key=103/value=val_103}
    +Storage Desc Params:
    + serialization.format 1
    +PREHOOK: query: select count(1) from srcpart where ds='2008-04-08'
    +PREHOOK: type: QUERY
    +PREHOOK: Input: default@srcpart
    +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11
    +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12
    +#### A masked pattern was here ####
    +POSTHOOK: query: select count(1) from srcpart where ds='2008-04-08'
    +POSTHOOK: type: QUERY
    +POSTHOOK: Input: default@srcpart
    +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11
    +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12
    +#### A masked pattern was here ####
    +1000
    +PREHOOK: query: select count(1) from list_bucketing_dynamic_part where ds='2008-04-08'
    +PREHOOK: type: QUERY
    +PREHOOK: Input: default@list_bucketing_dynamic_part
    +PREHOOK: Input: default@list_bucketing_dynamic_part@ds=2008-04-08/hr=11
    +PREHOOK: Input: default@list_bucketing_dynamic_part@ds=2008-04-08/hr=12
    +#### A masked pattern was here ####
    +POSTHOOK: query: select count(1) from list_bucketing_dynamic_part where ds='2008-04-08'
    +POSTHOOK: type: QUERY
    +POSTHOOK: Input: default@list_bucketing_dynamic_part
    +POSTHOOK: Input: default@list_bucketing_dynamic_part@ds=2008-04-08/hr=11
    +POSTHOOK: Input: default@list_bucketing_dynamic_part@ds=2008-04-08/hr=12
    +#### A masked pattern was here ####
    +1000
    +PREHOOK: query: select key, value from srcpart where ds='2008-04-08' and key = "103" and value ="val_103"
    +PREHOOK: type: QUERY
    +PREHOOK: Input: default@srcpart
    +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11
    +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12
    +#### A masked pattern was here ####
    +POSTHOOK: query: select key, value from srcpart where ds='2008-04-08' and key = "103" and value ="val_103"
    +POSTHOOK: type: QUERY
    +POSTHOOK: Input: default@srcpart
    +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11
    +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12
    +#### A masked pattern was here ####
    +103 val_103
    +103 val_103
    +103 val_103
    +103 val_103
    +PREHOOK: query: explain extended
    +select key, value, ds, hr from list_bucketing_dynamic_part where ds='2008-04-08' and key = "103" and value ="val_103"
    +PREHOOK: type: QUERY
    +POSTHOOK: query: explain extended
    +select key, value, ds, hr from list_bucketing_dynamic_part where ds='2008-04-08' and key = "103" and value ="val_103"
    +POSTHOOK: type: QUERY
    +ABSTRACT SYNTAX TREE:
    +
    +TOK_QUERY
    + TOK_FROM
    + TOK_TABREF
    + TOK_TABNAME
    + list_bucketing_dynamic_part
    + TOK_INSERT
    + TOK_DESTINATION
    + TOK_DIR
    + TOK_TMP_FILE
    + TOK_SELECT
    + TOK_SELEXPR
    + TOK_TABLE_OR_COL
    + key
    + TOK_SELEXPR
    + TOK_TABLE_OR_COL
    + value
    + TOK_SELEXPR
    + TOK_TABLE_OR_COL
    + ds
    + TOK_SELEXPR
    + TOK_TABLE_OR_COL
    + hr
    + TOK_WHERE
    + and
    + and
    + =
    + TOK_TABLE_OR_COL
    + ds
    + '2008-04-08'
    + =
    + TOK_TABLE_OR_COL
    + key
    + "103"
    + =
    + TOK_TABLE_OR_COL
    + value
    + "val_103"
    +
    +
    +STAGE DEPENDENCIES:
    + Stage-1 is a root stage
    + Stage-0 depends on stages: Stage-1
    +
    +STAGE PLANS:
    + Stage: Stage-1
    + Map Reduce
    + Map Operator Tree:
    + TableScan
    + alias: list_bucketing_dynamic_part
    + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE
    + GatherStats: false
    + Filter Operator
    + isSamplingPred: false
    + predicate: ((key = '103') and (value = 'val_103')) (type: boolean)
    + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
    + Select Operator
    + expressions: '103' (type: string), 'val_103' (type: string), '2008-04-08' (type: string), hr (type: string)
    + outputColumnNames: _col0, _col1, _col2, _col3
    + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
    + File Output Operator
    + compressed: false
    + GlobalTableId: 0
    +#### A masked pattern was here ####
    + NumFilesPerFileSink: 1
    + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
    +#### A masked pattern was here ####
    + table:
    + input format: org.apache.hadoop.mapred.TextInputFormat
    + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
    + properties:
    + columns _col0,_col1,_col2,_col3
    + columns.types string:string:string:string
    + escape.delim \
    + hive.serialization.extend.nesting.levels true
    + serialization.format 1
    + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
    + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
    + TotalFiles: 1
    + GatherStats: false
    + MultiFileSpray: false
    + Path -> Alias:
    +#### A masked pattern was here ####
    + Path -> Partition:
    +#### A masked pattern was here ####
    + Partition
    + base file name: value=val_103
    + input format: org.apache.hadoop.mapred.TextInputFormat
    + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
    + partition values:
    + ds 2008-04-08
    + hr 11
    + properties:
    + COLUMN_STATS_ACCURATE true
    + bucket_count -1
    + columns key,value
    + columns.comments
    + columns.types string:string
    +#### A masked pattern was here ####
    + name default.list_bucketing_dynamic_part
    + numFiles 3
    + numRows 500
    + partition_columns ds/hr
    + partition_columns.types string:string
    + rawDataSize 5312
    + serialization.ddl struct list_bucketing_dynamic_part { string key, string value}
    + serialization.format 1
    + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
    + totalSize 5812
    +#### A masked pattern was here ####
    + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
    +
    + input format: org.apache.hadoop.mapred.TextInputFormat
    + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
    + properties:
    + bucket_count -1
    + columns key,value
    + columns.comments
    + columns.types string:string
    +#### A masked pattern was here ####
    + name default.list_bucketing_dynamic_part
    + partition_columns ds/hr
    + partition_columns.types string:string
    + serialization.ddl struct list_bucketing_dynamic_part { string key, string value}
    + serialization.format 1
    + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
    +#### A masked pattern was here ####
    + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
    + name: default.list_bucketing_dynamic_part
    + name: default.list_bucketing_dynamic_part
    +#### A masked pattern was here ####
    + Partition
    + base file name: value=val_103
    + input format: org.apache.hadoop.mapred.TextInputFormat
    + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
    + partition values:
    + ds 2008-04-08
    + hr 12
    + properties:
    + COLUMN_STATS_ACCURATE true
    + bucket_count -1
    + columns key,value
    + columns.comments
    + columns.types string:string
    +#### A masked pattern was here ####
    + name default.list_bucketing_dynamic_part
    + numFiles 3
    + numRows 500
    + partition_columns ds/hr
    + partition_columns.types string:string
    + rawDataSize 5312
    + serialization.ddl struct list_bucketing_dynamic_part { string key, string value}
    + serialization.format 1
    + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
    + totalSize 5812
    +#### A masked pattern was here ####
    + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
    +
    + input format: org.apache.hadoop.mapred.TextInputFormat
    + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
    + properties:
    + bucket_count -1
    + columns key,value
    + columns.comments
    + columns.types string:string
    +#### A masked pattern was here ####
    + name default.list_bucketing_dynamic_part
    + partition_columns ds/hr
    + partition_columns.types string:string
    + serialization.ddl struct list_bucketing_dynamic_part { string key, string value}
    + serialization.format 1
    + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
    +#### A masked pattern was here ####
    + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
    + name: default.list_bucketing_dynamic_part
    + name: default.list_bucketing_dynamic_part
    + Truncated Path -> Alias:
    + /list_bucketing_dynamic_part/ds=2008-04-08/hr=11/key=103/value=val_103 [list_bucketing_dynamic_part]
    + /list_bucketing_dynamic_part/ds=2008-04-08/hr=12/key=103/value=val_103 [list_bucketing_dynamic_part]
    +
    + Stage: Stage-0
    + Fetch Operator
    + limit: -1
    + Processor Tree:
    + ListSink
    +
    +PREHOOK: query: select key, value, ds, hr from list_bucketing_dynamic_part where ds='2008-04-08' and key = "103" and value ="val_103"
    +PREHOOK: type: QUERY
    +PREHOOK: Input: default@list_bucketing_dynamic_part
    +PREHOOK: Input: default@list_bucketing_dynamic_part@ds=2008-04-08/hr=11
    +PREHOOK: Input: default@list_bucketing_dynamic_part@ds=2008-04-08/hr=12
    +#### A masked pattern was here ####
    +POSTHOOK: query: select key, value, ds, hr from list_bucketing_dynamic_part where ds='2008-04-08' and key = "103" and value ="val_103"
    +POSTHOOK: type: QUERY
    +POSTHOOK: Input: default@list_bucketing_dynamic_part
    +POSTHOOK: Input: default@list_bucketing_dynamic_part@ds=2008-04-08/hr=11
    +POSTHOOK: Input: default@list_bucketing_dynamic_part@ds=2008-04-08/hr=12
    +#### A masked pattern was here ####
    +103 val_103 2008-04-08 11
    +103 val_103 2008-04-08 11
    +103 val_103 2008-04-08 12
    +103 val_103 2008-04-08 12
    +PREHOOK: query: -- clean up resources
    +drop table list_bucketing_dynamic_part
    +PREHOOK: type: DROPTABLE
    +PREHOOK: Input: default@list_bucketing_dynamic_part
    +PREHOOK: Output: default@list_bucketing_dynamic_part
    +POSTHOOK: query: -- clean up resources
    +drop table list_bucketing_dynamic_part
    +POSTHOOK: type: DROPTABLE
    +POSTHOOK: Input: default@list_bucketing_dynamic_part
    +POSTHOOK: Output: default@list_bucketing_dynamic_part

    Added: hive/trunk/ql/src/test/results/clientpositive/list_bucket_dml_8.q.java1.7.out
    URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/list_bucket_dml_8.q.java1.7.out?rev=1647912&view=auto
    ==============================================================================
    --- hive/trunk/ql/src/test/results/clientpositive/list_bucket_dml_8.q.java1.7.out (added)
    +++ hive/trunk/ql/src/test/results/clientpositive/list_bucket_dml_8.q.java1.7.out Thu Dec 25 18:06:30 2014
    @@ -0,0 +1,718 @@
    +PREHOOK: query: -- list bucketing alter table ... concatenate:
    +-- Use list bucketing DML to generate mutilple files in partitions by turning off merge
    +-- dynamic partition. multiple skewed columns. merge.
    +-- The following explains merge example used in this test case
    +-- DML will generated 2 partitions
    +-- ds=2008-04-08/hr=a1
    +-- ds=2008-04-08/hr=b1
    +-- without merge, each partition has more files
    +-- ds=2008-04-08/hr=a1 has 2 files
    +-- ds=2008-04-08/hr=b1 has 6 files
    +-- with merge each partition has more files
    +-- ds=2008-04-08/hr=a1 has 1 files
    +-- ds=2008-04-08/hr=b1 has 4 files
    +-- The following shows file size and name in each directory
    +-- hr=a1/HIVE_DEFAULT_LIST_BUCKETING_DIR_NAME/HIVE_DEFAULT_LIST_BUCKETING_DIR_NAME:
    +-- without merge
    +-- 155 000000_0
    +-- 155 000001_0
    +-- with merge
    +-- 254 000000_0
    +-- hr=b1/key=103/value=val_103:
    +-- without merge
    +-- 99 000000_0
    +-- 99 000001_0
    +-- with merge
    +-- 142 000001_0
    +-- hr=b1/HIVE_DEFAULT_LIST_BUCKETING_DIR_NAME/HIVE_DEFAULT_LIST_BUCKETING_DIR_NAME:
    +-- without merge
    +-- 5181 000000_0
    +-- 5181 000001_0
    +-- with merge
    +-- 5181 000000_0
    +-- 5181 000001_0
    +-- hr=b1/key=484/value=val_484
    +-- without merge
    +-- 87 000000_0
    +-- 87 000001_0
    +-- with merge
    +-- 118 000002_0
    +
    +-- INCLUDE_HADOOP_MAJOR_VERSIONS(0.23)
    +-- JAVA_VERSION_SPECIFIC_OUTPUT
    +
    +-- create a skewed table
    +create table list_bucketing_dynamic_part (key String, value String)
    + partitioned by (ds String, hr String)
    + skewed by (key, value) on (('484','val_484'),('51','val_14'),('103','val_103'))
    + stored as DIRECTORIES
    + STORED AS RCFILE
    +PREHOOK: type: CREATETABLE
    +PREHOOK: Output: database:default
    +PREHOOK: Output: default@list_bucketing_dynamic_part
    +POSTHOOK: query: -- list bucketing alter table ... concatenate:
    +-- Use list bucketing DML to generate mutilple files in partitions by turning off merge
    +-- dynamic partition. multiple skewed columns. merge.
    +-- The following explains merge example used in this test case
    +-- DML will generated 2 partitions
    +-- ds=2008-04-08/hr=a1
    +-- ds=2008-04-08/hr=b1
    +-- without merge, each partition has more files
    +-- ds=2008-04-08/hr=a1 has 2 files
    +-- ds=2008-04-08/hr=b1 has 6 files
    +-- with merge each partition has more files
    +-- ds=2008-04-08/hr=a1 has 1 files
    +-- ds=2008-04-08/hr=b1 has 4 files
    +-- The following shows file size and name in each directory
    +-- hr=a1/HIVE_DEFAULT_LIST_BUCKETING_DIR_NAME/HIVE_DEFAULT_LIST_BUCKETING_DIR_NAME:
    +-- without merge
    +-- 155 000000_0
    +-- 155 000001_0
    +-- with merge
    +-- 254 000000_0
    +-- hr=b1/key=103/value=val_103:
    +-- without merge
    +-- 99 000000_0
    +-- 99 000001_0
    +-- with merge
    +-- 142 000001_0
    +-- hr=b1/HIVE_DEFAULT_LIST_BUCKETING_DIR_NAME/HIVE_DEFAULT_LIST_BUCKETING_DIR_NAME:
    +-- without merge
    +-- 5181 000000_0
    +-- 5181 000001_0
    +-- with merge
    +-- 5181 000000_0
    +-- 5181 000001_0
    +-- hr=b1/key=484/value=val_484
    +-- without merge
    +-- 87 000000_0
    +-- 87 000001_0
    +-- with merge
    +-- 118 000002_0
    +
    +-- INCLUDE_HADOOP_MAJOR_VERSIONS(0.23)
    +-- JAVA_VERSION_SPECIFIC_OUTPUT
    +
    +-- create a skewed table
    +create table list_bucketing_dynamic_part (key String, value String)
    + partitioned by (ds String, hr String)
    + skewed by (key, value) on (('484','val_484'),('51','val_14'),('103','val_103'))
    + stored as DIRECTORIES
    + STORED AS RCFILE
    +POSTHOOK: type: CREATETABLE
    +POSTHOOK: Output: database:default
    +POSTHOOK: Output: default@list_bucketing_dynamic_part
    +PREHOOK: query: -- list bucketing DML without merge. use bucketize to generate a few small files.
    +explain extended
    +insert overwrite table list_bucketing_dynamic_part partition (ds = '2008-04-08', hr)
    +select key, value, if(key % 100 == 0, 'a1', 'b1') from srcpart where ds = '2008-04-08'
    +PREHOOK: type: QUERY
    +POSTHOOK: query: -- list bucketing DML without merge. use bucketize to generate a few small files.
    +explain extended
    +insert overwrite table list_bucketing_dynamic_part partition (ds = '2008-04-08', hr)
    +select key, value, if(key % 100 == 0, 'a1', 'b1') from srcpart where ds = '2008-04-08'
    +POSTHOOK: type: QUERY
    +ABSTRACT SYNTAX TREE:
    +
    +TOK_QUERY
    + TOK_FROM
    + TOK_TABREF
    + TOK_TABNAME
    + srcpart
    + TOK_INSERT
    + TOK_DESTINATION
    + TOK_TAB
    + TOK_TABNAME
    + list_bucketing_dynamic_part
    + TOK_PARTSPEC
    + TOK_PARTVAL
    + ds
    + '2008-04-08'
    + TOK_PARTVAL
    + hr
    + TOK_SELECT
    + TOK_SELEXPR
    + TOK_TABLE_OR_COL
    + key
    + TOK_SELEXPR
    + TOK_TABLE_OR_COL
    + value
    + TOK_SELEXPR
    + TOK_FUNCTION
    + if
    + ==
    + %
    + TOK_TABLE_OR_COL
    + key
    + 100
    + 0
    + 'a1'
    + 'b1'
    + TOK_WHERE
    + =
    + TOK_TABLE_OR_COL
    + ds
    + '2008-04-08'
    +
    +
    +STAGE DEPENDENCIES:
    + Stage-1 is a root stage
    + Stage-0 depends on stages: Stage-1
    + Stage-2 depends on stages: Stage-0
    +
    +STAGE PLANS:
    + Stage: Stage-1
    + Map Reduce
    + Map Operator Tree:
    + TableScan
    + alias: srcpart
    + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE
    + GatherStats: false
    + Select Operator
    + expressions: key (type: string), value (type: string), if(((UDFToDouble(key) % 100.0) = 0.0), 'a1', 'b1') (type: string)
    + outputColumnNames: _col0, _col1, _col2
    + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE
    + File Output Operator
    + compressed: false
    + GlobalTableId: 1
    +#### A masked pattern was here ####
    + NumFilesPerFileSink: 1
    + Static Partition Specification: ds=2008-04-08/
    + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE
    +#### A masked pattern was here ####
    + table:
    + input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat
    + output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat
    + properties:
    + bucket_count -1
    + columns key,value
    + columns.comments
    + columns.types string:string
    +#### A masked pattern was here ####
    + name default.list_bucketing_dynamic_part
    + partition_columns ds/hr
    + partition_columns.types string:string
    + serialization.ddl struct list_bucketing_dynamic_part { string key, string value}
    + serialization.format 1
    + serialization.lib org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
    +#### A masked pattern was here ####
    + serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
    + name: default.list_bucketing_dynamic_part
    + TotalFiles: 1
    + GatherStats: true
    + MultiFileSpray: false
    + Path -> Alias:
    +#### A masked pattern was here ####
    + Path -> Partition:
    +#### A masked pattern was here ####
    + Partition
    + base file name: hr=11
    + input format: org.apache.hadoop.mapred.TextInputFormat
    + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
    + partition values:
    + ds 2008-04-08
    + hr 11
    + properties:
    + COLUMN_STATS_ACCURATE true
    + bucket_count -1
    + columns key,value
    + columns.comments 'default','default'
    + columns.types string:string
    +#### A masked pattern was here ####
    + name default.srcpart
    + numFiles 1
    + numRows 500
    + partition_columns ds/hr
    + partition_columns.types string:string
    + rawDataSize 5312
    + serialization.ddl struct srcpart { string key, string value}
    + serialization.format 1
    + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
    + totalSize 5812
    +#### A masked pattern was here ####
    + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
    +
    + input format: org.apache.hadoop.mapred.TextInputFormat
    + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
    + properties:
    + bucket_count -1
    + columns key,value
    + columns.comments 'default','default'
    + columns.types string:string
    +#### A masked pattern was here ####
    + name default.srcpart
    + partition_columns ds/hr
    + partition_columns.types string:string
    + serialization.ddl struct srcpart { string key, string value}
    + serialization.format 1
    + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
    +#### A masked pattern was here ####
    + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
    + name: default.srcpart
    + name: default.srcpart
    +#### A masked pattern was here ####
    + Partition
    + base file name: hr=12
    + input format: org.apache.hadoop.mapred.TextInputFormat
    + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
    + partition values:
    + ds 2008-04-08
    + hr 12
    + properties:
    + COLUMN_STATS_ACCURATE true
    + bucket_count -1
    + columns key,value
    + columns.comments 'default','default'
    + columns.types string:string
    +#### A masked pattern was here ####
    + name default.srcpart
    + numFiles 1
    + numRows 500
    + partition_columns ds/hr
    + partition_columns.types string:string
    + rawDataSize 5312
    + serialization.ddl struct srcpart { string key, string value}
    + serialization.format 1
    + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
    + totalSize 5812
    +#### A masked pattern was here ####
    + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
    +
    + input format: org.apache.hadoop.mapred.TextInputFormat
    + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
    + properties:
    + bucket_count -1
    + columns key,value
    + columns.comments 'default','default'
    + columns.types string:string
    +#### A masked pattern was here ####
    + name default.srcpart
    + partition_columns ds/hr
    + partition_columns.types string:string
    + serialization.ddl struct srcpart { string key, string value}
    + serialization.format 1
    + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
    +#### A masked pattern was here ####
    + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
    + name: default.srcpart
    + name: default.srcpart
    + Truncated Path -> Alias:
    + /srcpart/ds=2008-04-08/hr=11 [$hdt$_0:srcpart]
    + /srcpart/ds=2008-04-08/hr=12 [$hdt$_0:srcpart]
    +
    + Stage: Stage-0
    + Move Operator
    + tables:
    + partition:
    + ds 2008-04-08
    + hr
    + replace: true
    +#### A masked pattern was here ####
    + table:
    + input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat
    + output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat
    + properties:
    + bucket_count -1
    + columns key,value
    + columns.comments
    + columns.types string:string
    +#### A masked pattern was here ####
    + name default.list_bucketing_dynamic_part
    + partition_columns ds/hr
    + partition_columns.types string:string
    + serialization.ddl struct list_bucketing_dynamic_part { string key, string value}
    + serialization.format 1
    + serialization.lib org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
    +#### A masked pattern was here ####
    + serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
    + name: default.list_bucketing_dynamic_part
    +
    + Stage: Stage-2
    + Stats-Aggr Operator
    +#### A masked pattern was here ####
    +
    +PREHOOK: query: insert overwrite table list_bucketing_dynamic_part partition (ds = '2008-04-08', hr)
    +select key, value, if(key % 100 == 0, 'a1', 'b1') from srcpart where ds = '2008-04-08'
    +PREHOOK: type: QUERY
    +PREHOOK: Input: default@srcpart
    +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11
    +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12
    +PREHOOK: Output: default@list_bucketing_dynamic_part@ds=2008-04-08
    +POSTHOOK: query: insert overwrite table list_bucketing_dynamic_part partition (ds = '2008-04-08', hr)
    +select key, value, if(key % 100 == 0, 'a1', 'b1') from srcpart where ds = '2008-04-08'
    +POSTHOOK: type: QUERY
    +POSTHOOK: Input: default@srcpart
    +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11
    +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12
    +POSTHOOK: Output: default@list_bucketing_dynamic_part@ds=2008-04-08/hr=a1
    +POSTHOOK: Output: default@list_bucketing_dynamic_part@ds=2008-04-08/hr=b1
    +POSTHOOK: Lineage: list_bucketing_dynamic_part PARTITION(ds=2008-04-08,hr=a1).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ]
    +POSTHOOK: Lineage: list_bucketing_dynamic_part PARTITION(ds=2008-04-08,hr=a1).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ]
    +POSTHOOK: Lineage: list_bucketing_dynamic_part PARTITION(ds=2008-04-08,hr=b1).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ]
    +POSTHOOK: Lineage: list_bucketing_dynamic_part PARTITION(ds=2008-04-08,hr=b1).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ]
    +PREHOOK: query: -- check DML result
    +show partitions list_bucketing_dynamic_part
    +PREHOOK: type: SHOWPARTITIONS
    +PREHOOK: Input: default@list_bucketing_dynamic_part
    +POSTHOOK: query: -- check DML result
    +show partitions list_bucketing_dynamic_part
    +POSTHOOK: type: SHOWPARTITIONS
    +POSTHOOK: Input: default@list_bucketing_dynamic_part
    +ds=2008-04-08/hr=a1
    +ds=2008-04-08/hr=b1
    +PREHOOK: query: desc formatted list_bucketing_dynamic_part partition (ds='2008-04-08', hr='a1')
    +PREHOOK: type: DESCTABLE
    +PREHOOK: Input: default@list_bucketing_dynamic_part
    +POSTHOOK: query: desc formatted list_bucketing_dynamic_part partition (ds='2008-04-08', hr='a1')
    +POSTHOOK: type: DESCTABLE
    +POSTHOOK: Input: default@list_bucketing_dynamic_part
    +# col_name data_type comment
    +
    +key string
    +value string
    +
    +# Partition Information
    +# col_name data_type comment
    +
    +ds string
    +hr string
    +
    +# Detailed Partition Information
    +Partition Value: [2008-04-08, a1]
    +Database: default
    +Table: list_bucketing_dynamic_part
    +#### A masked pattern was here ####
    +Protect Mode: None
    +#### A masked pattern was here ####
    +Partition Parameters:
    + COLUMN_STATS_ACCURATE true
    + numFiles 2
    + numRows 16
    + rawDataSize 136
    + totalSize 310
    +#### A masked pattern was here ####
    +
    +# Storage Information
    +SerDe Library: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
    +InputFormat: org.apache.hadoop.hive.ql.io.RCFileInputFormat
    +OutputFormat: org.apache.hadoop.hive.ql.io.RCFileOutputFormat
    +Compressed: No
    +Num Buckets: -1
    +Bucket Columns: []
    +Sort Columns: []
    +Stored As SubDirectories: Yes
    +Skewed Columns: [key, value]
    +Skewed Values: [[484, val_484], [51, val_14], [103, val_103]]
    +Storage Desc Params:
    + serialization.format 1
    +PREHOOK: query: desc formatted list_bucketing_dynamic_part partition (ds='2008-04-08', hr='b1')
    +PREHOOK: type: DESCTABLE
    +PREHOOK: Input: default@list_bucketing_dynamic_part
    +POSTHOOK: query: desc formatted list_bucketing_dynamic_part partition (ds='2008-04-08', hr='b1')
    +POSTHOOK: type: DESCTABLE
    +POSTHOOK: Input: default@list_bucketing_dynamic_part
    +# col_name data_type comment
    +
    +key string
    +value string
    +
    +# Partition Information
    +# col_name data_type comment
    +
    +ds string
    +hr string
    +
    +# Detailed Partition Information
    +Partition Value: [2008-04-08, b1]
    +Database: default
    +Table: list_bucketing_dynamic_part
    +#### A masked pattern was here ####
    +Protect Mode: None
    +#### A masked pattern was here ####
    +Partition Parameters:
    + COLUMN_STATS_ACCURATE true
    + numFiles 6
    + numRows 984
    + rawDataSize 9488
    + totalSize 10734
    +#### A masked pattern was here ####
    +
    +# Storage Information
    +SerDe Library: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
    +InputFormat: org.apache.hadoop.hive.ql.io.RCFileInputFormat
    +OutputFormat: org.apache.hadoop.hive.ql.io.RCFileOutputFormat
    +Compressed: No
    +Num Buckets: -1
    +Bucket Columns: []
    +Sort Columns: []
    +Stored As SubDirectories: Yes
    +Skewed Columns: [key, value]
    +Skewed Values: [[484, val_484], [51, val_14], [103, val_103]]
    +#### A masked pattern was here ####
    +Skewed Value to Truncated Path: {[103, val_103]=/list_bucketing_dynamic_part/ds=2008-04-08/hr=b1/key=103/value=val_103, [484, val_484]=/list_bucketing_dynamic_part/ds=2008-04-08/hr=b1/key=484/value=val_484}
    +Storage Desc Params:
    + serialization.format 1
    +PREHOOK: query: -- concatenate the partition and it will merge files
    +alter table list_bucketing_dynamic_part partition (ds='2008-04-08', hr='b1') concatenate
    +PREHOOK: type: ALTER_PARTITION_MERGE
    +PREHOOK: Input: default@list_bucketing_dynamic_part
    +PREHOOK: Output: default@list_bucketing_dynamic_part@ds=2008-04-08/hr=b1
    +POSTHOOK: query: -- concatenate the partition and it will merge files
    +alter table list_bucketing_dynamic_part partition (ds='2008-04-08', hr='b1') concatenate
    +POSTHOOK: type: ALTER_PARTITION_MERGE
    +POSTHOOK: Input: default@list_bucketing_dynamic_part
    +POSTHOOK: Output: default@list_bucketing_dynamic_part@ds=2008-04-08/hr=b1
    +PREHOOK: query: desc formatted list_bucketing_dynamic_part partition (ds='2008-04-08', hr='b1')
    +PREHOOK: type: DESCTABLE
    +PREHOOK: Input: default@list_bucketing_dynamic_part
    +POSTHOOK: query: desc formatted list_bucketing_dynamic_part partition (ds='2008-04-08', hr='b1')
    +POSTHOOK: type: DESCTABLE
    +POSTHOOK: Input: default@list_bucketing_dynamic_part
    +# col_name data_type comment
    +
    +key string
    +value string
    +
    +# Partition Information
    +# col_name data_type comment
    +
    +ds string
    +hr string
    +
    +# Detailed Partition Information
    +Partition Value: [2008-04-08, b1]
    +Database: default
    +Table: list_bucketing_dynamic_part
    +#### A masked pattern was here ####
    +Protect Mode: None
    +#### A masked pattern was here ####
    +Partition Parameters:
    + COLUMN_STATS_ACCURATE true
    + numFiles 3
    + numRows 0
    + rawDataSize 0
    + totalSize 10586
    +#### A masked pattern was here ####
    +
    +# Storage Information
    +SerDe Library: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
    +InputFormat: org.apache.hadoop.hive.ql.io.RCFileInputFormat
    +OutputFormat: org.apache.hadoop.hive.ql.io.RCFileOutputFormat
    +Compressed: No
    +Num Buckets: -1
    +Bucket Columns: []
    +Sort Columns: []
    +Stored As SubDirectories: Yes
    +Skewed Columns: [key, value]
    +Skewed Values: [[484, val_484], [51, val_14], [103, val_103]]
    +#### A masked pattern was here ####
    +Skewed Value to Truncated Path: {[103, val_103]=/list_bucketing_dynamic_part/ds=2008-04-08/hr=b1/key=103/value=val_103, [484, val_484]=/list_bucketing_dynamic_part/ds=2008-04-08/hr=b1/key=484/value=val_484}
    +Storage Desc Params:
    + serialization.format 1
    +PREHOOK: query: select count(1) from srcpart where ds = '2008-04-08'
    +PREHOOK: type: QUERY
    +PREHOOK: Input: default@srcpart
    +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11
    +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12
    +#### A masked pattern was here ####
    +POSTHOOK: query: select count(1) from srcpart where ds = '2008-04-08'
    +POSTHOOK: type: QUERY
    +POSTHOOK: Input: default@srcpart
    +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11
    +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12
    +#### A masked pattern was here ####
    +1000
    +PREHOOK: query: select count(*) from list_bucketing_dynamic_part
    +PREHOOK: type: QUERY
    +PREHOOK: Input: default@list_bucketing_dynamic_part
    +PREHOOK: Input: default@list_bucketing_dynamic_part@ds=2008-04-08/hr=a1
    +PREHOOK: Input: default@list_bucketing_dynamic_part@ds=2008-04-08/hr=b1
    +#### A masked pattern was here ####
    +POSTHOOK: query: select count(*) from list_bucketing_dynamic_part
    +POSTHOOK: type: QUERY
    +POSTHOOK: Input: default@list_bucketing_dynamic_part
    +POSTHOOK: Input: default@list_bucketing_dynamic_part@ds=2008-04-08/hr=a1
    +POSTHOOK: Input: default@list_bucketing_dynamic_part@ds=2008-04-08/hr=b1
    +#### A masked pattern was here ####
    +1000
    +PREHOOK: query: explain extended
    +select * from list_bucketing_dynamic_part where key = '484' and value = 'val_484'
    +PREHOOK: type: QUERY
    +POSTHOOK: query: explain extended
    +select * from list_bucketing_dynamic_part where key = '484' and value = 'val_484'
    +POSTHOOK: type: QUERY
    +ABSTRACT SYNTAX TREE:
    +
    +TOK_QUERY
    + TOK_FROM
    + TOK_TABREF
    + TOK_TABNAME
    + list_bucketing_dynamic_part
    + TOK_INSERT
    + TOK_DESTINATION
    + TOK_DIR
    + TOK_TMP_FILE
    + TOK_SELECT
    + TOK_SELEXPR
    + TOK_ALLCOLREF
    + TOK_WHERE
    + and
    + =
    + TOK_TABLE_OR_COL
    + key
    + '484'
    + =
    + TOK_TABLE_OR_COL
    + value
    + 'val_484'
    +
    +
    +STAGE DEPENDENCIES:
    + Stage-0 is a root stage
    +
    +STAGE PLANS:
    + Stage: Stage-0
    + Fetch Operator
    + limit: -1
    + Partition Description:
    + Partition
    + input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat
    + output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat
    + partition values:
    + ds 2008-04-08
    + hr a1
    + properties:
    + COLUMN_STATS_ACCURATE true
    + bucket_count -1
    + columns key,value
    + columns.comments
    + columns.types string:string
    +#### A masked pattern was here ####
    + name default.list_bucketing_dynamic_part
    + numFiles 2
    + numRows 16
    + partition_columns ds/hr
    + partition_columns.types string:string
    + rawDataSize 136
    + serialization.ddl struct list_bucketing_dynamic_part { string key, string value}
    + serialization.format 1
    + serialization.lib org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
    + totalSize 310
    +#### A masked pattern was here ####
    + serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
    +
    + input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat
    + output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat
    + properties:
    + bucket_count -1
    + columns key,value
    + columns.comments
    + columns.types string:string
    +#### A masked pattern was here ####
    + name default.list_bucketing_dynamic_part
    + partition_columns ds/hr
    + partition_columns.types string:string
    + serialization.ddl struct list_bucketing_dynamic_part { string key, string value}
    + serialization.format 1
    + serialization.lib org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
    +#### A masked pattern was here ####
    + serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
    + name: default.list_bucketing_dynamic_part
    + name: default.list_bucketing_dynamic_part
    + Partition
    + input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat
    + output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat
    + partition values:
    + ds 2008-04-08
    + hr b1
    + properties:
    + COLUMN_STATS_ACCURATE true
    + bucket_count -1
    + columns key,value
    + columns.comments
    + columns.types string:string
    +#### A masked pattern was here ####
    + name default.list_bucketing_dynamic_part
    + numFiles 3
    + numRows 0
    + partition_columns ds/hr
    + partition_columns.types string:string
    + rawDataSize 0
    + serialization.ddl struct list_bucketing_dynamic_part { string key, string value}
    + serialization.format 1
    + serialization.lib org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
    + totalSize 10586
    +#### A masked pattern was here ####
    + serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
    +
    + input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat
    + output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat
    + properties:
    + bucket_count -1
    + columns key,value
    + columns.comments
    + columns.types string:string
    +#### A masked pattern was here ####
    + name default.list_bucketing_dynamic_part
    + partition_columns ds/hr
    + partition_columns.types string:string
    + serialization.ddl struct list_bucketing_dynamic_part { string key, string value}
    + serialization.format 1
    + serialization.lib org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
    +#### A masked pattern was here ####
    + serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
    + name: default.list_bucketing_dynamic_part
    + name: default.list_bucketing_dynamic_part
    + Processor Tree:
    + TableScan
    + alias: list_bucketing_dynamic_part
    + Statistics: Num rows: 16 Data size: 136 Basic stats: PARTIAL Column stats: NONE
    + GatherStats: false
    + Filter Operator
    + isSamplingPred: false
    + predicate: ((key = '484') and (value = 'val_484')) (type: boolean)
    + Statistics: Num rows: 4 Data size: 34 Basic stats: COMPLETE Column stats: NONE
    + Select Operator
    + expressions: '484' (type: string), 'val_484' (type: string), ds (type: string), hr (type: string)
    + outputColumnNames: _col0, _col1, _col2, _col3
    + Statistics: Num rows: 4 Data size: 34 Basic stats: COMPLETE Column stats: NONE
    + ListSink
    +
    +PREHOOK: query: select * from list_bucketing_dynamic_part where key = '484' and value = 'val_484'
    +PREHOOK: type: QUERY
    +PREHOOK: Input: default@list_bucketing_dynamic_part
    +PREHOOK: Input: default@list_bucketing_dynamic_part@ds=2008-04-08/hr=a1
    +PREHOOK: Input: default@list_bucketing_dynamic_part@ds=2008-04-08/hr=b1
    +#### A masked pattern was here ####
    +POSTHOOK: query: select * from list_bucketing_dynamic_part where key = '484' and value = 'val_484'
    +POSTHOOK: type: QUERY
    +POSTHOOK: Input: default@list_bucketing_dynamic_part
    +POSTHOOK: Input: default@list_bucketing_dynamic_part@ds=2008-04-08/hr=a1
    +POSTHOOK: Input: default@list_bucketing_dynamic_part@ds=2008-04-08/hr=b1
    +#### A masked pattern was here ####
    +484 val_484 2008-04-08 b1
    +484 val_484 2008-04-08 b1
    +PREHOOK: query: select * from srcpart where ds = '2008-04-08' and key = '484' and value = 'val_484' order by hr
    +PREHOOK: type: QUERY
    +PREHOOK: Input: default@srcpart
    +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11
    +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12
    +#### A masked pattern was here ####
    +POSTHOOK: query: select * from srcpart where ds = '2008-04-08' and key = '484' and value = 'val_484' order by hr
    +POSTHOOK: type: QUERY
    +POSTHOOK: Input: default@srcpart
    +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11
    +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12
    +#### A masked pattern was here ####
    +484 val_484 2008-04-08 11
    +484 val_484 2008-04-08 12
    +PREHOOK: query: -- clean up
    +drop table list_bucketing_dynamic_part
    +PREHOOK: type: DROPTABLE
    +PREHOOK: Input: default@list_bucketing_dynamic_part
    +PREHOOK: Output: default@list_bucketing_dynamic_part
    +POSTHOOK: query: -- clean up
    +drop table list_bucketing_dynamic_part
    +POSTHOOK: type: DROPTABLE
    +POSTHOOK: Input: default@list_bucketing_dynamic_part
    +POSTHOOK: Output: default@list_bucketing_dynamic_part
  • Brock at Dec 25, 2014 at 6:06 pm
    Added: hive/trunk/ql/src/test/results/clientpositive/list_bucket_dml_4.q.java1.7.out
    URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/list_bucket_dml_4.q.java1.7.out?rev=1647912&view=auto
    ==============================================================================
    --- hive/trunk/ql/src/test/results/clientpositive/list_bucket_dml_4.q.java1.7.out (added)
    +++ hive/trunk/ql/src/test/results/clientpositive/list_bucket_dml_4.q.java1.7.out Thu Dec 25 18:06:30 2014
    @@ -0,0 +1,951 @@
    +PREHOOK: query: -- INCLUDE_HADOOP_MAJOR_VERSIONS(0.23)
    +-- SORT_QUERY_RESULTS
    +-- JAVA_VERSION_SPECIFIC_OUTPUT
    +
    +-- list bucketing DML: static partition. multiple skewed columns. merge.
    +-- ds=2008-04-08/hr=11/HIVE_DEFAULT_LIST_BUCKETING_DIR_NAME:
    +-- 5263 000000_0
    +-- 5263 000001_0
    +-- ds=2008-04-08/hr=11/key=103/value=val_103:
    +-- 99 000000_0
    +-- 99 000001_0
    +-- after merge
    +-- 142 000000_0
    +-- ds=2008-04-08/hr=11/key=484/value=val_484:
    +-- 87 000000_0
    +-- 87 000001_0
    +-- after merge
    +-- 118 000001_0
    +
    +-- create a skewed table
    +create table list_bucketing_static_part (key String, value String)
    + partitioned by (ds String, hr String)
    + skewed by (key, value) on (('484','val_484'),('51','val_14'),('103','val_103'))
    + stored as DIRECTORIES
    + STORED AS RCFILE
    +PREHOOK: type: CREATETABLE
    +PREHOOK: Output: database:default
    +PREHOOK: Output: default@list_bucketing_static_part
    +POSTHOOK: query: -- INCLUDE_HADOOP_MAJOR_VERSIONS(0.23)
    +-- SORT_QUERY_RESULTS
    +-- JAVA_VERSION_SPECIFIC_OUTPUT
    +
    +-- list bucketing DML: static partition. multiple skewed columns. merge.
    +-- ds=2008-04-08/hr=11/HIVE_DEFAULT_LIST_BUCKETING_DIR_NAME:
    +-- 5263 000000_0
    +-- 5263 000001_0
    +-- ds=2008-04-08/hr=11/key=103/value=val_103:
    +-- 99 000000_0
    +-- 99 000001_0
    +-- after merge
    +-- 142 000000_0
    +-- ds=2008-04-08/hr=11/key=484/value=val_484:
    +-- 87 000000_0
    +-- 87 000001_0
    +-- after merge
    +-- 118 000001_0
    +
    +-- create a skewed table
    +create table list_bucketing_static_part (key String, value String)
    + partitioned by (ds String, hr String)
    + skewed by (key, value) on (('484','val_484'),('51','val_14'),('103','val_103'))
    + stored as DIRECTORIES
    + STORED AS RCFILE
    +POSTHOOK: type: CREATETABLE
    +POSTHOOK: Output: database:default
    +POSTHOOK: Output: default@list_bucketing_static_part
    +PREHOOK: query: -- list bucketing DML without merge. use bucketize to generate a few small files.
    +explain extended
    +insert overwrite table list_bucketing_static_part partition (ds = '2008-04-08', hr = '11')
    +select key, value from srcpart where ds = '2008-04-08'
    +PREHOOK: type: QUERY
    +POSTHOOK: query: -- list bucketing DML without merge. use bucketize to generate a few small files.
    +explain extended
    +insert overwrite table list_bucketing_static_part partition (ds = '2008-04-08', hr = '11')
    +select key, value from srcpart where ds = '2008-04-08'
    +POSTHOOK: type: QUERY
    +ABSTRACT SYNTAX TREE:
    +
    +TOK_QUERY
    + TOK_FROM
    + TOK_TABREF
    + TOK_TABNAME
    + srcpart
    + TOK_INSERT
    + TOK_DESTINATION
    + TOK_TAB
    + TOK_TABNAME
    + list_bucketing_static_part
    + TOK_PARTSPEC
    + TOK_PARTVAL
    + ds
    + '2008-04-08'
    + TOK_PARTVAL
    + hr
    + '11'
    + TOK_SELECT
    + TOK_SELEXPR
    + TOK_TABLE_OR_COL
    + key
    + TOK_SELEXPR
    + TOK_TABLE_OR_COL
    + value
    + TOK_WHERE
    + =
    + TOK_TABLE_OR_COL
    + ds
    + '2008-04-08'
    +
    +
    +STAGE DEPENDENCIES:
    + Stage-1 is a root stage
    + Stage-0 depends on stages: Stage-1
    + Stage-2 depends on stages: Stage-0
    +
    +STAGE PLANS:
    + Stage: Stage-1
    + Map Reduce
    + Map Operator Tree:
    + TableScan
    + alias: srcpart
    + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE
    + GatherStats: false
    + Select Operator
    + expressions: key (type: string), value (type: string)
    + outputColumnNames: _col0, _col1
    + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE
    + File Output Operator
    + compressed: false
    + GlobalTableId: 1
    +#### A masked pattern was here ####
    + NumFilesPerFileSink: 1
    + Static Partition Specification: ds=2008-04-08/hr=11/
    + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE
    +#### A masked pattern was here ####
    + table:
    + input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat
    + output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat
    + properties:
    + bucket_count -1
    + columns key,value
    + columns.comments
    + columns.types string:string
    +#### A masked pattern was here ####
    + name default.list_bucketing_static_part
    + partition_columns ds/hr
    + partition_columns.types string:string
    + serialization.ddl struct list_bucketing_static_part { string key, string value}
    + serialization.format 1
    + serialization.lib org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
    +#### A masked pattern was here ####
    + serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
    + name: default.list_bucketing_static_part
    + TotalFiles: 1
    + GatherStats: true
    + MultiFileSpray: false
    + Path -> Alias:
    +#### A masked pattern was here ####
    + Path -> Partition:
    +#### A masked pattern was here ####
    + Partition
    + base file name: hr=11
    + input format: org.apache.hadoop.mapred.TextInputFormat
    + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
    + partition values:
    + ds 2008-04-08
    + hr 11
    + properties:
    + COLUMN_STATS_ACCURATE true
    + bucket_count -1
    + columns key,value
    + columns.comments 'default','default'
    + columns.types string:string
    +#### A masked pattern was here ####
    + name default.srcpart
    + numFiles 1
    + numRows 500
    + partition_columns ds/hr
    + partition_columns.types string:string
    + rawDataSize 5312
    + serialization.ddl struct srcpart { string key, string value}
    + serialization.format 1
    + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
    + totalSize 5812
    +#### A masked pattern was here ####
    + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
    +
    + input format: org.apache.hadoop.mapred.TextInputFormat
    + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
    + properties:
    + bucket_count -1
    + columns key,value
    + columns.comments 'default','default'
    + columns.types string:string
    +#### A masked pattern was here ####
    + name default.srcpart
    + partition_columns ds/hr
    + partition_columns.types string:string
    + serialization.ddl struct srcpart { string key, string value}
    + serialization.format 1
    + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
    +#### A masked pattern was here ####
    + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
    + name: default.srcpart
    + name: default.srcpart
    +#### A masked pattern was here ####
    + Partition
    + base file name: hr=12
    + input format: org.apache.hadoop.mapred.TextInputFormat
    + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
    + partition values:
    + ds 2008-04-08
    + hr 12
    + properties:
    + COLUMN_STATS_ACCURATE true
    + bucket_count -1
    + columns key,value
    + columns.comments 'default','default'
    + columns.types string:string
    +#### A masked pattern was here ####
    + name default.srcpart
    + numFiles 1
    + numRows 500
    + partition_columns ds/hr
    + partition_columns.types string:string
    + rawDataSize 5312
    + serialization.ddl struct srcpart { string key, string value}
    + serialization.format 1
    + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
    + totalSize 5812
    +#### A masked pattern was here ####
    + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
    +
    + input format: org.apache.hadoop.mapred.TextInputFormat
    + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
    + properties:
    + bucket_count -1
    + columns key,value
    + columns.comments 'default','default'
    + columns.types string:string
    +#### A masked pattern was here ####
    + name default.srcpart
    + partition_columns ds/hr
    + partition_columns.types string:string
    + serialization.ddl struct srcpart { string key, string value}
    + serialization.format 1
    + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
    +#### A masked pattern was here ####
    + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
    + name: default.srcpart
    + name: default.srcpart
    + Truncated Path -> Alias:
    + /srcpart/ds=2008-04-08/hr=11 [$hdt$_0:srcpart]
    + /srcpart/ds=2008-04-08/hr=12 [$hdt$_0:srcpart]
    +
    + Stage: Stage-0
    + Move Operator
    + tables:
    + partition:
    + ds 2008-04-08
    + hr 11
    + replace: true
    +#### A masked pattern was here ####
    + table:
    + input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat
    + output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat
    + properties:
    + bucket_count -1
    + columns key,value
    + columns.comments
    + columns.types string:string
    +#### A masked pattern was here ####
    + name default.list_bucketing_static_part
    + partition_columns ds/hr
    + partition_columns.types string:string
    + serialization.ddl struct list_bucketing_static_part { string key, string value}
    + serialization.format 1
    + serialization.lib org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
    +#### A masked pattern was here ####
    + serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
    + name: default.list_bucketing_static_part
    +
    + Stage: Stage-2
    + Stats-Aggr Operator
    +#### A masked pattern was here ####
    +
    +PREHOOK: query: insert overwrite table list_bucketing_static_part partition (ds = '2008-04-08', hr = '11')
    +select key, value from srcpart where ds = '2008-04-08'
    +PREHOOK: type: QUERY
    +PREHOOK: Input: default@srcpart
    +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11
    +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12
    +PREHOOK: Output: default@list_bucketing_static_part@ds=2008-04-08/hr=11
    +POSTHOOK: query: insert overwrite table list_bucketing_static_part partition (ds = '2008-04-08', hr = '11')
    +select key, value from srcpart where ds = '2008-04-08'
    +POSTHOOK: type: QUERY
    +POSTHOOK: Input: default@srcpart
    +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11
    +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12
    +POSTHOOK: Output: default@list_bucketing_static_part@ds=2008-04-08/hr=11
    +POSTHOOK: Lineage: list_bucketing_static_part PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ]
    +POSTHOOK: Lineage: list_bucketing_static_part PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ]
    +PREHOOK: query: -- check DML result
    +show partitions list_bucketing_static_part
    +PREHOOK: type: SHOWPARTITIONS
    +PREHOOK: Input: default@list_bucketing_static_part
    +POSTHOOK: query: -- check DML result
    +show partitions list_bucketing_static_part
    +POSTHOOK: type: SHOWPARTITIONS
    +POSTHOOK: Input: default@list_bucketing_static_part
    +ds=2008-04-08/hr=11
    +PREHOOK: query: desc formatted list_bucketing_static_part partition (ds='2008-04-08', hr='11')
    +PREHOOK: type: DESCTABLE
    +PREHOOK: Input: default@list_bucketing_static_part
    +POSTHOOK: query: desc formatted list_bucketing_static_part partition (ds='2008-04-08', hr='11')
    +POSTHOOK: type: DESCTABLE
    +POSTHOOK: Input: default@list_bucketing_static_part
    +# col_name data_type comment
    +
    +key string
    +value string
    +
    +# Partition Information
    +# col_name data_type comment
    +
    +ds string
    +hr string
    +
    +# Detailed Partition Information
    +Partition Value: [2008-04-08, 11]
    +Database: default
    +Table: list_bucketing_static_part
    +#### A masked pattern was here ####
    +Protect Mode: None
    +#### A masked pattern was here ####
    +Partition Parameters:
    + COLUMN_STATS_ACCURATE true
    + numFiles 6
    + numRows 1000
    + rawDataSize 9624
    + totalSize 10898
    +#### A masked pattern was here ####
    +
    +# Storage Information
    +SerDe Library: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
    +InputFormat: org.apache.hadoop.hive.ql.io.RCFileInputFormat
    +OutputFormat: org.apache.hadoop.hive.ql.io.RCFileOutputFormat
    +Compressed: No
    +Num Buckets: -1
    +Bucket Columns: []
    +Sort Columns: []
    +Stored As SubDirectories: Yes
    +Skewed Columns: [key, value]
    +Skewed Values: [[484, val_484], [51, val_14], [103, val_103]]
    +#### A masked pattern was here ####
    +Skewed Value to Truncated Path: {[103, val_103]=/list_bucketing_static_part/ds=2008-04-08/hr=11/key=103/value=val_103, [484, val_484]=/list_bucketing_static_part/ds=2008-04-08/hr=11/key=484/value=val_484}
    +Storage Desc Params:
    + serialization.format 1
    +PREHOOK: query: -- list bucketing DML with merge. use bucketize to generate a few small files.
    +explain extended
    +insert overwrite table list_bucketing_static_part partition (ds = '2008-04-08', hr = '11')
    +select key, value from srcpart where ds = '2008-04-08'
    +PREHOOK: type: QUERY
    +POSTHOOK: query: -- list bucketing DML with merge. use bucketize to generate a few small files.
    +explain extended
    +insert overwrite table list_bucketing_static_part partition (ds = '2008-04-08', hr = '11')
    +select key, value from srcpart where ds = '2008-04-08'
    +POSTHOOK: type: QUERY
    +ABSTRACT SYNTAX TREE:
    +
    +TOK_QUERY
    + TOK_FROM
    + TOK_TABREF
    + TOK_TABNAME
    + srcpart
    + TOK_INSERT
    + TOK_DESTINATION
    + TOK_TAB
    + TOK_TABNAME
    + list_bucketing_static_part
    + TOK_PARTSPEC
    + TOK_PARTVAL
    + ds
    + '2008-04-08'
    + TOK_PARTVAL
    + hr
    + '11'
    + TOK_SELECT
    + TOK_SELEXPR
    + TOK_TABLE_OR_COL
    + key
    + TOK_SELEXPR
    + TOK_TABLE_OR_COL
    + value
    + TOK_WHERE
    + =
    + TOK_TABLE_OR_COL
    + ds
    + '2008-04-08'
    +
    +
    +STAGE DEPENDENCIES:
    + Stage-1 is a root stage
    + Stage-7 depends on stages: Stage-1 , consists of Stage-4, Stage-3, Stage-5
    + Stage-4
    + Stage-0 depends on stages: Stage-4, Stage-3, Stage-6
    + Stage-2 depends on stages: Stage-0
    + Stage-3
    + Stage-5
    + Stage-6 depends on stages: Stage-5
    +
    +STAGE PLANS:
    + Stage: Stage-1
    + Map Reduce
    + Map Operator Tree:
    + TableScan
    + alias: srcpart
    + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE
    + GatherStats: false
    + Select Operator
    + expressions: key (type: string), value (type: string)
    + outputColumnNames: _col0, _col1
    + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE
    + File Output Operator
    + compressed: false
    + GlobalTableId: 1
    +#### A masked pattern was here ####
    + NumFilesPerFileSink: 1
    + Static Partition Specification: ds=2008-04-08/hr=11/
    + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE
    +#### A masked pattern was here ####
    + table:
    + input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat
    + output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat
    + properties:
    + bucket_count -1
    + columns key,value
    + columns.comments
    + columns.types string:string
    +#### A masked pattern was here ####
    + name default.list_bucketing_static_part
    + partition_columns.types string:string
    + serialization.ddl struct list_bucketing_static_part { string key, string value}
    + serialization.format 1
    + serialization.lib org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
    +#### A masked pattern was here ####
    + serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
    + name: default.list_bucketing_static_part
    + TotalFiles: 1
    + GatherStats: true
    + MultiFileSpray: false
    + Path -> Alias:
    +#### A masked pattern was here ####
    + Path -> Partition:
    +#### A masked pattern was here ####
    + Partition
    + base file name: hr=11
    + input format: org.apache.hadoop.mapred.TextInputFormat
    + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
    + partition values:
    + ds 2008-04-08
    + hr 11
    + properties:
    + COLUMN_STATS_ACCURATE true
    + bucket_count -1
    + columns key,value
    + columns.comments 'default','default'
    + columns.types string:string
    +#### A masked pattern was here ####
    + name default.srcpart
    + numFiles 1
    + numRows 500
    + partition_columns ds/hr
    + partition_columns.types string:string
    + rawDataSize 5312
    + serialization.ddl struct srcpart { string key, string value}
    + serialization.format 1
    + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
    + totalSize 5812
    +#### A masked pattern was here ####
    + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
    +
    + input format: org.apache.hadoop.mapred.TextInputFormat
    + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
    + properties:
    + bucket_count -1
    + columns key,value
    + columns.comments 'default','default'
    + columns.types string:string
    +#### A masked pattern was here ####
    + name default.srcpart
    + partition_columns ds/hr
    + partition_columns.types string:string
    + serialization.ddl struct srcpart { string key, string value}
    + serialization.format 1
    + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
    +#### A masked pattern was here ####
    + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
    + name: default.srcpart
    + name: default.srcpart
    +#### A masked pattern was here ####
    + Partition
    + base file name: hr=12
    + input format: org.apache.hadoop.mapred.TextInputFormat
    + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
    + partition values:
    + ds 2008-04-08
    + hr 12
    + properties:
    + COLUMN_STATS_ACCURATE true
    + bucket_count -1
    + columns key,value
    + columns.comments 'default','default'
    + columns.types string:string
    +#### A masked pattern was here ####
    + name default.srcpart
    + numFiles 1
    + numRows 500
    + partition_columns ds/hr
    + partition_columns.types string:string
    + rawDataSize 5312
    + serialization.ddl struct srcpart { string key, string value}
    + serialization.format 1
    + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
    + totalSize 5812
    +#### A masked pattern was here ####
    + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
    +
    + input format: org.apache.hadoop.mapred.TextInputFormat
    + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
    + properties:
    + bucket_count -1
    + columns key,value
    + columns.comments 'default','default'
    + columns.types string:string
    +#### A masked pattern was here ####
    + name default.srcpart
    + partition_columns ds/hr
    + partition_columns.types string:string
    + serialization.ddl struct srcpart { string key, string value}
    + serialization.format 1
    + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
    +#### A masked pattern was here ####
    + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
    + name: default.srcpart
    + name: default.srcpart
    + Truncated Path -> Alias:
    + /srcpart/ds=2008-04-08/hr=11 [$hdt$_0:srcpart]
    + /srcpart/ds=2008-04-08/hr=12 [$hdt$_0:srcpart]
    +
    + Stage: Stage-7
    + Conditional Operator
    +
    + Stage: Stage-4
    + Move Operator
    + files:
    + hdfs directory: true
    +#### A masked pattern was here ####
    +
    + Stage: Stage-0
    + Move Operator
    + tables:
    + partition:
    + ds 2008-04-08
    + hr 11
    + replace: true
    +#### A masked pattern was here ####
    + table:
    + input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat
    + output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat
    + properties:
    + bucket_count -1
    + columns key,value
    + columns.comments
    + columns.types string:string
    +#### A masked pattern was here ####
    + name default.list_bucketing_static_part
    + partition_columns.types string:string
    + serialization.ddl struct list_bucketing_static_part { string key, string value}
    + serialization.format 1
    + serialization.lib org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
    +#### A masked pattern was here ####
    + serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
    + name: default.list_bucketing_static_part
    +
    + Stage: Stage-2
    + Stats-Aggr Operator
    +#### A masked pattern was here ####
    +
    + Stage: Stage-3
    + Merge File Operator
    + Map Operator Tree:
    + RCFile Merge Operator
    + merge level: block
    + Path -> Alias:
    +#### A masked pattern was here ####
    + Path -> Partition:
    +#### A masked pattern was here ####
    + Partition
    + input format: org.apache.hadoop.hive.ql.io.rcfile.merge.RCFileBlockMergeInputFormat
    + output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat
    + properties:
    + bucket_count -1
    + columns key,value
    + columns.comments
    + columns.types string:string
    +#### A masked pattern was here ####
    + name default.list_bucketing_static_part
    + partition_columns.types string:string
    + serialization.ddl struct list_bucketing_static_part { string key, string value}
    + serialization.format 1
    + serialization.lib org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
    +#### A masked pattern was here ####
    + serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
    +
    + input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat
    + output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat
    + properties:
    + bucket_count -1
    + columns key,value
    + columns.comments
    + columns.types string:string
    +#### A masked pattern was here ####
    + name default.list_bucketing_static_part
    + partition_columns.types string:string
    + serialization.ddl struct list_bucketing_static_part { string key, string value}
    + serialization.format 1
    + serialization.lib org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
    +#### A masked pattern was here ####
    + serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
    + name: default.list_bucketing_static_part
    + name: default.list_bucketing_static_part
    + input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat
    + Truncated Path -> Alias:
    +#### A masked pattern was here ####
    +
    + Stage: Stage-5
    + Merge File Operator
    + Map Operator Tree:
    + RCFile Merge Operator
    + merge level: block
    + Path -> Alias:
    +#### A masked pattern was here ####
    + Path -> Partition:
    +#### A masked pattern was here ####
    + Partition
    + input format: org.apache.hadoop.hive.ql.io.rcfile.merge.RCFileBlockMergeInputFormat
    + output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat
    + properties:
    + bucket_count -1
    + columns key,value
    + columns.comments
    + columns.types string:string
    +#### A masked pattern was here ####
    + name default.list_bucketing_static_part
    + partition_columns.types string:string
    + serialization.ddl struct list_bucketing_static_part { string key, string value}
    + serialization.format 1
    + serialization.lib org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
    +#### A masked pattern was here ####
    + serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
    +
    + input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat
    + output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat
    + properties:
    + bucket_count -1
    + columns key,value
    + columns.comments
    + columns.types string:string
    +#### A masked pattern was here ####
    + name default.list_bucketing_static_part
    + partition_columns.types string:string
    + serialization.ddl struct list_bucketing_static_part { string key, string value}
    + serialization.format 1
    + serialization.lib org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
    +#### A masked pattern was here ####
    + serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
    + name: default.list_bucketing_static_part
    + name: default.list_bucketing_static_part
    + input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat
    + Truncated Path -> Alias:
    +#### A masked pattern was here ####
    +
    + Stage: Stage-6
    + Move Operator
    + files:
    + hdfs directory: true
    +#### A masked pattern was here ####
    +
    +PREHOOK: query: insert overwrite table list_bucketing_static_part partition (ds = '2008-04-08', hr = '11')
    +select key, value from srcpart where ds = '2008-04-08'
    +PREHOOK: type: QUERY
    +PREHOOK: Input: default@srcpart
    +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11
    +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12
    +PREHOOK: Output: default@list_bucketing_static_part@ds=2008-04-08/hr=11
    +POSTHOOK: query: insert overwrite table list_bucketing_static_part partition (ds = '2008-04-08', hr = '11')
    +select key, value from srcpart where ds = '2008-04-08'
    +POSTHOOK: type: QUERY
    +POSTHOOK: Input: default@srcpart
    +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11
    +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12
    +POSTHOOK: Output: default@list_bucketing_static_part@ds=2008-04-08/hr=11
    +POSTHOOK: Lineage: list_bucketing_static_part PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ]
    +POSTHOOK: Lineage: list_bucketing_static_part PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ]
    +PREHOOK: query: -- check DML result
    +show partitions list_bucketing_static_part
    +PREHOOK: type: SHOWPARTITIONS
    +PREHOOK: Input: default@list_bucketing_static_part
    +POSTHOOK: query: -- check DML result
    +show partitions list_bucketing_static_part
    +POSTHOOK: type: SHOWPARTITIONS
    +POSTHOOK: Input: default@list_bucketing_static_part
    +ds=2008-04-08/hr=11
    +PREHOOK: query: desc formatted list_bucketing_static_part partition (ds='2008-04-08', hr='11')
    +PREHOOK: type: DESCTABLE
    +PREHOOK: Input: default@list_bucketing_static_part
    +POSTHOOK: query: desc formatted list_bucketing_static_part partition (ds='2008-04-08', hr='11')
    +POSTHOOK: type: DESCTABLE
    +POSTHOOK: Input: default@list_bucketing_static_part
    +# col_name data_type comment
    +
    +key string
    +value string
    +
    +# Partition Information
    +# col_name data_type comment
    +
    +ds string
    +hr string
    +
    +# Detailed Partition Information
    +Partition Value: [2008-04-08, 11]
    +Database: default
    +Table: list_bucketing_static_part
    +#### A masked pattern was here ####
    +Protect Mode: None
    +#### A masked pattern was here ####
    +Partition Parameters:
    + COLUMN_STATS_ACCURATE true
    + numFiles 4
    + numRows 1000
    + rawDataSize 9624
    + totalSize 10786
    +#### A masked pattern was here ####
    +
    +# Storage Information
    +SerDe Library: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
    +InputFormat: org.apache.hadoop.hive.ql.io.RCFileInputFormat
    +OutputFormat: org.apache.hadoop.hive.ql.io.RCFileOutputFormat
    +Compressed: No
    +Num Buckets: -1
    +Bucket Columns: []
    +Sort Columns: []
    +Stored As SubDirectories: Yes
    +Skewed Columns: [key, value]
    +Skewed Values: [[484, val_484], [51, val_14], [103, val_103]]
    +#### A masked pattern was here ####
    +Skewed Value to Truncated Path: {[103, val_103]=/list_bucketing_static_part/ds=2008-04-08/hr=11/key=103/value=val_103, [484, val_484]=/list_bucketing_static_part/ds=2008-04-08/hr=11/key=484/value=val_484}
    +Storage Desc Params:
    + serialization.format 1
    +PREHOOK: query: select count(1) from srcpart where ds = '2008-04-08'
    +PREHOOK: type: QUERY
    +PREHOOK: Input: default@srcpart
    +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11
    +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12
    +#### A masked pattern was here ####
    +POSTHOOK: query: select count(1) from srcpart where ds = '2008-04-08'
    +POSTHOOK: type: QUERY
    +POSTHOOK: Input: default@srcpart
    +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11
    +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12
    +#### A masked pattern was here ####
    +1000
    +PREHOOK: query: select count(*) from list_bucketing_static_part
    +PREHOOK: type: QUERY
    +PREHOOK: Input: default@list_bucketing_static_part
    +PREHOOK: Input: default@list_bucketing_static_part@ds=2008-04-08/hr=11
    +#### A masked pattern was here ####
    +POSTHOOK: query: select count(*) from list_bucketing_static_part
    +POSTHOOK: type: QUERY
    +POSTHOOK: Input: default@list_bucketing_static_part
    +POSTHOOK: Input: default@list_bucketing_static_part@ds=2008-04-08/hr=11
    +#### A masked pattern was here ####
    +1000
    +PREHOOK: query: explain extended
    +select * from list_bucketing_static_part where ds = '2008-04-08' and hr = '11' and key = '484' and value = 'val_484'
    +PREHOOK: type: QUERY
    +POSTHOOK: query: explain extended
    +select * from list_bucketing_static_part where ds = '2008-04-08' and hr = '11' and key = '484' and value = 'val_484'
    +POSTHOOK: type: QUERY
    +ABSTRACT SYNTAX TREE:
    +
    +TOK_QUERY
    + TOK_FROM
    + TOK_TABREF
    + TOK_TABNAME
    + list_bucketing_static_part
    + TOK_INSERT
    + TOK_DESTINATION
    + TOK_DIR
    + TOK_TMP_FILE
    + TOK_SELECT
    + TOK_SELEXPR
    + TOK_ALLCOLREF
    + TOK_WHERE
    + and
    + and
    + and
    + =
    + TOK_TABLE_OR_COL
    + ds
    + '2008-04-08'
    + =
    + TOK_TABLE_OR_COL
    + hr
    + '11'
    + =
    + TOK_TABLE_OR_COL
    + key
    + '484'
    + =
    + TOK_TABLE_OR_COL
    + value
    + 'val_484'
    +
    +
    +STAGE DEPENDENCIES:
    + Stage-1 is a root stage
    + Stage-0 depends on stages: Stage-1
    +
    +STAGE PLANS:
    + Stage: Stage-1
    + Map Reduce
    + Map Operator Tree:
    + TableScan
    + alias: list_bucketing_static_part
    + Statistics: Num rows: 1000 Data size: 9624 Basic stats: COMPLETE Column stats: NONE
    + GatherStats: false
    + Filter Operator
    + isSamplingPred: false
    + predicate: ((key = '484') and (value = 'val_484')) (type: boolean)
    + Statistics: Num rows: 250 Data size: 2406 Basic stats: COMPLETE Column stats: NONE
    + Select Operator
    + expressions: '484' (type: string), 'val_484' (type: string), '2008-04-08' (type: string), '11' (type: string)
    + outputColumnNames: _col0, _col1, _col2, _col3
    + Statistics: Num rows: 250 Data size: 2406 Basic stats: COMPLETE Column stats: NONE
    + File Output Operator
    + compressed: false
    + GlobalTableId: 0
    +#### A masked pattern was here ####
    + NumFilesPerFileSink: 1
    + Statistics: Num rows: 250 Data size: 2406 Basic stats: COMPLETE Column stats: NONE
    +#### A masked pattern was here ####
    + table:
    + input format: org.apache.hadoop.mapred.TextInputFormat
    + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
    + properties:
    + columns _col0,_col1,_col2,_col3
    + columns.types string:string:string:string
    + escape.delim \
    + hive.serialization.extend.nesting.levels true
    + serialization.format 1
    + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
    + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
    + TotalFiles: 1
    + GatherStats: false
    + MultiFileSpray: false
    + Path -> Alias:
    +#### A masked pattern was here ####
    + Path -> Partition:
    +#### A masked pattern was here ####
    + Partition
    + base file name: value=val_484
    + input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat
    + output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat
    + partition values:
    + ds 2008-04-08
    + hr 11
    + properties:
    + COLUMN_STATS_ACCURATE true
    + bucket_count -1
    + columns key,value
    + columns.comments
    + columns.types string:string
    +#### A masked pattern was here ####
    + name default.list_bucketing_static_part
    + numFiles 4
    + numRows 1000
    + partition_columns ds/hr
    + partition_columns.types string:string
    + rawDataSize 9624
    + serialization.ddl struct list_bucketing_static_part { string key, string value}
    + serialization.format 1
    + serialization.lib org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
    + totalSize 10786
    +#### A masked pattern was here ####
    + serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
    +
    + input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat
    + output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat
    + properties:
    + bucket_count -1
    + columns key,value
    + columns.comments
    + columns.types string:string
    +#### A masked pattern was here ####
    + name default.list_bucketing_static_part
    + partition_columns ds/hr
    + partition_columns.types string:string
    + serialization.ddl struct list_bucketing_static_part { string key, string value}
    + serialization.format 1
    + serialization.lib org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
    +#### A masked pattern was here ####
    + serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
    + name: default.list_bucketing_static_part
    + name: default.list_bucketing_static_part
    + Truncated Path -> Alias:
    + /list_bucketing_static_part/ds=2008-04-08/hr=11/key=484/value=val_484 [list_bucketing_static_part]
    +
    + Stage: Stage-0
    + Fetch Operator
    + limit: -1
    + Processor Tree:
    + ListSink
    +
    +PREHOOK: query: select * from list_bucketing_static_part where ds = '2008-04-08' and hr = '11' and key = '484' and value = 'val_484'
    +PREHOOK: type: QUERY
    +PREHOOK: Input: default@list_bucketing_static_part
    +PREHOOK: Input: default@list_bucketing_static_part@ds=2008-04-08/hr=11
    +#### A masked pattern was here ####
    +POSTHOOK: query: select * from list_bucketing_static_part where ds = '2008-04-08' and hr = '11' and key = '484' and value = 'val_484'
    +POSTHOOK: type: QUERY
    +POSTHOOK: Input: default@list_bucketing_static_part
    +POSTHOOK: Input: default@list_bucketing_static_part@ds=2008-04-08/hr=11
    +#### A masked pattern was here ####
    +484 val_484 2008-04-08 11
    +484 val_484 2008-04-08 11
    +PREHOOK: query: select * from srcpart where ds = '2008-04-08' and key = '484' and value = 'val_484'
    +PREHOOK: type: QUERY
    +PREHOOK: Input: default@srcpart
    +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11
    +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12
    +#### A masked pattern was here ####
    +POSTHOOK: query: select * from srcpart where ds = '2008-04-08' and key = '484' and value = 'val_484'
    +POSTHOOK: type: QUERY
    +POSTHOOK: Input: default@srcpart
    +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11
    +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12
    +#### A masked pattern was here ####
    +484 val_484 2008-04-08 11
    +484 val_484 2008-04-08 12
    +PREHOOK: query: -- clean up
    +drop table list_bucketing_static_part
    +PREHOOK: type: DROPTABLE
    +PREHOOK: Input: default@list_bucketing_static_part
    +PREHOOK: Output: default@list_bucketing_static_part
    +POSTHOOK: query: -- clean up
    +drop table list_bucketing_static_part
    +POSTHOOK: type: DROPTABLE
    +POSTHOOK: Input: default@list_bucketing_static_part
    +POSTHOOK: Output: default@list_bucketing_static_part

    Added: hive/trunk/ql/src/test/results/clientpositive/list_bucket_dml_4.q.java1.8.out
    URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/list_bucket_dml_4.q.java1.8.out?rev=1647912&view=auto
    ==============================================================================
    --- hive/trunk/ql/src/test/results/clientpositive/list_bucket_dml_4.q.java1.8.out (added)
    +++ hive/trunk/ql/src/test/results/clientpositive/list_bucket_dml_4.q.java1.8.out Thu Dec 25 18:06:30 2014
    @@ -0,0 +1,951 @@
    +PREHOOK: query: -- INCLUDE_HADOOP_MAJOR_VERSIONS(0.23)
    +-- SORT_QUERY_RESULTS
    +-- JAVA_VERSION_SPECIFIC_OUTPUT
    +
    +-- list bucketing DML: static partition. multiple skewed columns. merge.
    +-- ds=2008-04-08/hr=11/HIVE_DEFAULT_LIST_BUCKETING_DIR_NAME:
    +-- 5263 000000_0
    +-- 5263 000001_0
    +-- ds=2008-04-08/hr=11/key=103/value=val_103:
    +-- 99 000000_0
    +-- 99 000001_0
    +-- after merge
    +-- 142 000000_0
    +-- ds=2008-04-08/hr=11/key=484/value=val_484:
    +-- 87 000000_0
    +-- 87 000001_0
    +-- after merge
    +-- 118 000001_0
    +
    +-- create a skewed table
    +create table list_bucketing_static_part (key String, value String)
    + partitioned by (ds String, hr String)
    + skewed by (key, value) on (('484','val_484'),('51','val_14'),('103','val_103'))
    + stored as DIRECTORIES
    + STORED AS RCFILE
    +PREHOOK: type: CREATETABLE
    +PREHOOK: Output: database:default
    +PREHOOK: Output: default@list_bucketing_static_part
    +POSTHOOK: query: -- INCLUDE_HADOOP_MAJOR_VERSIONS(0.23)
    +-- SORT_QUERY_RESULTS
    +-- JAVA_VERSION_SPECIFIC_OUTPUT
    +
    +-- list bucketing DML: static partition. multiple skewed columns. merge.
    +-- ds=2008-04-08/hr=11/HIVE_DEFAULT_LIST_BUCKETING_DIR_NAME:
    +-- 5263 000000_0
    +-- 5263 000001_0
    +-- ds=2008-04-08/hr=11/key=103/value=val_103:
    +-- 99 000000_0
    +-- 99 000001_0
    +-- after merge
    +-- 142 000000_0
    +-- ds=2008-04-08/hr=11/key=484/value=val_484:
    +-- 87 000000_0
    +-- 87 000001_0
    +-- after merge
    +-- 118 000001_0
    +
    +-- create a skewed table
    +create table list_bucketing_static_part (key String, value String)
    + partitioned by (ds String, hr String)
    + skewed by (key, value) on (('484','val_484'),('51','val_14'),('103','val_103'))
    + stored as DIRECTORIES
    + STORED AS RCFILE
    +POSTHOOK: type: CREATETABLE
    +POSTHOOK: Output: database:default
    +POSTHOOK: Output: default@list_bucketing_static_part
    +PREHOOK: query: -- list bucketing DML without merge. use bucketize to generate a few small files.
    +explain extended
    +insert overwrite table list_bucketing_static_part partition (ds = '2008-04-08', hr = '11')
    +select key, value from srcpart where ds = '2008-04-08'
    +PREHOOK: type: QUERY
    +POSTHOOK: query: -- list bucketing DML without merge. use bucketize to generate a few small files.
    +explain extended
    +insert overwrite table list_bucketing_static_part partition (ds = '2008-04-08', hr = '11')
    +select key, value from srcpart where ds = '2008-04-08'
    +POSTHOOK: type: QUERY
    +ABSTRACT SYNTAX TREE:
    +
    +TOK_QUERY
    + TOK_FROM
    + TOK_TABREF
    + TOK_TABNAME
    + srcpart
    + TOK_INSERT
    + TOK_DESTINATION
    + TOK_TAB
    + TOK_TABNAME
    + list_bucketing_static_part
    + TOK_PARTSPEC
    + TOK_PARTVAL
    + ds
    + '2008-04-08'
    + TOK_PARTVAL
    + hr
    + '11'
    + TOK_SELECT
    + TOK_SELEXPR
    + TOK_TABLE_OR_COL
    + key
    + TOK_SELEXPR
    + TOK_TABLE_OR_COL
    + value
    + TOK_WHERE
    + =
    + TOK_TABLE_OR_COL
    + ds
    + '2008-04-08'
    +
    +
    +STAGE DEPENDENCIES:
    + Stage-1 is a root stage
    + Stage-0 depends on stages: Stage-1
    + Stage-2 depends on stages: Stage-0
    +
    +STAGE PLANS:
    + Stage: Stage-1
    + Map Reduce
    + Map Operator Tree:
    + TableScan
    + alias: srcpart
    + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE
    + GatherStats: false
    + Select Operator
    + expressions: key (type: string), value (type: string)
    + outputColumnNames: _col0, _col1
    + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE
    + File Output Operator
    + compressed: false
    + GlobalTableId: 1
    +#### A masked pattern was here ####
    + NumFilesPerFileSink: 1
    + Static Partition Specification: ds=2008-04-08/hr=11/
    + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE
    +#### A masked pattern was here ####
    + table:
    + input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat
    + output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat
    + properties:
    + bucket_count -1
    + columns key,value
    + columns.comments
    + columns.types string:string
    +#### A masked pattern was here ####
    + name default.list_bucketing_static_part
    + partition_columns ds/hr
    + partition_columns.types string:string
    + serialization.ddl struct list_bucketing_static_part { string key, string value}
    + serialization.format 1
    + serialization.lib org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
    +#### A masked pattern was here ####
    + serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
    + name: default.list_bucketing_static_part
    + TotalFiles: 1
    + GatherStats: true
    + MultiFileSpray: false
    + Path -> Alias:
    +#### A masked pattern was here ####
    + Path -> Partition:
    +#### A masked pattern was here ####
    + Partition
    + base file name: hr=11
    + input format: org.apache.hadoop.mapred.TextInputFormat
    + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
    + partition values:
    + ds 2008-04-08
    + hr 11
    + properties:
    + COLUMN_STATS_ACCURATE true
    + bucket_count -1
    + columns key,value
    + columns.comments 'default','default'
    + columns.types string:string
    +#### A masked pattern was here ####
    + name default.srcpart
    + numFiles 1
    + numRows 500
    + partition_columns ds/hr
    + partition_columns.types string:string
    + rawDataSize 5312
    + serialization.ddl struct srcpart { string key, string value}
    + serialization.format 1
    + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
    + totalSize 5812
    +#### A masked pattern was here ####
    + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
    +
    + input format: org.apache.hadoop.mapred.TextInputFormat
    + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
    + properties:
    + bucket_count -1
    + columns key,value
    + columns.comments 'default','default'
    + columns.types string:string
    +#### A masked pattern was here ####
    + name default.srcpart
    + partition_columns ds/hr
    + partition_columns.types string:string
    + serialization.ddl struct srcpart { string key, string value}
    + serialization.format 1
    + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
    +#### A masked pattern was here ####
    + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
    + name: default.srcpart
    + name: default.srcpart
    +#### A masked pattern was here ####
    + Partition
    + base file name: hr=12
    + input format: org.apache.hadoop.mapred.TextInputFormat
    + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
    + partition values:
    + ds 2008-04-08
    + hr 12
    + properties:
    + COLUMN_STATS_ACCURATE true
    + bucket_count -1
    + columns key,value
    + columns.comments 'default','default'
    + columns.types string:string
    +#### A masked pattern was here ####
    + name default.srcpart
    + numFiles 1
    + numRows 500
    + partition_columns ds/hr
    + partition_columns.types string:string
    + rawDataSize 5312
    + serialization.ddl struct srcpart { string key, string value}
    + serialization.format 1
    + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
    + totalSize 5812
    +#### A masked pattern was here ####
    + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
    +
    + input format: org.apache.hadoop.mapred.TextInputFormat
    + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
    + properties:
    + bucket_count -1
    + columns key,value
    + columns.comments 'default','default'
    + columns.types string:string
    +#### A masked pattern was here ####
    + name default.srcpart
    + partition_columns ds/hr
    + partition_columns.types string:string
    + serialization.ddl struct srcpart { string key, string value}
    + serialization.format 1
    + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
    +#### A masked pattern was here ####
    + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
    + name: default.srcpart
    + name: default.srcpart
    + Truncated Path -> Alias:
    + /srcpart/ds=2008-04-08/hr=11 [$hdt$_0:srcpart]
    + /srcpart/ds=2008-04-08/hr=12 [$hdt$_0:srcpart]
    +
    + Stage: Stage-0
    + Move Operator
    + tables:
    + partition:
    + ds 2008-04-08
    + hr 11
    + replace: true
    +#### A masked pattern was here ####
    + table:
    + input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat
    + output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat
    + properties:
    + bucket_count -1
    + columns key,value
    + columns.comments
    + columns.types string:string
    +#### A masked pattern was here ####
    + name default.list_bucketing_static_part
    + partition_columns ds/hr
    + partition_columns.types string:string
    + serialization.ddl struct list_bucketing_static_part { string key, string value}
    + serialization.format 1
    + serialization.lib org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
    +#### A masked pattern was here ####
    + serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
    + name: default.list_bucketing_static_part
    +
    + Stage: Stage-2
    + Stats-Aggr Operator
    +#### A masked pattern was here ####
    +
    +PREHOOK: query: insert overwrite table list_bucketing_static_part partition (ds = '2008-04-08', hr = '11')
    +select key, value from srcpart where ds = '2008-04-08'
    +PREHOOK: type: QUERY
    +PREHOOK: Input: default@srcpart
    +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11
    +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12
    +PREHOOK: Output: default@list_bucketing_static_part@ds=2008-04-08/hr=11
    +POSTHOOK: query: insert overwrite table list_bucketing_static_part partition (ds = '2008-04-08', hr = '11')
    +select key, value from srcpart where ds = '2008-04-08'
    +POSTHOOK: type: QUERY
    +POSTHOOK: Input: default@srcpart
    +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11
    +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12
    +POSTHOOK: Output: default@list_bucketing_static_part@ds=2008-04-08/hr=11
    +POSTHOOK: Lineage: list_bucketing_static_part PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ]
    +POSTHOOK: Lineage: list_bucketing_static_part PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ]
    +PREHOOK: query: -- check DML result
    +show partitions list_bucketing_static_part
    +PREHOOK: type: SHOWPARTITIONS
    +PREHOOK: Input: default@list_bucketing_static_part
    +POSTHOOK: query: -- check DML result
    +show partitions list_bucketing_static_part
    +POSTHOOK: type: SHOWPARTITIONS
    +POSTHOOK: Input: default@list_bucketing_static_part
    +ds=2008-04-08/hr=11
    +PREHOOK: query: desc formatted list_bucketing_static_part partition (ds='2008-04-08', hr='11')
    +PREHOOK: type: DESCTABLE
    +PREHOOK: Input: default@list_bucketing_static_part
    +POSTHOOK: query: desc formatted list_bucketing_static_part partition (ds='2008-04-08', hr='11')
    +POSTHOOK: type: DESCTABLE
    +POSTHOOK: Input: default@list_bucketing_static_part
    +# col_name data_type comment
    +
    +key string
    +value string
    +
    +# Partition Information
    +# col_name data_type comment
    +
    +ds string
    +hr string
    +
    +# Detailed Partition Information
    +Partition Value: [2008-04-08, 11]
    +Database: default
    +Table: list_bucketing_static_part
    +#### A masked pattern was here ####
    +Protect Mode: None
    +#### A masked pattern was here ####
    +Partition Parameters:
    + COLUMN_STATS_ACCURATE true
    + numFiles 6
    + numRows 1000
    + rawDataSize 9624
    + totalSize 10898
    +#### A masked pattern was here ####
    +
    +# Storage Information
    +SerDe Library: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
    +InputFormat: org.apache.hadoop.hive.ql.io.RCFileInputFormat
    +OutputFormat: org.apache.hadoop.hive.ql.io.RCFileOutputFormat
    +Compressed: No
    +Num Buckets: -1
    +Bucket Columns: []
    +Sort Columns: []
    +Stored As SubDirectories: Yes
    +Skewed Columns: [key, value]
    +Skewed Values: [[484, val_484], [51, val_14], [103, val_103]]
    +#### A masked pattern was here ####
    +Skewed Value to Truncated Path: {[484, val_484]=/list_bucketing_static_part/ds=2008-04-08/hr=11/key=484/value=val_484, [103, val_103]=/list_bucketing_static_part/ds=2008-04-08/hr=11/key=103/value=val_103}
    +Storage Desc Params:
    + serialization.format 1
    +PREHOOK: query: -- list bucketing DML with merge. use bucketize to generate a few small files.
    +explain extended
    +insert overwrite table list_bucketing_static_part partition (ds = '2008-04-08', hr = '11')
    +select key, value from srcpart where ds = '2008-04-08'
    +PREHOOK: type: QUERY
    +POSTHOOK: query: -- list bucketing DML with merge. use bucketize to generate a few small files.
    +explain extended
    +insert overwrite table list_bucketing_static_part partition (ds = '2008-04-08', hr = '11')
    +select key, value from srcpart where ds = '2008-04-08'
    +POSTHOOK: type: QUERY
    +ABSTRACT SYNTAX TREE:
    +
    +TOK_QUERY
    + TOK_FROM
    + TOK_TABREF
    + TOK_TABNAME
    + srcpart
    + TOK_INSERT
    + TOK_DESTINATION
    + TOK_TAB
    + TOK_TABNAME
    + list_bucketing_static_part
    + TOK_PARTSPEC
    + TOK_PARTVAL
    + ds
    + '2008-04-08'
    + TOK_PARTVAL
    + hr
    + '11'
    + TOK_SELECT
    + TOK_SELEXPR
    + TOK_TABLE_OR_COL
    + key
    + TOK_SELEXPR
    + TOK_TABLE_OR_COL
    + value
    + TOK_WHERE
    + =
    + TOK_TABLE_OR_COL
    + ds
    + '2008-04-08'
    +
    +
    +STAGE DEPENDENCIES:
    + Stage-1 is a root stage
    + Stage-7 depends on stages: Stage-1 , consists of Stage-4, Stage-3, Stage-5
    + Stage-4
    + Stage-0 depends on stages: Stage-4, Stage-3, Stage-6
    + Stage-2 depends on stages: Stage-0
    + Stage-3
    + Stage-5
    + Stage-6 depends on stages: Stage-5
    +
    +STAGE PLANS:
    + Stage: Stage-1
    + Map Reduce
    + Map Operator Tree:
    + TableScan
    + alias: srcpart
    + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE
    + GatherStats: false
    + Select Operator
    + expressions: key (type: string), value (type: string)
    + outputColumnNames: _col0, _col1
    + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE
    + File Output Operator
    + compressed: false
    + GlobalTableId: 1
    +#### A masked pattern was here ####
    + NumFilesPerFileSink: 1
    + Static Partition Specification: ds=2008-04-08/hr=11/
    + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE
    +#### A masked pattern was here ####
    + table:
    + input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat
    + output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat
    + properties:
    + bucket_count -1
    + columns key,value
    + columns.comments
    + columns.types string:string
    +#### A masked pattern was here ####
    + name default.list_bucketing_static_part
    + partition_columns.types string:string
    + serialization.ddl struct list_bucketing_static_part { string key, string value}
    + serialization.format 1
    + serialization.lib org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
    +#### A masked pattern was here ####
    + serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
    + name: default.list_bucketing_static_part
    + TotalFiles: 1
    + GatherStats: true
    + MultiFileSpray: false
    + Path -> Alias:
    +#### A masked pattern was here ####
    + Path -> Partition:
    +#### A masked pattern was here ####
    + Partition
    + base file name: hr=11
    + input format: org.apache.hadoop.mapred.TextInputFormat
    + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
    + partition values:
    + ds 2008-04-08
    + hr 11
    + properties:
    + COLUMN_STATS_ACCURATE true
    + bucket_count -1
    + columns key,value
    + columns.comments 'default','default'
    + columns.types string:string
    +#### A masked pattern was here ####
    + name default.srcpart
    + numFiles 1
    + numRows 500
    + partition_columns ds/hr
    + partition_columns.types string:string
    + rawDataSize 5312
    + serialization.ddl struct srcpart { string key, string value}
    + serialization.format 1
    + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
    + totalSize 5812
    +#### A masked pattern was here ####
    + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
    +
    + input format: org.apache.hadoop.mapred.TextInputFormat
    + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
    + properties:
    + bucket_count -1
    + columns key,value
    + columns.comments 'default','default'
    + columns.types string:string
    +#### A masked pattern was here ####
    + name default.srcpart
    + partition_columns ds/hr
    + partition_columns.types string:string
    + serialization.ddl struct srcpart { string key, string value}
    + serialization.format 1
    + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
    +#### A masked pattern was here ####
    + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
    + name: default.srcpart
    + name: default.srcpart
    +#### A masked pattern was here ####
    + Partition
    + base file name: hr=12
    + input format: org.apache.hadoop.mapred.TextInputFormat
    + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
    + partition values:
    + ds 2008-04-08
    + hr 12
    + properties:
    + COLUMN_STATS_ACCURATE true
    + bucket_count -1
    + columns key,value
    + columns.comments 'default','default'
    + columns.types string:string
    +#### A masked pattern was here ####
    + name default.srcpart
    + numFiles 1
    + numRows 500
    + partition_columns ds/hr
    + partition_columns.types string:string
    + rawDataSize 5312
    + serialization.ddl struct srcpart { string key, string value}
    + serialization.format 1
    + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
    + totalSize 5812
    +#### A masked pattern was here ####
    + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
    +
    + input format: org.apache.hadoop.mapred.TextInputFormat
    + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
    + properties:
    + bucket_count -1
    + columns key,value
    + columns.comments 'default','default'
    + columns.types string:string
    +#### A masked pattern was here ####
    + name default.srcpart
    + partition_columns ds/hr
    + partition_columns.types string:string
    + serialization.ddl struct srcpart { string key, string value}
    + serialization.format 1
    + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
    +#### A masked pattern was here ####
    + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
    + name: default.srcpart
    + name: default.srcpart
    + Truncated Path -> Alias:
    + /srcpart/ds=2008-04-08/hr=11 [$hdt$_0:srcpart]
    + /srcpart/ds=2008-04-08/hr=12 [$hdt$_0:srcpart]
    +
    + Stage: Stage-7
    + Conditional Operator
    +
    + Stage: Stage-4
    + Move Operator
    + files:
    + hdfs directory: true
    +#### A masked pattern was here ####
    +
    + Stage: Stage-0
    + Move Operator
    + tables:
    + partition:
    + ds 2008-04-08
    + hr 11
    + replace: true
    +#### A masked pattern was here ####
    + table:
    + input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat
    + output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat
    + properties:
    + bucket_count -1
    + columns key,value
    + columns.comments
    + columns.types string:string
    +#### A masked pattern was here ####
    + name default.list_bucketing_static_part
    + partition_columns.types string:string
    + serialization.ddl struct list_bucketing_static_part { string key, string value}
    + serialization.format 1
    + serialization.lib org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
    +#### A masked pattern was here ####
    + serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
    + name: default.list_bucketing_static_part
    +
    + Stage: Stage-2
    + Stats-Aggr Operator
    +#### A masked pattern was here ####
    +
    + Stage: Stage-3
    + Merge File Operator
    + Map Operator Tree:
    + RCFile Merge Operator
    + merge level: block
    + Path -> Alias:
    +#### A masked pattern was here ####
    + Path -> Partition:
    +#### A masked pattern was here ####
    + Partition
    + input format: org.apache.hadoop.hive.ql.io.rcfile.merge.RCFileBlockMergeInputFormat
    + output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat
    + properties:
    + bucket_count -1
    + columns key,value
    + columns.comments
    + columns.types string:string
    +#### A masked pattern was here ####
    + name default.list_bucketing_static_part
    + partition_columns.types string:string
    + serialization.ddl struct list_bucketing_static_part { string key, string value}
    + serialization.format 1
    + serialization.lib org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
    +#### A masked pattern was here ####
    + serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
    +
    + input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat
    + output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat
    + properties:
    + bucket_count -1
    + columns key,value
    + columns.comments
    + columns.types string:string
    +#### A masked pattern was here ####
    + name default.list_bucketing_static_part
    + partition_columns.types string:string
    + serialization.ddl struct list_bucketing_static_part { string key, string value}
    + serialization.format 1
    + serialization.lib org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
    +#### A masked pattern was here ####
    + serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
    + name: default.list_bucketing_static_part
    + name: default.list_bucketing_static_part
    + input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat
    + Truncated Path -> Alias:
    +#### A masked pattern was here ####
    +
    + Stage: Stage-5
    + Merge File Operator
    + Map Operator Tree:
    + RCFile Merge Operator
    + merge level: block
    + Path -> Alias:
    +#### A masked pattern was here ####
    + Path -> Partition:
    +#### A masked pattern was here ####
    + Partition
    + input format: org.apache.hadoop.hive.ql.io.rcfile.merge.RCFileBlockMergeInputFormat
    + output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat
    + properties:
    + bucket_count -1
    + columns key,value
    + columns.comments
    + columns.types string:string
    +#### A masked pattern was here ####
    + name default.list_bucketing_static_part
    + partition_columns.types string:string
    + serialization.ddl struct list_bucketing_static_part { string key, string value}
    + serialization.format 1
    + serialization.lib org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
    +#### A masked pattern was here ####
    + serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
    +
    + input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat
    + output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat
    + properties:
    + bucket_count -1
    + columns key,value
    + columns.comments
    + columns.types string:string
    +#### A masked pattern was here ####
    + name default.list_bucketing_static_part
    + partition_columns.types string:string
    + serialization.ddl struct list_bucketing_static_part { string key, string value}
    + serialization.format 1
    + serialization.lib org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
    +#### A masked pattern was here ####
    + serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
    + name: default.list_bucketing_static_part
    + name: default.list_bucketing_static_part
    + input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat
    + Truncated Path -> Alias:
    +#### A masked pattern was here ####
    +
    + Stage: Stage-6
    + Move Operator
    + files:
    + hdfs directory: true
    +#### A masked pattern was here ####
    +
    +PREHOOK: query: insert overwrite table list_bucketing_static_part partition (ds = '2008-04-08', hr = '11')
    +select key, value from srcpart where ds = '2008-04-08'
    +PREHOOK: type: QUERY
    +PREHOOK: Input: default@srcpart
    +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11
    +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12
    +PREHOOK: Output: default@list_bucketing_static_part@ds=2008-04-08/hr=11
    +POSTHOOK: query: insert overwrite table list_bucketing_static_part partition (ds = '2008-04-08', hr = '11')
    +select key, value from srcpart where ds = '2008-04-08'
    +POSTHOOK: type: QUERY
    +POSTHOOK: Input: default@srcpart
    +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11
    +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12
    +POSTHOOK: Output: default@list_bucketing_static_part@ds=2008-04-08/hr=11
    +POSTHOOK: Lineage: list_bucketing_static_part PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ]
    +POSTHOOK: Lineage: list_bucketing_static_part PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ]
    +PREHOOK: query: -- check DML result
    +show partitions list_bucketing_static_part
    +PREHOOK: type: SHOWPARTITIONS
    +PREHOOK: Input: default@list_bucketing_static_part
    +POSTHOOK: query: -- check DML result
    +show partitions list_bucketing_static_part
    +POSTHOOK: type: SHOWPARTITIONS
    +POSTHOOK: Input: default@list_bucketing_static_part
    +ds=2008-04-08/hr=11
    +PREHOOK: query: desc formatted list_bucketing_static_part partition (ds='2008-04-08', hr='11')
    +PREHOOK: type: DESCTABLE
    +PREHOOK: Input: default@list_bucketing_static_part
    +POSTHOOK: query: desc formatted list_bucketing_static_part partition (ds='2008-04-08', hr='11')
    +POSTHOOK: type: DESCTABLE
    +POSTHOOK: Input: default@list_bucketing_static_part
    +# col_name data_type comment
    +
    +key string
    +value string
    +
    +# Partition Information
    +# col_name data_type comment
    +
    +ds string
    +hr string
    +
    +# Detailed Partition Information
    +Partition Value: [2008-04-08, 11]
    +Database: default
    +Table: list_bucketing_static_part
    +#### A masked pattern was here ####
    +Protect Mode: None
    +#### A masked pattern was here ####
    +Partition Parameters:
    + COLUMN_STATS_ACCURATE true
    + numFiles 4
    + numRows 1000
    + rawDataSize 9624
    + totalSize 10786
    +#### A masked pattern was here ####
    +
    +# Storage Information
    +SerDe Library: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
    +InputFormat: org.apache.hadoop.hive.ql.io.RCFileInputFormat
    +OutputFormat: org.apache.hadoop.hive.ql.io.RCFileOutputFormat
    +Compressed: No
    +Num Buckets: -1
    +Bucket Columns: []
    +Sort Columns: []
    +Stored As SubDirectories: Yes
    +Skewed Columns: [key, value]
    +Skewed Values: [[484, val_484], [51, val_14], [103, val_103]]
    +#### A masked pattern was here ####
    +Skewed Value to Truncated Path: {[484, val_484]=/list_bucketing_static_part/ds=2008-04-08/hr=11/key=484/value=val_484, [103, val_103]=/list_bucketing_static_part/ds=2008-04-08/hr=11/key=103/value=val_103}
    +Storage Desc Params:
    + serialization.format 1
    +PREHOOK: query: select count(1) from srcpart where ds = '2008-04-08'
    +PREHOOK: type: QUERY
    +PREHOOK: Input: default@srcpart
    +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11
    +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12
    +#### A masked pattern was here ####
    +POSTHOOK: query: select count(1) from srcpart where ds = '2008-04-08'
    +POSTHOOK: type: QUERY
    +POSTHOOK: Input: default@srcpart
    +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11
    +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12
    +#### A masked pattern was here ####
    +1000
    +PREHOOK: query: select count(*) from list_bucketing_static_part
    +PREHOOK: type: QUERY
    +PREHOOK: Input: default@list_bucketing_static_part
    +PREHOOK: Input: default@list_bucketing_static_part@ds=2008-04-08/hr=11
    +#### A masked pattern was here ####
    +POSTHOOK: query: select count(*) from list_bucketing_static_part
    +POSTHOOK: type: QUERY
    +POSTHOOK: Input: default@list_bucketing_static_part
    +POSTHOOK: Input: default@list_bucketing_static_part@ds=2008-04-08/hr=11
    +#### A masked pattern was here ####
    +1000
    +PREHOOK: query: explain extended
    +select * from list_bucketing_static_part where ds = '2008-04-08' and hr = '11' and key = '484' and value = 'val_484'
    +PREHOOK: type: QUERY
    +POSTHOOK: query: explain extended
    +select * from list_bucketing_static_part where ds = '2008-04-08' and hr = '11' and key = '484' and value = 'val_484'
    +POSTHOOK: type: QUERY
    +ABSTRACT SYNTAX TREE:
    +
    +TOK_QUERY
    + TOK_FROM
    + TOK_TABREF
    + TOK_TABNAME
    + list_bucketing_static_part
    + TOK_INSERT
    + TOK_DESTINATION
    + TOK_DIR
    + TOK_TMP_FILE
    + TOK_SELECT
    + TOK_SELEXPR
    + TOK_ALLCOLREF
    + TOK_WHERE
    + and
    + and
    + and
    + =
    + TOK_TABLE_OR_COL
    + ds
    + '2008-04-08'
    + =
    + TOK_TABLE_OR_COL
    + hr
    + '11'
    + =
    + TOK_TABLE_OR_COL
    + key
    + '484'
    + =
    + TOK_TABLE_OR_COL
    + value
    + 'val_484'
    +
    +
    +STAGE DEPENDENCIES:
    + Stage-1 is a root stage
    + Stage-0 depends on stages: Stage-1
    +
    +STAGE PLANS:
    + Stage: Stage-1
    + Map Reduce
    + Map Operator Tree:
    + TableScan
    + alias: list_bucketing_static_part
    + Statistics: Num rows: 1000 Data size: 9624 Basic stats: COMPLETE Column stats: NONE
    + GatherStats: false
    + Filter Operator
    + isSamplingPred: false
    + predicate: ((key = '484') and (value = 'val_484')) (type: boolean)
    + Statistics: Num rows: 250 Data size: 2406 Basic stats: COMPLETE Column stats: NONE
    + Select Operator
    + expressions: '484' (type: string), 'val_484' (type: string), '2008-04-08' (type: string), '11' (type: string)
    + outputColumnNames: _col0, _col1, _col2, _col3
    + Statistics: Num rows: 250 Data size: 2406 Basic stats: COMPLETE Column stats: NONE
    + File Output Operator
    + compressed: false
    + GlobalTableId: 0
    +#### A masked pattern was here ####
    + NumFilesPerFileSink: 1
    + Statistics: Num rows: 250 Data size: 2406 Basic stats: COMPLETE Column stats: NONE
    +#### A masked pattern was here ####
    + table:
    + input format: org.apache.hadoop.mapred.TextInputFormat
    + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
    + properties:
    + columns _col0,_col1,_col2,_col3
    + columns.types string:string:string:string
    + escape.delim \
    + hive.serialization.extend.nesting.levels true
    + serialization.format 1
    + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
    + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
    + TotalFiles: 1
    + GatherStats: false
    + MultiFileSpray: false
    + Path -> Alias:
    +#### A masked pattern was here ####
    + Path -> Partition:
    +#### A masked pattern was here ####
    + Partition
    + base file name: value=val_484
    + input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat
    + output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat
    + partition values:
    + ds 2008-04-08
    + hr 11
    + properties:
    + COLUMN_STATS_ACCURATE true
    + bucket_count -1
    + columns key,value
    + columns.comments
    + columns.types string:string
    +#### A masked pattern was here ####
    + name default.list_bucketing_static_part
    + numFiles 4
    + numRows 1000
    + partition_columns ds/hr
    + partition_columns.types string:string
    + rawDataSize 9624
    + serialization.ddl struct list_bucketing_static_part { string key, string value}
    + serialization.format 1
    + serialization.lib org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
    + totalSize 10786
    +#### A masked pattern was here ####
    + serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
    +
    + input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat
    + output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat
    + properties:
    + bucket_count -1
    + columns key,value
    + columns.comments
    + columns.types string:string
    +#### A masked pattern was here ####
    + name default.list_bucketing_static_part
    + partition_columns ds/hr
    + partition_columns.types string:string
    + serialization.ddl struct list_bucketing_static_part { string key, string value}
    + serialization.format 1
    + serialization.lib org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
    +#### A masked pattern was here ####
    + serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
    + name: default.list_bucketing_static_part
    + name: default.list_bucketing_static_part
    + Truncated Path -> Alias:
    + /list_bucketing_static_part/ds=2008-04-08/hr=11/key=484/value=val_484 [list_bucketing_static_part]
    +
    + Stage: Stage-0
    + Fetch Operator
    + limit: -1
    + Processor Tree:
    + ListSink
    +
    +PREHOOK: query: select * from list_bucketing_static_part where ds = '2008-04-08' and hr = '11' and key = '484' and value = 'val_484'
    +PREHOOK: type: QUERY
    +PREHOOK: Input: default@list_bucketing_static_part
    +PREHOOK: Input: default@list_bucketing_static_part@ds=2008-04-08/hr=11
    +#### A masked pattern was here ####
    +POSTHOOK: query: select * from list_bucketing_static_part where ds = '2008-04-08' and hr = '11' and key = '484' and value = 'val_484'
    +POSTHOOK: type: QUERY
    +POSTHOOK: Input: default@list_bucketing_static_part
    +POSTHOOK: Input: default@list_bucketing_static_part@ds=2008-04-08/hr=11
    +#### A masked pattern was here ####
    +484 val_484 2008-04-08 11
    +484 val_484 2008-04-08 11
    +PREHOOK: query: select * from srcpart where ds = '2008-04-08' and key = '484' and value = 'val_484'
    +PREHOOK: type: QUERY
    +PREHOOK: Input: default@srcpart
    +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11
    +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12
    +#### A masked pattern was here ####
    +POSTHOOK: query: select * from srcpart where ds = '2008-04-08' and key = '484' and value = 'val_484'
    +POSTHOOK: type: QUERY
    +POSTHOOK: Input: default@srcpart
    +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11
    +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12
    +#### A masked pattern was here ####
    +484 val_484 2008-04-08 11
    +484 val_484 2008-04-08 12
    +PREHOOK: query: -- clean up
    +drop table list_bucketing_static_part
    +PREHOOK: type: DROPTABLE
    +PREHOOK: Input: default@list_bucketing_static_part
    +PREHOOK: Output: default@list_bucketing_static_part
    +POSTHOOK: query: -- clean up
    +drop table list_bucketing_static_part
    +POSTHOOK: type: DROPTABLE
    +POSTHOOK: Input: default@list_bucketing_static_part
    +POSTHOOK: Output: default@list_bucketing_static_part
  • Brock at Dec 25, 2014 at 6:06 pm
    Added: hive/trunk/ql/src/test/results/clientpositive/list_bucket_dml_9.q.java1.8.out
    URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/list_bucket_dml_9.q.java1.8.out?rev=1647912&view=auto
    ==============================================================================
    --- hive/trunk/ql/src/test/results/clientpositive/list_bucket_dml_9.q.java1.8.out (added)
    +++ hive/trunk/ql/src/test/results/clientpositive/list_bucket_dml_9.q.java1.8.out Thu Dec 25 18:06:30 2014
    @@ -0,0 +1,951 @@
    +PREHOOK: query: -- INCLUDE_HADOOP_MAJOR_VERSIONS(0.23)
    +-- SORT_QUERY_RESULTS
    +-- JAVA_VERSION_SPECIFIC_OUTPUT
    +
    +-- list bucketing DML: static partition. multiple skewed columns. merge.
    +-- ds=2008-04-08/hr=11/HIVE_DEFAULT_LIST_BUCKETING_DIR_NAME:
    +-- 5263 000000_0
    +-- 5263 000001_0
    +-- ds=2008-04-08/hr=11/key=103:
    +-- 99 000000_0
    +-- 99 000001_0
    +-- after merge
    +-- 142 000000_0
    +-- ds=2008-04-08/hr=11/key=484:
    +-- 87 000000_0
    +-- 87 000001_0
    +-- after merge
    +-- 118 000001_0
    +
    +-- create a skewed table
    +create table list_bucketing_static_part (key String, value String)
    + partitioned by (ds String, hr String)
    + skewed by (key) on ('484','103')
    + stored as DIRECTORIES
    + STORED AS RCFILE
    +PREHOOK: type: CREATETABLE
    +PREHOOK: Output: database:default
    +PREHOOK: Output: default@list_bucketing_static_part
    +POSTHOOK: query: -- INCLUDE_HADOOP_MAJOR_VERSIONS(0.23)
    +-- SORT_QUERY_RESULTS
    +-- JAVA_VERSION_SPECIFIC_OUTPUT
    +
    +-- list bucketing DML: static partition. multiple skewed columns. merge.
    +-- ds=2008-04-08/hr=11/HIVE_DEFAULT_LIST_BUCKETING_DIR_NAME:
    +-- 5263 000000_0
    +-- 5263 000001_0
    +-- ds=2008-04-08/hr=11/key=103:
    +-- 99 000000_0
    +-- 99 000001_0
    +-- after merge
    +-- 142 000000_0
    +-- ds=2008-04-08/hr=11/key=484:
    +-- 87 000000_0
    +-- 87 000001_0
    +-- after merge
    +-- 118 000001_0
    +
    +-- create a skewed table
    +create table list_bucketing_static_part (key String, value String)
    + partitioned by (ds String, hr String)
    + skewed by (key) on ('484','103')
    + stored as DIRECTORIES
    + STORED AS RCFILE
    +POSTHOOK: type: CREATETABLE
    +POSTHOOK: Output: database:default
    +POSTHOOK: Output: default@list_bucketing_static_part
    +PREHOOK: query: -- list bucketing DML without merge. use bucketize to generate a few small files.
    +explain extended
    +insert overwrite table list_bucketing_static_part partition (ds = '2008-04-08', hr = '11')
    +select key, value from srcpart where ds = '2008-04-08'
    +PREHOOK: type: QUERY
    +POSTHOOK: query: -- list bucketing DML without merge. use bucketize to generate a few small files.
    +explain extended
    +insert overwrite table list_bucketing_static_part partition (ds = '2008-04-08', hr = '11')
    +select key, value from srcpart where ds = '2008-04-08'
    +POSTHOOK: type: QUERY
    +ABSTRACT SYNTAX TREE:
    +
    +TOK_QUERY
    + TOK_FROM
    + TOK_TABREF
    + TOK_TABNAME
    + srcpart
    + TOK_INSERT
    + TOK_DESTINATION
    + TOK_TAB
    + TOK_TABNAME
    + list_bucketing_static_part
    + TOK_PARTSPEC
    + TOK_PARTVAL
    + ds
    + '2008-04-08'
    + TOK_PARTVAL
    + hr
    + '11'
    + TOK_SELECT
    + TOK_SELEXPR
    + TOK_TABLE_OR_COL
    + key
    + TOK_SELEXPR
    + TOK_TABLE_OR_COL
    + value
    + TOK_WHERE
    + =
    + TOK_TABLE_OR_COL
    + ds
    + '2008-04-08'
    +
    +
    +STAGE DEPENDENCIES:
    + Stage-1 is a root stage
    + Stage-0 depends on stages: Stage-1
    + Stage-2 depends on stages: Stage-0
    +
    +STAGE PLANS:
    + Stage: Stage-1
    + Map Reduce
    + Map Operator Tree:
    + TableScan
    + alias: srcpart
    + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE
    + GatherStats: false
    + Select Operator
    + expressions: key (type: string), value (type: string)
    + outputColumnNames: _col0, _col1
    + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE
    + File Output Operator
    + compressed: false
    + GlobalTableId: 1
    +#### A masked pattern was here ####
    + NumFilesPerFileSink: 1
    + Static Partition Specification: ds=2008-04-08/hr=11/
    + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE
    +#### A masked pattern was here ####
    + table:
    + input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat
    + output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat
    + properties:
    + bucket_count -1
    + columns key,value
    + columns.comments
    + columns.types string:string
    +#### A masked pattern was here ####
    + name default.list_bucketing_static_part
    + partition_columns ds/hr
    + partition_columns.types string:string
    + serialization.ddl struct list_bucketing_static_part { string key, string value}
    + serialization.format 1
    + serialization.lib org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
    +#### A masked pattern was here ####
    + serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
    + name: default.list_bucketing_static_part
    + TotalFiles: 1
    + GatherStats: true
    + MultiFileSpray: false
    + Path -> Alias:
    +#### A masked pattern was here ####
    + Path -> Partition:
    +#### A masked pattern was here ####
    + Partition
    + base file name: hr=11
    + input format: org.apache.hadoop.mapred.TextInputFormat
    + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
    + partition values:
    + ds 2008-04-08
    + hr 11
    + properties:
    + COLUMN_STATS_ACCURATE true
    + bucket_count -1
    + columns key,value
    + columns.comments 'default','default'
    + columns.types string:string
    +#### A masked pattern was here ####
    + name default.srcpart
    + numFiles 1
    + numRows 500
    + partition_columns ds/hr
    + partition_columns.types string:string
    + rawDataSize 5312
    + serialization.ddl struct srcpart { string key, string value}
    + serialization.format 1
    + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
    + totalSize 5812
    +#### A masked pattern was here ####
    + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
    +
    + input format: org.apache.hadoop.mapred.TextInputFormat
    + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
    + properties:
    + bucket_count -1
    + columns key,value
    + columns.comments 'default','default'
    + columns.types string:string
    +#### A masked pattern was here ####
    + name default.srcpart
    + partition_columns ds/hr
    + partition_columns.types string:string
    + serialization.ddl struct srcpart { string key, string value}
    + serialization.format 1
    + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
    +#### A masked pattern was here ####
    + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
    + name: default.srcpart
    + name: default.srcpart
    +#### A masked pattern was here ####
    + Partition
    + base file name: hr=12
    + input format: org.apache.hadoop.mapred.TextInputFormat
    + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
    + partition values:
    + ds 2008-04-08
    + hr 12
    + properties:
    + COLUMN_STATS_ACCURATE true
    + bucket_count -1
    + columns key,value
    + columns.comments 'default','default'
    + columns.types string:string
    +#### A masked pattern was here ####
    + name default.srcpart
    + numFiles 1
    + numRows 500
    + partition_columns ds/hr
    + partition_columns.types string:string
    + rawDataSize 5312
    + serialization.ddl struct srcpart { string key, string value}
    + serialization.format 1
    + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
    + totalSize 5812
    +#### A masked pattern was here ####
    + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
    +
    + input format: org.apache.hadoop.mapred.TextInputFormat
    + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
    + properties:
    + bucket_count -1
    + columns key,value
    + columns.comments 'default','default'
    + columns.types string:string
    +#### A masked pattern was here ####
    + name default.srcpart
    + partition_columns ds/hr
    + partition_columns.types string:string
    + serialization.ddl struct srcpart { string key, string value}
    + serialization.format 1
    + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
    +#### A masked pattern was here ####
    + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
    + name: default.srcpart
    + name: default.srcpart
    + Truncated Path -> Alias:
    + /srcpart/ds=2008-04-08/hr=11 [$hdt$_0:srcpart]
    + /srcpart/ds=2008-04-08/hr=12 [$hdt$_0:srcpart]
    +
    + Stage: Stage-0
    + Move Operator
    + tables:
    + partition:
    + ds 2008-04-08
    + hr 11
    + replace: true
    +#### A masked pattern was here ####
    + table:
    + input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat
    + output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat
    + properties:
    + bucket_count -1
    + columns key,value
    + columns.comments
    + columns.types string:string
    +#### A masked pattern was here ####
    + name default.list_bucketing_static_part
    + partition_columns ds/hr
    + partition_columns.types string:string
    + serialization.ddl struct list_bucketing_static_part { string key, string value}
    + serialization.format 1
    + serialization.lib org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
    +#### A masked pattern was here ####
    + serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
    + name: default.list_bucketing_static_part
    +
    + Stage: Stage-2
    + Stats-Aggr Operator
    +#### A masked pattern was here ####
    +
    +PREHOOK: query: insert overwrite table list_bucketing_static_part partition (ds = '2008-04-08', hr = '11')
    +select key, value from srcpart where ds = '2008-04-08'
    +PREHOOK: type: QUERY
    +PREHOOK: Input: default@srcpart
    +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11
    +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12
    +PREHOOK: Output: default@list_bucketing_static_part@ds=2008-04-08/hr=11
    +POSTHOOK: query: insert overwrite table list_bucketing_static_part partition (ds = '2008-04-08', hr = '11')
    +select key, value from srcpart where ds = '2008-04-08'
    +POSTHOOK: type: QUERY
    +POSTHOOK: Input: default@srcpart
    +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11
    +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12
    +POSTHOOK: Output: default@list_bucketing_static_part@ds=2008-04-08/hr=11
    +POSTHOOK: Lineage: list_bucketing_static_part PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ]
    +POSTHOOK: Lineage: list_bucketing_static_part PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ]
    +PREHOOK: query: -- check DML result
    +show partitions list_bucketing_static_part
    +PREHOOK: type: SHOWPARTITIONS
    +PREHOOK: Input: default@list_bucketing_static_part
    +POSTHOOK: query: -- check DML result
    +show partitions list_bucketing_static_part
    +POSTHOOK: type: SHOWPARTITIONS
    +POSTHOOK: Input: default@list_bucketing_static_part
    +ds=2008-04-08/hr=11
    +PREHOOK: query: desc formatted list_bucketing_static_part partition (ds='2008-04-08', hr='11')
    +PREHOOK: type: DESCTABLE
    +PREHOOK: Input: default@list_bucketing_static_part
    +POSTHOOK: query: desc formatted list_bucketing_static_part partition (ds='2008-04-08', hr='11')
    +POSTHOOK: type: DESCTABLE
    +POSTHOOK: Input: default@list_bucketing_static_part
    +# col_name data_type comment
    +
    +key string
    +value string
    +
    +# Partition Information
    +# col_name data_type comment
    +
    +ds string
    +hr string
    +
    +# Detailed Partition Information
    +Partition Value: [2008-04-08, 11]
    +Database: default
    +Table: list_bucketing_static_part
    +#### A masked pattern was here ####
    +Protect Mode: None
    +#### A masked pattern was here ####
    +Partition Parameters:
    + COLUMN_STATS_ACCURATE true
    + numFiles 6
    + numRows 1000
    + rawDataSize 9624
    + totalSize 10898
    +#### A masked pattern was here ####
    +
    +# Storage Information
    +SerDe Library: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
    +InputFormat: org.apache.hadoop.hive.ql.io.RCFileInputFormat
    +OutputFormat: org.apache.hadoop.hive.ql.io.RCFileOutputFormat
    +Compressed: No
    +Num Buckets: -1
    +Bucket Columns: []
    +Sort Columns: []
    +Stored As SubDirectories: Yes
    +Skewed Columns: [key]
    +Skewed Values: [[484], [103]]
    +#### A masked pattern was here ####
    +Skewed Value to Truncated Path: {[103]=/list_bucketing_static_part/ds=2008-04-08/hr=11/key=103, [484]=/list_bucketing_static_part/ds=2008-04-08/hr=11/key=484}
    +Storage Desc Params:
    + serialization.format 1
    +PREHOOK: query: -- list bucketing DML with merge. use bucketize to generate a few small files.
    +explain extended
    +insert overwrite table list_bucketing_static_part partition (ds = '2008-04-08', hr = '11')
    +select key, value from srcpart where ds = '2008-04-08'
    +PREHOOK: type: QUERY
    +POSTHOOK: query: -- list bucketing DML with merge. use bucketize to generate a few small files.
    +explain extended
    +insert overwrite table list_bucketing_static_part partition (ds = '2008-04-08', hr = '11')
    +select key, value from srcpart where ds = '2008-04-08'
    +POSTHOOK: type: QUERY
    +ABSTRACT SYNTAX TREE:
    +
    +TOK_QUERY
    + TOK_FROM
    + TOK_TABREF
    + TOK_TABNAME
    + srcpart
    + TOK_INSERT
    + TOK_DESTINATION
    + TOK_TAB
    + TOK_TABNAME
    + list_bucketing_static_part
    + TOK_PARTSPEC
    + TOK_PARTVAL
    + ds
    + '2008-04-08'
    + TOK_PARTVAL
    + hr
    + '11'
    + TOK_SELECT
    + TOK_SELEXPR
    + TOK_TABLE_OR_COL
    + key
    + TOK_SELEXPR
    + TOK_TABLE_OR_COL
    + value
    + TOK_WHERE
    + =
    + TOK_TABLE_OR_COL
    + ds
    + '2008-04-08'
    +
    +
    +STAGE DEPENDENCIES:
    + Stage-1 is a root stage
    + Stage-7 depends on stages: Stage-1 , consists of Stage-4, Stage-3, Stage-5
    + Stage-4
    + Stage-0 depends on stages: Stage-4, Stage-3, Stage-6
    + Stage-2 depends on stages: Stage-0
    + Stage-3
    + Stage-5
    + Stage-6 depends on stages: Stage-5
    +
    +STAGE PLANS:
    + Stage: Stage-1
    + Map Reduce
    + Map Operator Tree:
    + TableScan
    + alias: srcpart
    + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE
    + GatherStats: false
    + Select Operator
    + expressions: key (type: string), value (type: string)
    + outputColumnNames: _col0, _col1
    + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE
    + File Output Operator
    + compressed: false
    + GlobalTableId: 1
    +#### A masked pattern was here ####
    + NumFilesPerFileSink: 1
    + Static Partition Specification: ds=2008-04-08/hr=11/
    + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE
    +#### A masked pattern was here ####
    + table:
    + input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat
    + output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat
    + properties:
    + bucket_count -1
    + columns key,value
    + columns.comments
    + columns.types string:string
    +#### A masked pattern was here ####
    + name default.list_bucketing_static_part
    + partition_columns.types string:string
    + serialization.ddl struct list_bucketing_static_part { string key, string value}
    + serialization.format 1
    + serialization.lib org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
    +#### A masked pattern was here ####
    + serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
    + name: default.list_bucketing_static_part
    + TotalFiles: 1
    + GatherStats: true
    + MultiFileSpray: false
    + Path -> Alias:
    +#### A masked pattern was here ####
    + Path -> Partition:
    +#### A masked pattern was here ####
    + Partition
    + base file name: hr=11
    + input format: org.apache.hadoop.mapred.TextInputFormat
    + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
    + partition values:
    + ds 2008-04-08
    + hr 11
    + properties:
    + COLUMN_STATS_ACCURATE true
    + bucket_count -1
    + columns key,value
    + columns.comments 'default','default'
    + columns.types string:string
    +#### A masked pattern was here ####
    + name default.srcpart
    + numFiles 1
    + numRows 500
    + partition_columns ds/hr
    + partition_columns.types string:string
    + rawDataSize 5312
    + serialization.ddl struct srcpart { string key, string value}
    + serialization.format 1
    + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
    + totalSize 5812
    +#### A masked pattern was here ####
    + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
    +
    + input format: org.apache.hadoop.mapred.TextInputFormat
    + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
    + properties:
    + bucket_count -1
    + columns key,value
    + columns.comments 'default','default'
    + columns.types string:string
    +#### A masked pattern was here ####
    + name default.srcpart
    + partition_columns ds/hr
    + partition_columns.types string:string
    + serialization.ddl struct srcpart { string key, string value}
    + serialization.format 1
    + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
    +#### A masked pattern was here ####
    + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
    + name: default.srcpart
    + name: default.srcpart
    +#### A masked pattern was here ####
    + Partition
    + base file name: hr=12
    + input format: org.apache.hadoop.mapred.TextInputFormat
    + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
    + partition values:
    + ds 2008-04-08
    + hr 12
    + properties:
    + COLUMN_STATS_ACCURATE true
    + bucket_count -1
    + columns key,value
    + columns.comments 'default','default'
    + columns.types string:string
    +#### A masked pattern was here ####
    + name default.srcpart
    + numFiles 1
    + numRows 500
    + partition_columns ds/hr
    + partition_columns.types string:string
    + rawDataSize 5312
    + serialization.ddl struct srcpart { string key, string value}
    + serialization.format 1
    + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
    + totalSize 5812
    +#### A masked pattern was here ####
    + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
    +
    + input format: org.apache.hadoop.mapred.TextInputFormat
    + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
    + properties:
    + bucket_count -1
    + columns key,value
    + columns.comments 'default','default'
    + columns.types string:string
    +#### A masked pattern was here ####
    + name default.srcpart
    + partition_columns ds/hr
    + partition_columns.types string:string
    + serialization.ddl struct srcpart { string key, string value}
    + serialization.format 1
    + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
    +#### A masked pattern was here ####
    + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
    + name: default.srcpart
    + name: default.srcpart
    + Truncated Path -> Alias:
    + /srcpart/ds=2008-04-08/hr=11 [$hdt$_0:srcpart]
    + /srcpart/ds=2008-04-08/hr=12 [$hdt$_0:srcpart]
    +
    + Stage: Stage-7
    + Conditional Operator
    +
    + Stage: Stage-4
    + Move Operator
    + files:
    + hdfs directory: true
    +#### A masked pattern was here ####
    +
    + Stage: Stage-0
    + Move Operator
    + tables:
    + partition:
    + ds 2008-04-08
    + hr 11
    + replace: true
    +#### A masked pattern was here ####
    + table:
    + input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat
    + output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat
    + properties:
    + bucket_count -1
    + columns key,value
    + columns.comments
    + columns.types string:string
    +#### A masked pattern was here ####
    + name default.list_bucketing_static_part
    + partition_columns.types string:string
    + serialization.ddl struct list_bucketing_static_part { string key, string value}
    + serialization.format 1
    + serialization.lib org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
    +#### A masked pattern was here ####
    + serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
    + name: default.list_bucketing_static_part
    +
    + Stage: Stage-2
    + Stats-Aggr Operator
    +#### A masked pattern was here ####
    +
    + Stage: Stage-3
    + Merge File Operator
    + Map Operator Tree:
    + RCFile Merge Operator
    + merge level: block
    + Path -> Alias:
    +#### A masked pattern was here ####
    + Path -> Partition:
    +#### A masked pattern was here ####
    + Partition
    + input format: org.apache.hadoop.hive.ql.io.rcfile.merge.RCFileBlockMergeInputFormat
    + output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat
    + properties:
    + bucket_count -1
    + columns key,value
    + columns.comments
    + columns.types string:string
    +#### A masked pattern was here ####
    + name default.list_bucketing_static_part
    + partition_columns.types string:string
    + serialization.ddl struct list_bucketing_static_part { string key, string value}
    + serialization.format 1
    + serialization.lib org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
    +#### A masked pattern was here ####
    + serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
    +
    + input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat
    + output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat
    + properties:
    + bucket_count -1
    + columns key,value
    + columns.comments
    + columns.types string:string
    +#### A masked pattern was here ####
    + name default.list_bucketing_static_part
    + partition_columns.types string:string
    + serialization.ddl struct list_bucketing_static_part { string key, string value}
    + serialization.format 1
    + serialization.lib org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
    +#### A masked pattern was here ####
    + serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
    + name: default.list_bucketing_static_part
    + name: default.list_bucketing_static_part
    + input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat
    + Truncated Path -> Alias:
    +#### A masked pattern was here ####
    +
    + Stage: Stage-5
    + Merge File Operator
    + Map Operator Tree:
    + RCFile Merge Operator
    + merge level: block
    + Path -> Alias:
    +#### A masked pattern was here ####
    + Path -> Partition:
    +#### A masked pattern was here ####
    + Partition
    + input format: org.apache.hadoop.hive.ql.io.rcfile.merge.RCFileBlockMergeInputFormat
    + output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat
    + properties:
    + bucket_count -1
    + columns key,value
    + columns.comments
    + columns.types string:string
    +#### A masked pattern was here ####
    + name default.list_bucketing_static_part
    + partition_columns.types string:string
    + serialization.ddl struct list_bucketing_static_part { string key, string value}
    + serialization.format 1
    + serialization.lib org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
    +#### A masked pattern was here ####
    + serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
    +
    + input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat
    + output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat
    + properties:
    + bucket_count -1
    + columns key,value
    + columns.comments
    + columns.types string:string
    +#### A masked pattern was here ####
    + name default.list_bucketing_static_part
    + partition_columns.types string:string
    + serialization.ddl struct list_bucketing_static_part { string key, string value}
    + serialization.format 1
    + serialization.lib org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
    +#### A masked pattern was here ####
    + serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
    + name: default.list_bucketing_static_part
    + name: default.list_bucketing_static_part
    + input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat
    + Truncated Path -> Alias:
    +#### A masked pattern was here ####
    +
    + Stage: Stage-6
    + Move Operator
    + files:
    + hdfs directory: true
    +#### A masked pattern was here ####
    +
    +PREHOOK: query: insert overwrite table list_bucketing_static_part partition (ds = '2008-04-08', hr = '11')
    +select key, value from srcpart where ds = '2008-04-08'
    +PREHOOK: type: QUERY
    +PREHOOK: Input: default@srcpart
    +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11
    +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12
    +PREHOOK: Output: default@list_bucketing_static_part@ds=2008-04-08/hr=11
    +POSTHOOK: query: insert overwrite table list_bucketing_static_part partition (ds = '2008-04-08', hr = '11')
    +select key, value from srcpart where ds = '2008-04-08'
    +POSTHOOK: type: QUERY
    +POSTHOOK: Input: default@srcpart
    +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11
    +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12
    +POSTHOOK: Output: default@list_bucketing_static_part@ds=2008-04-08/hr=11
    +POSTHOOK: Lineage: list_bucketing_static_part PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ]
    +POSTHOOK: Lineage: list_bucketing_static_part PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ]
    +PREHOOK: query: -- check DML result
    +show partitions list_bucketing_static_part
    +PREHOOK: type: SHOWPARTITIONS
    +PREHOOK: Input: default@list_bucketing_static_part
    +POSTHOOK: query: -- check DML result
    +show partitions list_bucketing_static_part
    +POSTHOOK: type: SHOWPARTITIONS
    +POSTHOOK: Input: default@list_bucketing_static_part
    +ds=2008-04-08/hr=11
    +PREHOOK: query: desc formatted list_bucketing_static_part partition (ds='2008-04-08', hr='11')
    +PREHOOK: type: DESCTABLE
    +PREHOOK: Input: default@list_bucketing_static_part
    +POSTHOOK: query: desc formatted list_bucketing_static_part partition (ds='2008-04-08', hr='11')
    +POSTHOOK: type: DESCTABLE
    +POSTHOOK: Input: default@list_bucketing_static_part
    +# col_name data_type comment
    +
    +key string
    +value string
    +
    +# Partition Information
    +# col_name data_type comment
    +
    +ds string
    +hr string
    +
    +# Detailed Partition Information
    +Partition Value: [2008-04-08, 11]
    +Database: default
    +Table: list_bucketing_static_part
    +#### A masked pattern was here ####
    +Protect Mode: None
    +#### A masked pattern was here ####
    +Partition Parameters:
    + COLUMN_STATS_ACCURATE true
    + numFiles 4
    + numRows 1000
    + rawDataSize 9624
    + totalSize 10786
    +#### A masked pattern was here ####
    +
    +# Storage Information
    +SerDe Library: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
    +InputFormat: org.apache.hadoop.hive.ql.io.RCFileInputFormat
    +OutputFormat: org.apache.hadoop.hive.ql.io.RCFileOutputFormat
    +Compressed: No
    +Num Buckets: -1
    +Bucket Columns: []
    +Sort Columns: []
    +Stored As SubDirectories: Yes
    +Skewed Columns: [key]
    +Skewed Values: [[484], [103]]
    +#### A masked pattern was here ####
    +Skewed Value to Truncated Path: {[103]=/list_bucketing_static_part/ds=2008-04-08/hr=11/key=103, [484]=/list_bucketing_static_part/ds=2008-04-08/hr=11/key=484}
    +Storage Desc Params:
    + serialization.format 1
    +PREHOOK: query: select count(1) from srcpart where ds = '2008-04-08'
    +PREHOOK: type: QUERY
    +PREHOOK: Input: default@srcpart
    +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11
    +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12
    +#### A masked pattern was here ####
    +POSTHOOK: query: select count(1) from srcpart where ds = '2008-04-08'
    +POSTHOOK: type: QUERY
    +POSTHOOK: Input: default@srcpart
    +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11
    +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12
    +#### A masked pattern was here ####
    +1000
    +PREHOOK: query: select count(*) from list_bucketing_static_part
    +PREHOOK: type: QUERY
    +PREHOOK: Input: default@list_bucketing_static_part
    +PREHOOK: Input: default@list_bucketing_static_part@ds=2008-04-08/hr=11
    +#### A masked pattern was here ####
    +POSTHOOK: query: select count(*) from list_bucketing_static_part
    +POSTHOOK: type: QUERY
    +POSTHOOK: Input: default@list_bucketing_static_part
    +POSTHOOK: Input: default@list_bucketing_static_part@ds=2008-04-08/hr=11
    +#### A masked pattern was here ####
    +1000
    +PREHOOK: query: explain extended
    +select * from list_bucketing_static_part where ds = '2008-04-08' and hr = '11' and key = '484' and value = 'val_484'
    +PREHOOK: type: QUERY
    +POSTHOOK: query: explain extended
    +select * from list_bucketing_static_part where ds = '2008-04-08' and hr = '11' and key = '484' and value = 'val_484'
    +POSTHOOK: type: QUERY
    +ABSTRACT SYNTAX TREE:
    +
    +TOK_QUERY
    + TOK_FROM
    + TOK_TABREF
    + TOK_TABNAME
    + list_bucketing_static_part
    + TOK_INSERT
    + TOK_DESTINATION
    + TOK_DIR
    + TOK_TMP_FILE
    + TOK_SELECT
    + TOK_SELEXPR
    + TOK_ALLCOLREF
    + TOK_WHERE
    + and
    + and
    + and
    + =
    + TOK_TABLE_OR_COL
    + ds
    + '2008-04-08'
    + =
    + TOK_TABLE_OR_COL
    + hr
    + '11'
    + =
    + TOK_TABLE_OR_COL
    + key
    + '484'
    + =
    + TOK_TABLE_OR_COL
    + value
    + 'val_484'
    +
    +
    +STAGE DEPENDENCIES:
    + Stage-1 is a root stage
    + Stage-0 depends on stages: Stage-1
    +
    +STAGE PLANS:
    + Stage: Stage-1
    + Map Reduce
    + Map Operator Tree:
    + TableScan
    + alias: list_bucketing_static_part
    + Statistics: Num rows: 1000 Data size: 9624 Basic stats: COMPLETE Column stats: NONE
    + GatherStats: false
    + Filter Operator
    + isSamplingPred: false
    + predicate: ((key = '484') and (value = 'val_484')) (type: boolean)
    + Statistics: Num rows: 250 Data size: 2406 Basic stats: COMPLETE Column stats: NONE
    + Select Operator
    + expressions: '484' (type: string), 'val_484' (type: string), '2008-04-08' (type: string), '11' (type: string)
    + outputColumnNames: _col0, _col1, _col2, _col3
    + Statistics: Num rows: 250 Data size: 2406 Basic stats: COMPLETE Column stats: NONE
    + File Output Operator
    + compressed: false
    + GlobalTableId: 0
    +#### A masked pattern was here ####
    + NumFilesPerFileSink: 1
    + Statistics: Num rows: 250 Data size: 2406 Basic stats: COMPLETE Column stats: NONE
    +#### A masked pattern was here ####
    + table:
    + input format: org.apache.hadoop.mapred.TextInputFormat
    + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
    + properties:
    + columns _col0,_col1,_col2,_col3
    + columns.types string:string:string:string
    + escape.delim \
    + hive.serialization.extend.nesting.levels true
    + serialization.format 1
    + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
    + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
    + TotalFiles: 1
    + GatherStats: false
    + MultiFileSpray: false
    + Path -> Alias:
    +#### A masked pattern was here ####
    + Path -> Partition:
    +#### A masked pattern was here ####
    + Partition
    + base file name: key=484
    + input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat
    + output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat
    + partition values:
    + ds 2008-04-08
    + hr 11
    + properties:
    + COLUMN_STATS_ACCURATE true
    + bucket_count -1
    + columns key,value
    + columns.comments
    + columns.types string:string
    +#### A masked pattern was here ####
    + name default.list_bucketing_static_part
    + numFiles 4
    + numRows 1000
    + partition_columns ds/hr
    + partition_columns.types string:string
    + rawDataSize 9624
    + serialization.ddl struct list_bucketing_static_part { string key, string value}
    + serialization.format 1
    + serialization.lib org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
    + totalSize 10786
    +#### A masked pattern was here ####
    + serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
    +
    + input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat
    + output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat
    + properties:
    + bucket_count -1
    + columns key,value
    + columns.comments
    + columns.types string:string
    +#### A masked pattern was here ####
    + name default.list_bucketing_static_part
    + partition_columns ds/hr
    + partition_columns.types string:string
    + serialization.ddl struct list_bucketing_static_part { string key, string value}
    + serialization.format 1
    + serialization.lib org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
    +#### A masked pattern was here ####
    + serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
    + name: default.list_bucketing_static_part
    + name: default.list_bucketing_static_part
    + Truncated Path -> Alias:
    + /list_bucketing_static_part/ds=2008-04-08/hr=11/key=484 [list_bucketing_static_part]
    +
    + Stage: Stage-0
    + Fetch Operator
    + limit: -1
    + Processor Tree:
    + ListSink
    +
    +PREHOOK: query: select * from list_bucketing_static_part where ds = '2008-04-08' and hr = '11' and key = '484' and value = 'val_484'
    +PREHOOK: type: QUERY
    +PREHOOK: Input: default@list_bucketing_static_part
    +PREHOOK: Input: default@list_bucketing_static_part@ds=2008-04-08/hr=11
    +#### A masked pattern was here ####
    +POSTHOOK: query: select * from list_bucketing_static_part where ds = '2008-04-08' and hr = '11' and key = '484' and value = 'val_484'
    +POSTHOOK: type: QUERY
    +POSTHOOK: Input: default@list_bucketing_static_part
    +POSTHOOK: Input: default@list_bucketing_static_part@ds=2008-04-08/hr=11
    +#### A masked pattern was here ####
    +484 val_484 2008-04-08 11
    +484 val_484 2008-04-08 11
    +PREHOOK: query: select * from srcpart where ds = '2008-04-08' and key = '484' and value = 'val_484'
    +PREHOOK: type: QUERY
    +PREHOOK: Input: default@srcpart
    +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11
    +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12
    +#### A masked pattern was here ####
    +POSTHOOK: query: select * from srcpart where ds = '2008-04-08' and key = '484' and value = 'val_484'
    +POSTHOOK: type: QUERY
    +POSTHOOK: Input: default@srcpart
    +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11
    +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12
    +#### A masked pattern was here ####
    +484 val_484 2008-04-08 11
    +484 val_484 2008-04-08 12
    +PREHOOK: query: -- clean up
    +drop table list_bucketing_static_part
    +PREHOOK: type: DROPTABLE
    +PREHOOK: Input: default@list_bucketing_static_part
    +PREHOOK: Output: default@list_bucketing_static_part
    +POSTHOOK: query: -- clean up
    +drop table list_bucketing_static_part
    +POSTHOOK: type: DROPTABLE
    +POSTHOOK: Input: default@list_bucketing_static_part
    +POSTHOOK: Output: default@list_bucketing_static_part

    Modified: hive/trunk/ql/src/test/results/clientpositive/partitions_json.q.out
    URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/partitions_json.q.out?rev=1647912&r1=1647911&r2=1647912&view=diff
    ==============================================================================
    --- hive/trunk/ql/src/test/results/clientpositive/partitions_json.q.out (original)
    +++ hive/trunk/ql/src/test/results/clientpositive/partitions_json.q.out Thu Dec 25 18:06:30 2014
    @@ -26,7 +26,7 @@ PREHOOK: Input: default@add_part_test
      POSTHOOK: query: SHOW PARTITIONS add_part_test
      POSTHOOK: type: SHOWPARTITIONS
      POSTHOOK: Input: default@add_part_test
    -{"partitions":[{"values":[{"columnName":"ds","columnValue":"2010-01-01"}],"name":"ds='2010-01-01'"}]}
    +{"partitions":[{"name":"ds='2010-01-01'","values":[{"columnName":"ds","columnValue":"2010-01-01"}]}]}
      PREHOOK: query: ALTER TABLE add_part_test ADD IF NOT EXISTS PARTITION (ds='2010-01-01')
      PREHOOK: type: ALTERTABLE_ADDPARTS
      PREHOOK: Output: default@add_part_test
    @@ -39,7 +39,7 @@ PREHOOK: Input: default@add_part_test
      POSTHOOK: query: SHOW PARTITIONS add_part_test
      POSTHOOK: type: SHOWPARTITIONS
      POSTHOOK: Input: default@add_part_test
    -{"partitions":[{"values":[{"columnName":"ds","columnValue":"2010-01-01"}],"name":"ds='2010-01-01'"}]}
    +{"partitions":[{"name":"ds='2010-01-01'","values":[{"columnName":"ds","columnValue":"2010-01-01"}]}]}
      PREHOOK: query: ALTER TABLE add_part_test ADD IF NOT EXISTS PARTITION (ds='2010-01-02')
      PREHOOK: type: ALTERTABLE_ADDPARTS
      PREHOOK: Output: default@add_part_test
    @@ -53,7 +53,7 @@ PREHOOK: Input: default@add_part_test
      POSTHOOK: query: SHOW PARTITIONS add_part_test
      POSTHOOK: type: SHOWPARTITIONS
      POSTHOOK: Input: default@add_part_test
    -{"partitions":[{"values":[{"columnName":"ds","columnValue":"2010-01-01"}],"name":"ds='2010-01-01'"},{"values":[{"columnName":"ds","columnValue":"2010-01-02"}],"name":"ds='2010-01-02'"}]}
    +{"partitions":[{"name":"ds='2010-01-01'","values":[{"columnName":"ds","columnValue":"2010-01-01"}]},{"name":"ds='2010-01-02'","values":[{"columnName":"ds","columnValue":"2010-01-02"}]}]}
      PREHOOK: query: SHOW TABLE EXTENDED LIKE add_part_test PARTITION (ds='2010-01-02')
      PREHOOK: type: SHOW_TABLESTATUS
      POSTHOOK: query: SHOW TABLE EXTENDED LIKE add_part_test PARTITION (ds='2010-01-02')

    Added: hive/trunk/ql/src/test/results/clientpositive/stats_list_bucket.q.java1.7.out
    URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/stats_list_bucket.q.java1.7.out?rev=1647912&view=auto
    ==============================================================================
    --- hive/trunk/ql/src/test/results/clientpositive/stats_list_bucket.q.java1.7.out (added)
    +++ hive/trunk/ql/src/test/results/clientpositive/stats_list_bucket.q.java1.7.out Thu Dec 25 18:06:30 2014
    @@ -0,0 +1,196 @@
    +PREHOOK: query: -- INCLUDE_HADOOP_MAJOR_VERSIONS(0.23)
    +-- JAVA_VERSION_SPECIFIC_OUTPUT
    +
    +drop table stats_list_bucket
    +PREHOOK: type: DROPTABLE
    +POSTHOOK: query: -- INCLUDE_HADOOP_MAJOR_VERSIONS(0.23)
    +-- JAVA_VERSION_SPECIFIC_OUTPUT
    +
    +drop table stats_list_bucket
    +POSTHOOK: type: DROPTABLE
    +PREHOOK: query: drop table stats_list_bucket_1
    +PREHOOK: type: DROPTABLE
    +POSTHOOK: query: drop table stats_list_bucket_1
    +POSTHOOK: type: DROPTABLE
    +PREHOOK: query: create table stats_list_bucket (
    + c1 string,
    + c2 string
    +) partitioned by (ds string, hr string)
    +skewed by (c1, c2) on (('466','val_466'),('287','val_287'),('82','val_82'))
    +stored as directories
    +stored as rcfile
    +PREHOOK: type: CREATETABLE
    +PREHOOK: Output: database:default
    +PREHOOK: Output: default@stats_list_bucket
    +POSTHOOK: query: create table stats_list_bucket (
    + c1 string,
    + c2 string
    +) partitioned by (ds string, hr string)
    +skewed by (c1, c2) on (('466','val_466'),('287','val_287'),('82','val_82'))
    +stored as directories
    +stored as rcfile
    +POSTHOOK: type: CREATETABLE
    +POSTHOOK: Output: database:default
    +POSTHOOK: Output: default@stats_list_bucket
    +PREHOOK: query: -- Make sure we use hashed IDs during stats publishing.
    +-- Try partitioned table with list bucketing.
    +-- The stats should show 500 rows loaded, as many rows as the src table has.
    +
    +insert overwrite table stats_list_bucket partition (ds = '2008-04-08', hr = '11')
    + select key, value from src
    +PREHOOK: type: QUERY
    +PREHOOK: Input: default@src
    +PREHOOK: Output: default@stats_list_bucket@ds=2008-04-08/hr=11
    +POSTHOOK: query: -- Make sure we use hashed IDs during stats publishing.
    +-- Try partitioned table with list bucketing.
    +-- The stats should show 500 rows loaded, as many rows as the src table has.
    +
    +insert overwrite table stats_list_bucket partition (ds = '2008-04-08', hr = '11')
    + select key, value from src
    +POSTHOOK: type: QUERY
    +POSTHOOK: Input: default@src
    +POSTHOOK: Output: default@stats_list_bucket@ds=2008-04-08/hr=11
    +POSTHOOK: Lineage: stats_list_bucket PARTITION(ds=2008-04-08,hr=11).c1 SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ]
    +POSTHOOK: Lineage: stats_list_bucket PARTITION(ds=2008-04-08,hr=11).c2 SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
    +PREHOOK: query: desc formatted stats_list_bucket partition (ds = '2008-04-08', hr = '11')
    +PREHOOK: type: DESCTABLE
    +PREHOOK: Input: default@stats_list_bucket
    +POSTHOOK: query: desc formatted stats_list_bucket partition (ds = '2008-04-08', hr = '11')
    +POSTHOOK: type: DESCTABLE
    +POSTHOOK: Input: default@stats_list_bucket
    +# col_name data_type comment
    +
    +c1 string
    +c2 string
    +
    +# Partition Information
    +# col_name data_type comment
    +
    +ds string
    +hr string
    +
    +# Detailed Partition Information
    +Partition Value: [2008-04-08, 11]
    +Database: default
    +Table: stats_list_bucket
    +#### A masked pattern was here ####
    +Protect Mode: None
    +#### A masked pattern was here ####
    +Partition Parameters:
    + COLUMN_STATS_ACCURATE true
    + numFiles 4
    + numRows 500
    + rawDataSize 4812
    + totalSize 5522
    +#### A masked pattern was here ####
    +
    +# Storage Information
    +SerDe Library: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
    +InputFormat: org.apache.hadoop.hive.ql.io.RCFileInputFormat
    +OutputFormat: org.apache.hadoop.hive.ql.io.RCFileOutputFormat
    +Compressed: No
    +Num Buckets: -1
    +Bucket Columns: []
    +Sort Columns: []
    +Stored As SubDirectories: Yes
    +Skewed Columns: [c1, c2]
    +Skewed Values: [[466, val_466], [287, val_287], [82, val_82]]
    +#### A masked pattern was here ####
    +Skewed Value to Truncated Path: {[82, val_82]=/stats_list_bucket/ds=2008-04-08/hr=11/c1=82/c2=val_82, [466, val_466]=/stats_list_bucket/ds=2008-04-08/hr=11/c1=466/c2=val_466, [287, val_287]=/stats_list_bucket/ds=2008-04-08/hr=11/c1=287/c2=val_287}
    +Storage Desc Params:
    + serialization.format 1
    +PREHOOK: query: -- Also try non-partitioned table with list bucketing.
    +-- Stats should show the same number of rows.
    +
    +create table stats_list_bucket_1 (
    + c1 string,
    + c2 string
    +)
    +skewed by (c1, c2) on (('466','val_466'),('287','val_287'),('82','val_82'))
    +stored as directories
    +stored as rcfile
    +PREHOOK: type: CREATETABLE
    +PREHOOK: Output: database:default
    +PREHOOK: Output: default@stats_list_bucket_1
    +POSTHOOK: query: -- Also try non-partitioned table with list bucketing.
    +-- Stats should show the same number of rows.
    +
    +create table stats_list_bucket_1 (
    + c1 string,
    + c2 string
    +)
    +skewed by (c1, c2) on (('466','val_466'),('287','val_287'),('82','val_82'))
    +stored as directories
    +stored as rcfile
    +POSTHOOK: type: CREATETABLE
    +POSTHOOK: Output: database:default
    +POSTHOOK: Output: default@stats_list_bucket_1
    +PREHOOK: query: insert overwrite table stats_list_bucket_1
    + select key, value from src
    +PREHOOK: type: QUERY
    +PREHOOK: Input: default@src
    +PREHOOK: Output: default@stats_list_bucket_1
    +POSTHOOK: query: insert overwrite table stats_list_bucket_1
    + select key, value from src
    +POSTHOOK: type: QUERY
    +POSTHOOK: Input: default@src
    +POSTHOOK: Output: default@stats_list_bucket_1
    +POSTHOOK: Lineage: stats_list_bucket_1.c1 SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ]
    +POSTHOOK: Lineage: stats_list_bucket_1.c2 SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
    +PREHOOK: query: desc formatted stats_list_bucket_1
    +PREHOOK: type: DESCTABLE
    +PREHOOK: Input: default@stats_list_bucket_1
    +POSTHOOK: query: desc formatted stats_list_bucket_1
    +POSTHOOK: type: DESCTABLE
    +POSTHOOK: Input: default@stats_list_bucket_1
    +# col_name data_type comment
    +
    +c1 string
    +c2 string
    +
    +# Detailed Table Information
    +Database: default
    +#### A masked pattern was here ####
    +Protect Mode: None
    +Retention: 0
    +#### A masked pattern was here ####
    +Table Type: MANAGED_TABLE
    +Table Parameters:
    + COLUMN_STATS_ACCURATE true
    + numFiles 4
    + numRows 500
    + rawDataSize 4812
    + totalSize 5522
    +#### A masked pattern was here ####
    +
    +# Storage Information
    +SerDe Library: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
    +InputFormat: org.apache.hadoop.hive.ql.io.RCFileInputFormat
    +OutputFormat: org.apache.hadoop.hive.ql.io.RCFileOutputFormat
    +Compressed: No
    +Num Buckets: -1
    +Bucket Columns: []
    +Sort Columns: []
    +Stored As SubDirectories: Yes
    +Skewed Columns: [c1, c2]
    +Skewed Values: [[466, val_466], [287, val_287], [82, val_82]]
    +#### A masked pattern was here ####
    +Skewed Value to Truncated Path: {[82, val_82]=/stats_list_bucket_1/c1=82/c2=val_82, [466, val_466]=/stats_list_bucket_1/c1=466/c2=val_466, [287, val_287]=/stats_list_bucket_1/c1=287/c2=val_287}
    +Storage Desc Params:
    + serialization.format 1
    +PREHOOK: query: drop table stats_list_bucket
    +PREHOOK: type: DROPTABLE
    +PREHOOK: Input: default@stats_list_bucket
    +PREHOOK: Output: default@stats_list_bucket
    +POSTHOOK: query: drop table stats_list_bucket
    +POSTHOOK: type: DROPTABLE
    +POSTHOOK: Input: default@stats_list_bucket
    +POSTHOOK: Output: default@stats_list_bucket
    +PREHOOK: query: drop table stats_list_bucket_1
    +PREHOOK: type: DROPTABLE
    +PREHOOK: Input: default@stats_list_bucket_1
    +PREHOOK: Output: default@stats_list_bucket_1
    +POSTHOOK: query: drop table stats_list_bucket_1
    +POSTHOOK: type: DROPTABLE
    +POSTHOOK: Input: default@stats_list_bucket_1
    +POSTHOOK: Output: default@stats_list_bucket_1

    Added: hive/trunk/ql/src/test/results/clientpositive/stats_list_bucket.q.java1.8.out
    URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/stats_list_bucket.q.java1.8.out?rev=1647912&view=auto
    ==============================================================================
    --- hive/trunk/ql/src/test/results/clientpositive/stats_list_bucket.q.java1.8.out (added)
    +++ hive/trunk/ql/src/test/results/clientpositive/stats_list_bucket.q.java1.8.out Thu Dec 25 18:06:30 2014
    @@ -0,0 +1,196 @@
    +PREHOOK: query: -- INCLUDE_HADOOP_MAJOR_VERSIONS(0.23)
    +-- JAVA_VERSION_SPECIFIC_OUTPUT
    +
    +drop table stats_list_bucket
    +PREHOOK: type: DROPTABLE
    +POSTHOOK: query: -- INCLUDE_HADOOP_MAJOR_VERSIONS(0.23)
    +-- JAVA_VERSION_SPECIFIC_OUTPUT
    +
    +drop table stats_list_bucket
    +POSTHOOK: type: DROPTABLE
    +PREHOOK: query: drop table stats_list_bucket_1
    +PREHOOK: type: DROPTABLE
    +POSTHOOK: query: drop table stats_list_bucket_1
    +POSTHOOK: type: DROPTABLE
    +PREHOOK: query: create table stats_list_bucket (
    + c1 string,
    + c2 string
    +) partitioned by (ds string, hr string)
    +skewed by (c1, c2) on (('466','val_466'),('287','val_287'),('82','val_82'))
    +stored as directories
    +stored as rcfile
    +PREHOOK: type: CREATETABLE
    +PREHOOK: Output: database:default
    +PREHOOK: Output: default@stats_list_bucket
    +POSTHOOK: query: create table stats_list_bucket (
    + c1 string,
    + c2 string
    +) partitioned by (ds string, hr string)
    +skewed by (c1, c2) on (('466','val_466'),('287','val_287'),('82','val_82'))
    +stored as directories
    +stored as rcfile
    +POSTHOOK: type: CREATETABLE
    +POSTHOOK: Output: database:default
    +POSTHOOK: Output: default@stats_list_bucket
    +PREHOOK: query: -- Make sure we use hashed IDs during stats publishing.
    +-- Try partitioned table with list bucketing.
    +-- The stats should show 500 rows loaded, as many rows as the src table has.
    +
    +insert overwrite table stats_list_bucket partition (ds = '2008-04-08', hr = '11')
    + select key, value from src
    +PREHOOK: type: QUERY
    +PREHOOK: Input: default@src
    +PREHOOK: Output: default@stats_list_bucket@ds=2008-04-08/hr=11
    +POSTHOOK: query: -- Make sure we use hashed IDs during stats publishing.
    +-- Try partitioned table with list bucketing.
    +-- The stats should show 500 rows loaded, as many rows as the src table has.
    +
    +insert overwrite table stats_list_bucket partition (ds = '2008-04-08', hr = '11')
    + select key, value from src
    +POSTHOOK: type: QUERY
    +POSTHOOK: Input: default@src
    +POSTHOOK: Output: default@stats_list_bucket@ds=2008-04-08/hr=11
    +POSTHOOK: Lineage: stats_list_bucket PARTITION(ds=2008-04-08,hr=11).c1 SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ]
    +POSTHOOK: Lineage: stats_list_bucket PARTITION(ds=2008-04-08,hr=11).c2 SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
    +PREHOOK: query: desc formatted stats_list_bucket partition (ds = '2008-04-08', hr = '11')
    +PREHOOK: type: DESCTABLE
    +PREHOOK: Input: default@stats_list_bucket
    +POSTHOOK: query: desc formatted stats_list_bucket partition (ds = '2008-04-08', hr = '11')
    +POSTHOOK: type: DESCTABLE
    +POSTHOOK: Input: default@stats_list_bucket
    +# col_name data_type comment
    +
    +c1 string
    +c2 string
    +
    +# Partition Information
    +# col_name data_type comment
    +
    +ds string
    +hr string
    +
    +# Detailed Partition Information
    +Partition Value: [2008-04-08, 11]
    +Database: default
    +Table: stats_list_bucket
    +#### A masked pattern was here ####
    +Protect Mode: None
    +#### A masked pattern was here ####
    +Partition Parameters:
    + COLUMN_STATS_ACCURATE true
    + numFiles 4
    + numRows 500
    + rawDataSize 4812
    + totalSize 5522
    +#### A masked pattern was here ####
    +
    +# Storage Information
    +SerDe Library: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
    +InputFormat: org.apache.hadoop.hive.ql.io.RCFileInputFormat
    +OutputFormat: org.apache.hadoop.hive.ql.io.RCFileOutputFormat
    +Compressed: No
    +Num Buckets: -1
    +Bucket Columns: []
    +Sort Columns: []
    +Stored As SubDirectories: Yes
    +Skewed Columns: [c1, c2]
    +Skewed Values: [[466, val_466], [287, val_287], [82, val_82]]
    +#### A masked pattern was here ####
    +Skewed Value to Truncated Path: {[466, val_466]=/stats_list_bucket/ds=2008-04-08/hr=11/c1=466/c2=val_466, [287, val_287]=/stats_list_bucket/ds=2008-04-08/hr=11/c1=287/c2=val_287, [82, val_82]=/stats_list_bucket/ds=2008-04-08/hr=11/c1=82/c2=val_82}
    +Storage Desc Params:
    + serialization.format 1
    +PREHOOK: query: -- Also try non-partitioned table with list bucketing.
    +-- Stats should show the same number of rows.
    +
    +create table stats_list_bucket_1 (
    + c1 string,
    + c2 string
    +)
    +skewed by (c1, c2) on (('466','val_466'),('287','val_287'),('82','val_82'))
    +stored as directories
    +stored as rcfile
    +PREHOOK: type: CREATETABLE
    +PREHOOK: Output: database:default
    +PREHOOK: Output: default@stats_list_bucket_1
    +POSTHOOK: query: -- Also try non-partitioned table with list bucketing.
    +-- Stats should show the same number of rows.
    +
    +create table stats_list_bucket_1 (
    + c1 string,
    + c2 string
    +)
    +skewed by (c1, c2) on (('466','val_466'),('287','val_287'),('82','val_82'))
    +stored as directories
    +stored as rcfile
    +POSTHOOK: type: CREATETABLE
    +POSTHOOK: Output: database:default
    +POSTHOOK: Output: default@stats_list_bucket_1
    +PREHOOK: query: insert overwrite table stats_list_bucket_1
    + select key, value from src
    +PREHOOK: type: QUERY
    +PREHOOK: Input: default@src
    +PREHOOK: Output: default@stats_list_bucket_1
    +POSTHOOK: query: insert overwrite table stats_list_bucket_1
    + select key, value from src
    +POSTHOOK: type: QUERY
    +POSTHOOK: Input: default@src
    +POSTHOOK: Output: default@stats_list_bucket_1
    +POSTHOOK: Lineage: stats_list_bucket_1.c1 SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ]
    +POSTHOOK: Lineage: stats_list_bucket_1.c2 SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
    +PREHOOK: query: desc formatted stats_list_bucket_1
    +PREHOOK: type: DESCTABLE
    +PREHOOK: Input: default@stats_list_bucket_1
    +POSTHOOK: query: desc formatted stats_list_bucket_1
    +POSTHOOK: type: DESCTABLE
    +POSTHOOK: Input: default@stats_list_bucket_1
    +# col_name data_type comment
    +
    +c1 string
    +c2 string
    +
    +# Detailed Table Information
    +Database: default
    +#### A masked pattern was here ####
    +Protect Mode: None
    +Retention: 0
    +#### A masked pattern was here ####
    +Table Type: MANAGED_TABLE
    +Table Parameters:
    + COLUMN_STATS_ACCURATE true
    + numFiles 4
    + numRows 500
    + rawDataSize 4812
    + totalSize 5522
    +#### A masked pattern was here ####
    +
    +# Storage Information
    +SerDe Library: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
    +InputFormat: org.apache.hadoop.hive.ql.io.RCFileInputFormat
    +OutputFormat: org.apache.hadoop.hive.ql.io.RCFileOutputFormat
    +Compressed: No
    +Num Buckets: -1
    +Bucket Columns: []
    +Sort Columns: []
    +Stored As SubDirectories: Yes
    +Skewed Columns: [c1, c2]
    +Skewed Values: [[466, val_466], [287, val_287], [82, val_82]]
    +#### A masked pattern was here ####
    +Skewed Value to Truncated Path: {[466, val_466]=/stats_list_bucket_1/c1=466/c2=val_466, [287, val_287]=/stats_list_bucket_1/c1=287/c2=val_287, [82, val_82]=/stats_list_bucket_1/c1=82/c2=val_82}
    +Storage Desc Params:
    + serialization.format 1
    +PREHOOK: query: drop table stats_list_bucket
    +PREHOOK: type: DROPTABLE
    +PREHOOK: Input: default@stats_list_bucket
    +PREHOOK: Output: default@stats_list_bucket
    +POSTHOOK: query: drop table stats_list_bucket
    +POSTHOOK: type: DROPTABLE
    +POSTHOOK: Input: default@stats_list_bucket
    +POSTHOOK: Output: default@stats_list_bucket
    +PREHOOK: query: drop table stats_list_bucket_1
    +PREHOOK: type: DROPTABLE
    +PREHOOK: Input: default@stats_list_bucket_1
    +PREHOOK: Output: default@stats_list_bucket_1
    +POSTHOOK: query: drop table stats_list_bucket_1
    +POSTHOOK: type: DROPTABLE
    +POSTHOOK: Input: default@stats_list_bucket_1
    +POSTHOOK: Output: default@stats_list_bucket_1
  • Brock at Dec 25, 2014 at 6:06 pm
    Added: hive/trunk/ql/src/test/results/clientpositive/list_bucket_dml_8.q.java1.8.out
    URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/list_bucket_dml_8.q.java1.8.out?rev=1647912&view=auto
    ==============================================================================
    --- hive/trunk/ql/src/test/results/clientpositive/list_bucket_dml_8.q.java1.8.out (added)
    +++ hive/trunk/ql/src/test/results/clientpositive/list_bucket_dml_8.q.java1.8.out Thu Dec 25 18:06:30 2014
    @@ -0,0 +1,718 @@
    +PREHOOK: query: -- list bucketing alter table ... concatenate:
    +-- Use list bucketing DML to generate mutilple files in partitions by turning off merge
    +-- dynamic partition. multiple skewed columns. merge.
    +-- The following explains merge example used in this test case
    +-- DML will generated 2 partitions
    +-- ds=2008-04-08/hr=a1
    +-- ds=2008-04-08/hr=b1
    +-- without merge, each partition has more files
    +-- ds=2008-04-08/hr=a1 has 2 files
    +-- ds=2008-04-08/hr=b1 has 6 files
    +-- with merge each partition has more files
    +-- ds=2008-04-08/hr=a1 has 1 files
    +-- ds=2008-04-08/hr=b1 has 4 files
    +-- The following shows file size and name in each directory
    +-- hr=a1/HIVE_DEFAULT_LIST_BUCKETING_DIR_NAME/HIVE_DEFAULT_LIST_BUCKETING_DIR_NAME:
    +-- without merge
    +-- 155 000000_0
    +-- 155 000001_0
    +-- with merge
    +-- 254 000000_0
    +-- hr=b1/key=103/value=val_103:
    +-- without merge
    +-- 99 000000_0
    +-- 99 000001_0
    +-- with merge
    +-- 142 000001_0
    +-- hr=b1/HIVE_DEFAULT_LIST_BUCKETING_DIR_NAME/HIVE_DEFAULT_LIST_BUCKETING_DIR_NAME:
    +-- without merge
    +-- 5181 000000_0
    +-- 5181 000001_0
    +-- with merge
    +-- 5181 000000_0
    +-- 5181 000001_0
    +-- hr=b1/key=484/value=val_484
    +-- without merge
    +-- 87 000000_0
    +-- 87 000001_0
    +-- with merge
    +-- 118 000002_0
    +
    +-- INCLUDE_HADOOP_MAJOR_VERSIONS(0.23)
    +-- JAVA_VERSION_SPECIFIC_OUTPUT
    +
    +-- create a skewed table
    +create table list_bucketing_dynamic_part (key String, value String)
    + partitioned by (ds String, hr String)
    + skewed by (key, value) on (('484','val_484'),('51','val_14'),('103','val_103'))
    + stored as DIRECTORIES
    + STORED AS RCFILE
    +PREHOOK: type: CREATETABLE
    +PREHOOK: Output: database:default
    +PREHOOK: Output: default@list_bucketing_dynamic_part
    +POSTHOOK: query: -- list bucketing alter table ... concatenate:
    +-- Use list bucketing DML to generate mutilple files in partitions by turning off merge
    +-- dynamic partition. multiple skewed columns. merge.
    +-- The following explains merge example used in this test case
    +-- DML will generated 2 partitions
    +-- ds=2008-04-08/hr=a1
    +-- ds=2008-04-08/hr=b1
    +-- without merge, each partition has more files
    +-- ds=2008-04-08/hr=a1 has 2 files
    +-- ds=2008-04-08/hr=b1 has 6 files
    +-- with merge each partition has more files
    +-- ds=2008-04-08/hr=a1 has 1 files
    +-- ds=2008-04-08/hr=b1 has 4 files
    +-- The following shows file size and name in each directory
    +-- hr=a1/HIVE_DEFAULT_LIST_BUCKETING_DIR_NAME/HIVE_DEFAULT_LIST_BUCKETING_DIR_NAME:
    +-- without merge
    +-- 155 000000_0
    +-- 155 000001_0
    +-- with merge
    +-- 254 000000_0
    +-- hr=b1/key=103/value=val_103:
    +-- without merge
    +-- 99 000000_0
    +-- 99 000001_0
    +-- with merge
    +-- 142 000001_0
    +-- hr=b1/HIVE_DEFAULT_LIST_BUCKETING_DIR_NAME/HIVE_DEFAULT_LIST_BUCKETING_DIR_NAME:
    +-- without merge
    +-- 5181 000000_0
    +-- 5181 000001_0
    +-- with merge
    +-- 5181 000000_0
    +-- 5181 000001_0
    +-- hr=b1/key=484/value=val_484
    +-- without merge
    +-- 87 000000_0
    +-- 87 000001_0
    +-- with merge
    +-- 118 000002_0
    +
    +-- INCLUDE_HADOOP_MAJOR_VERSIONS(0.23)
    +-- JAVA_VERSION_SPECIFIC_OUTPUT
    +
    +-- create a skewed table
    +create table list_bucketing_dynamic_part (key String, value String)
    + partitioned by (ds String, hr String)
    + skewed by (key, value) on (('484','val_484'),('51','val_14'),('103','val_103'))
    + stored as DIRECTORIES
    + STORED AS RCFILE
    +POSTHOOK: type: CREATETABLE
    +POSTHOOK: Output: database:default
    +POSTHOOK: Output: default@list_bucketing_dynamic_part
    +PREHOOK: query: -- list bucketing DML without merge. use bucketize to generate a few small files.
    +explain extended
    +insert overwrite table list_bucketing_dynamic_part partition (ds = '2008-04-08', hr)
    +select key, value, if(key % 100 == 0, 'a1', 'b1') from srcpart where ds = '2008-04-08'
    +PREHOOK: type: QUERY
    +POSTHOOK: query: -- list bucketing DML without merge. use bucketize to generate a few small files.
    +explain extended
    +insert overwrite table list_bucketing_dynamic_part partition (ds = '2008-04-08', hr)
    +select key, value, if(key % 100 == 0, 'a1', 'b1') from srcpart where ds = '2008-04-08'
    +POSTHOOK: type: QUERY
    +ABSTRACT SYNTAX TREE:
    +
    +TOK_QUERY
    + TOK_FROM
    + TOK_TABREF
    + TOK_TABNAME
    + srcpart
    + TOK_INSERT
    + TOK_DESTINATION
    + TOK_TAB
    + TOK_TABNAME
    + list_bucketing_dynamic_part
    + TOK_PARTSPEC
    + TOK_PARTVAL
    + ds
    + '2008-04-08'
    + TOK_PARTVAL
    + hr
    + TOK_SELECT
    + TOK_SELEXPR
    + TOK_TABLE_OR_COL
    + key
    + TOK_SELEXPR
    + TOK_TABLE_OR_COL
    + value
    + TOK_SELEXPR
    + TOK_FUNCTION
    + if
    + ==
    + %
    + TOK_TABLE_OR_COL
    + key
    + 100
    + 0
    + 'a1'
    + 'b1'
    + TOK_WHERE
    + =
    + TOK_TABLE_OR_COL
    + ds
    + '2008-04-08'
    +
    +
    +STAGE DEPENDENCIES:
    + Stage-1 is a root stage
    + Stage-0 depends on stages: Stage-1
    + Stage-2 depends on stages: Stage-0
    +
    +STAGE PLANS:
    + Stage: Stage-1
    + Map Reduce
    + Map Operator Tree:
    + TableScan
    + alias: srcpart
    + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE
    + GatherStats: false
    + Select Operator
    + expressions: key (type: string), value (type: string), if(((UDFToDouble(key) % 100.0) = 0.0), 'a1', 'b1') (type: string)
    + outputColumnNames: _col0, _col1, _col2
    + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE
    + File Output Operator
    + compressed: false
    + GlobalTableId: 1
    +#### A masked pattern was here ####
    + NumFilesPerFileSink: 1
    + Static Partition Specification: ds=2008-04-08/
    + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE
    +#### A masked pattern was here ####
    + table:
    + input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat
    + output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat
    + properties:
    + bucket_count -1
    + columns key,value
    + columns.comments
    + columns.types string:string
    +#### A masked pattern was here ####
    + name default.list_bucketing_dynamic_part
    + partition_columns ds/hr
    + partition_columns.types string:string
    + serialization.ddl struct list_bucketing_dynamic_part { string key, string value}
    + serialization.format 1
    + serialization.lib org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
    +#### A masked pattern was here ####
    + serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
    + name: default.list_bucketing_dynamic_part
    + TotalFiles: 1
    + GatherStats: true
    + MultiFileSpray: false
    + Path -> Alias:
    +#### A masked pattern was here ####
    + Path -> Partition:
    +#### A masked pattern was here ####
    + Partition
    + base file name: hr=11
    + input format: org.apache.hadoop.mapred.TextInputFormat
    + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
    + partition values:
    + ds 2008-04-08
    + hr 11
    + properties:
    + COLUMN_STATS_ACCURATE true
    + bucket_count -1
    + columns key,value
    + columns.comments 'default','default'
    + columns.types string:string
    +#### A masked pattern was here ####
    + name default.srcpart
    + numFiles 1
    + numRows 500
    + partition_columns ds/hr
    + partition_columns.types string:string
    + rawDataSize 5312
    + serialization.ddl struct srcpart { string key, string value}
    + serialization.format 1
    + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
    + totalSize 5812
    +#### A masked pattern was here ####
    + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
    +
    + input format: org.apache.hadoop.mapred.TextInputFormat
    + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
    + properties:
    + bucket_count -1
    + columns key,value
    + columns.comments 'default','default'
    + columns.types string:string
    +#### A masked pattern was here ####
    + name default.srcpart
    + partition_columns ds/hr
    + partition_columns.types string:string
    + serialization.ddl struct srcpart { string key, string value}
    + serialization.format 1
    + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
    +#### A masked pattern was here ####
    + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
    + name: default.srcpart
    + name: default.srcpart
    +#### A masked pattern was here ####
    + Partition
    + base file name: hr=12
    + input format: org.apache.hadoop.mapred.TextInputFormat
    + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
    + partition values:
    + ds 2008-04-08
    + hr 12
    + properties:
    + COLUMN_STATS_ACCURATE true
    + bucket_count -1
    + columns key,value
    + columns.comments 'default','default'
    + columns.types string:string
    +#### A masked pattern was here ####
    + name default.srcpart
    + numFiles 1
    + numRows 500
    + partition_columns ds/hr
    + partition_columns.types string:string
    + rawDataSize 5312
    + serialization.ddl struct srcpart { string key, string value}
    + serialization.format 1
    + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
    + totalSize 5812
    +#### A masked pattern was here ####
    + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
    +
    + input format: org.apache.hadoop.mapred.TextInputFormat
    + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
    + properties:
    + bucket_count -1
    + columns key,value
    + columns.comments 'default','default'
    + columns.types string:string
    +#### A masked pattern was here ####
    + name default.srcpart
    + partition_columns ds/hr
    + partition_columns.types string:string
    + serialization.ddl struct srcpart { string key, string value}
    + serialization.format 1
    + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
    +#### A masked pattern was here ####
    + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
    + name: default.srcpart
    + name: default.srcpart
    + Truncated Path -> Alias:
    + /srcpart/ds=2008-04-08/hr=11 [$hdt$_0:srcpart]
    + /srcpart/ds=2008-04-08/hr=12 [$hdt$_0:srcpart]
    +
    + Stage: Stage-0
    + Move Operator
    + tables:
    + partition:
    + ds 2008-04-08
    + hr
    + replace: true
    +#### A masked pattern was here ####
    + table:
    + input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat
    + output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat
    + properties:
    + bucket_count -1
    + columns key,value
    + columns.comments
    + columns.types string:string
    +#### A masked pattern was here ####
    + name default.list_bucketing_dynamic_part
    + partition_columns ds/hr
    + partition_columns.types string:string
    + serialization.ddl struct list_bucketing_dynamic_part { string key, string value}
    + serialization.format 1
    + serialization.lib org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
    +#### A masked pattern was here ####
    + serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
    + name: default.list_bucketing_dynamic_part
    +
    + Stage: Stage-2
    + Stats-Aggr Operator
    +#### A masked pattern was here ####
    +
    +PREHOOK: query: insert overwrite table list_bucketing_dynamic_part partition (ds = '2008-04-08', hr)
    +select key, value, if(key % 100 == 0, 'a1', 'b1') from srcpart where ds = '2008-04-08'
    +PREHOOK: type: QUERY
    +PREHOOK: Input: default@srcpart
    +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11
    +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12
    +PREHOOK: Output: default@list_bucketing_dynamic_part@ds=2008-04-08
    +POSTHOOK: query: insert overwrite table list_bucketing_dynamic_part partition (ds = '2008-04-08', hr)
    +select key, value, if(key % 100 == 0, 'a1', 'b1') from srcpart where ds = '2008-04-08'
    +POSTHOOK: type: QUERY
    +POSTHOOK: Input: default@srcpart
    +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11
    +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12
    +POSTHOOK: Output: default@list_bucketing_dynamic_part@ds=2008-04-08/hr=a1
    +POSTHOOK: Output: default@list_bucketing_dynamic_part@ds=2008-04-08/hr=b1
    +POSTHOOK: Lineage: list_bucketing_dynamic_part PARTITION(ds=2008-04-08,hr=a1).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ]
    +POSTHOOK: Lineage: list_bucketing_dynamic_part PARTITION(ds=2008-04-08,hr=a1).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ]
    +POSTHOOK: Lineage: list_bucketing_dynamic_part PARTITION(ds=2008-04-08,hr=b1).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ]
    +POSTHOOK: Lineage: list_bucketing_dynamic_part PARTITION(ds=2008-04-08,hr=b1).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ]
    +PREHOOK: query: -- check DML result
    +show partitions list_bucketing_dynamic_part
    +PREHOOK: type: SHOWPARTITIONS
    +PREHOOK: Input: default@list_bucketing_dynamic_part
    +POSTHOOK: query: -- check DML result
    +show partitions list_bucketing_dynamic_part
    +POSTHOOK: type: SHOWPARTITIONS
    +POSTHOOK: Input: default@list_bucketing_dynamic_part
    +ds=2008-04-08/hr=a1
    +ds=2008-04-08/hr=b1
    +PREHOOK: query: desc formatted list_bucketing_dynamic_part partition (ds='2008-04-08', hr='a1')
    +PREHOOK: type: DESCTABLE
    +PREHOOK: Input: default@list_bucketing_dynamic_part
    +POSTHOOK: query: desc formatted list_bucketing_dynamic_part partition (ds='2008-04-08', hr='a1')
    +POSTHOOK: type: DESCTABLE
    +POSTHOOK: Input: default@list_bucketing_dynamic_part
    +# col_name data_type comment
    +
    +key string
    +value string
    +
    +# Partition Information
    +# col_name data_type comment
    +
    +ds string
    +hr string
    +
    +# Detailed Partition Information
    +Partition Value: [2008-04-08, a1]
    +Database: default
    +Table: list_bucketing_dynamic_part
    +#### A masked pattern was here ####
    +Protect Mode: None
    +#### A masked pattern was here ####
    +Partition Parameters:
    + COLUMN_STATS_ACCURATE true
    + numFiles 2
    + numRows 16
    + rawDataSize 136
    + totalSize 310
    +#### A masked pattern was here ####
    +
    +# Storage Information
    +SerDe Library: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
    +InputFormat: org.apache.hadoop.hive.ql.io.RCFileInputFormat
    +OutputFormat: org.apache.hadoop.hive.ql.io.RCFileOutputFormat
    +Compressed: No
    +Num Buckets: -1
    +Bucket Columns: []
    +Sort Columns: []
    +Stored As SubDirectories: Yes
    +Skewed Columns: [key, value]
    +Skewed Values: [[484, val_484], [51, val_14], [103, val_103]]
    +Storage Desc Params:
    + serialization.format 1
    +PREHOOK: query: desc formatted list_bucketing_dynamic_part partition (ds='2008-04-08', hr='b1')
    +PREHOOK: type: DESCTABLE
    +PREHOOK: Input: default@list_bucketing_dynamic_part
    +POSTHOOK: query: desc formatted list_bucketing_dynamic_part partition (ds='2008-04-08', hr='b1')
    +POSTHOOK: type: DESCTABLE
    +POSTHOOK: Input: default@list_bucketing_dynamic_part
    +# col_name data_type comment
    +
    +key string
    +value string
    +
    +# Partition Information
    +# col_name data_type comment
    +
    +ds string
    +hr string
    +
    +# Detailed Partition Information
    +Partition Value: [2008-04-08, b1]
    +Database: default
    +Table: list_bucketing_dynamic_part
    +#### A masked pattern was here ####
    +Protect Mode: None
    +#### A masked pattern was here ####
    +Partition Parameters:
    + COLUMN_STATS_ACCURATE true
    + numFiles 6
    + numRows 984
    + rawDataSize 9488
    + totalSize 10734
    +#### A masked pattern was here ####
    +
    +# Storage Information
    +SerDe Library: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
    +InputFormat: org.apache.hadoop.hive.ql.io.RCFileInputFormat
    +OutputFormat: org.apache.hadoop.hive.ql.io.RCFileOutputFormat
    +Compressed: No
    +Num Buckets: -1
    +Bucket Columns: []
    +Sort Columns: []
    +Stored As SubDirectories: Yes
    +Skewed Columns: [key, value]
    +Skewed Values: [[484, val_484], [51, val_14], [103, val_103]]
    +#### A masked pattern was here ####
    +Skewed Value to Truncated Path: {[484, val_484]=/list_bucketing_dynamic_part/ds=2008-04-08/hr=b1/key=484/value=val_484, [103, val_103]=/list_bucketing_dynamic_part/ds=2008-04-08/hr=b1/key=103/value=val_103}
    +Storage Desc Params:
    + serialization.format 1
    +PREHOOK: query: -- concatenate the partition and it will merge files
    +alter table list_bucketing_dynamic_part partition (ds='2008-04-08', hr='b1') concatenate
    +PREHOOK: type: ALTER_PARTITION_MERGE
    +PREHOOK: Input: default@list_bucketing_dynamic_part
    +PREHOOK: Output: default@list_bucketing_dynamic_part@ds=2008-04-08/hr=b1
    +POSTHOOK: query: -- concatenate the partition and it will merge files
    +alter table list_bucketing_dynamic_part partition (ds='2008-04-08', hr='b1') concatenate
    +POSTHOOK: type: ALTER_PARTITION_MERGE
    +POSTHOOK: Input: default@list_bucketing_dynamic_part
    +POSTHOOK: Output: default@list_bucketing_dynamic_part@ds=2008-04-08/hr=b1
    +PREHOOK: query: desc formatted list_bucketing_dynamic_part partition (ds='2008-04-08', hr='b1')
    +PREHOOK: type: DESCTABLE
    +PREHOOK: Input: default@list_bucketing_dynamic_part
    +POSTHOOK: query: desc formatted list_bucketing_dynamic_part partition (ds='2008-04-08', hr='b1')
    +POSTHOOK: type: DESCTABLE
    +POSTHOOK: Input: default@list_bucketing_dynamic_part
    +# col_name data_type comment
    +
    +key string
    +value string
    +
    +# Partition Information
    +# col_name data_type comment
    +
    +ds string
    +hr string
    +
    +# Detailed Partition Information
    +Partition Value: [2008-04-08, b1]
    +Database: default
    +Table: list_bucketing_dynamic_part
    +#### A masked pattern was here ####
    +Protect Mode: None
    +#### A masked pattern was here ####
    +Partition Parameters:
    + COLUMN_STATS_ACCURATE true
    + numFiles 3
    + numRows 0
    + rawDataSize 0
    + totalSize 10586
    +#### A masked pattern was here ####
    +
    +# Storage Information
    +SerDe Library: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
    +InputFormat: org.apache.hadoop.hive.ql.io.RCFileInputFormat
    +OutputFormat: org.apache.hadoop.hive.ql.io.RCFileOutputFormat
    +Compressed: No
    +Num Buckets: -1
    +Bucket Columns: []
    +Sort Columns: []
    +Stored As SubDirectories: Yes
    +Skewed Columns: [key, value]
    +Skewed Values: [[484, val_484], [51, val_14], [103, val_103]]
    +#### A masked pattern was here ####
    +Skewed Value to Truncated Path: {[484, val_484]=/list_bucketing_dynamic_part/ds=2008-04-08/hr=b1/key=484/value=val_484, [103, val_103]=/list_bucketing_dynamic_part/ds=2008-04-08/hr=b1/key=103/value=val_103}
    +Storage Desc Params:
    + serialization.format 1
    +PREHOOK: query: select count(1) from srcpart where ds = '2008-04-08'
    +PREHOOK: type: QUERY
    +PREHOOK: Input: default@srcpart
    +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11
    +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12
    +#### A masked pattern was here ####
    +POSTHOOK: query: select count(1) from srcpart where ds = '2008-04-08'
    +POSTHOOK: type: QUERY
    +POSTHOOK: Input: default@srcpart
    +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11
    +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12
    +#### A masked pattern was here ####
    +1000
    +PREHOOK: query: select count(*) from list_bucketing_dynamic_part
    +PREHOOK: type: QUERY
    +PREHOOK: Input: default@list_bucketing_dynamic_part
    +PREHOOK: Input: default@list_bucketing_dynamic_part@ds=2008-04-08/hr=a1
    +PREHOOK: Input: default@list_bucketing_dynamic_part@ds=2008-04-08/hr=b1
    +#### A masked pattern was here ####
    +POSTHOOK: query: select count(*) from list_bucketing_dynamic_part
    +POSTHOOK: type: QUERY
    +POSTHOOK: Input: default@list_bucketing_dynamic_part
    +POSTHOOK: Input: default@list_bucketing_dynamic_part@ds=2008-04-08/hr=a1
    +POSTHOOK: Input: default@list_bucketing_dynamic_part@ds=2008-04-08/hr=b1
    +#### A masked pattern was here ####
    +1000
    +PREHOOK: query: explain extended
    +select * from list_bucketing_dynamic_part where key = '484' and value = 'val_484'
    +PREHOOK: type: QUERY
    +POSTHOOK: query: explain extended
    +select * from list_bucketing_dynamic_part where key = '484' and value = 'val_484'
    +POSTHOOK: type: QUERY
    +ABSTRACT SYNTAX TREE:
    +
    +TOK_QUERY
    + TOK_FROM
    + TOK_TABREF
    + TOK_TABNAME
    + list_bucketing_dynamic_part
    + TOK_INSERT
    + TOK_DESTINATION
    + TOK_DIR
    + TOK_TMP_FILE
    + TOK_SELECT
    + TOK_SELEXPR
    + TOK_ALLCOLREF
    + TOK_WHERE
    + and
    + =
    + TOK_TABLE_OR_COL
    + key
    + '484'
    + =
    + TOK_TABLE_OR_COL
    + value
    + 'val_484'
    +
    +
    +STAGE DEPENDENCIES:
    + Stage-0 is a root stage
    +
    +STAGE PLANS:
    + Stage: Stage-0
    + Fetch Operator
    + limit: -1
    + Partition Description:
    + Partition
    + input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat
    + output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat
    + partition values:
    + ds 2008-04-08
    + hr a1
    + properties:
    + COLUMN_STATS_ACCURATE true
    + bucket_count -1
    + columns key,value
    + columns.comments
    + columns.types string:string
    +#### A masked pattern was here ####
    + name default.list_bucketing_dynamic_part
    + numFiles 2
    + numRows 16
    + partition_columns ds/hr
    + partition_columns.types string:string
    + rawDataSize 136
    + serialization.ddl struct list_bucketing_dynamic_part { string key, string value}
    + serialization.format 1
    + serialization.lib org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
    + totalSize 310
    +#### A masked pattern was here ####
    + serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
    +
    + input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat
    + output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat
    + properties:
    + bucket_count -1
    + columns key,value
    + columns.comments
    + columns.types string:string
    +#### A masked pattern was here ####
    + name default.list_bucketing_dynamic_part
    + partition_columns ds/hr
    + partition_columns.types string:string
    + serialization.ddl struct list_bucketing_dynamic_part { string key, string value}
    + serialization.format 1
    + serialization.lib org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
    +#### A masked pattern was here ####
    + serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
    + name: default.list_bucketing_dynamic_part
    + name: default.list_bucketing_dynamic_part
    + Partition
    + input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat
    + output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat
    + partition values:
    + ds 2008-04-08
    + hr b1
    + properties:
    + COLUMN_STATS_ACCURATE true
    + bucket_count -1
    + columns key,value
    + columns.comments
    + columns.types string:string
    +#### A masked pattern was here ####
    + name default.list_bucketing_dynamic_part
    + numFiles 3
    + numRows 0
    + partition_columns ds/hr
    + partition_columns.types string:string
    + rawDataSize 0
    + serialization.ddl struct list_bucketing_dynamic_part { string key, string value}
    + serialization.format 1
    + serialization.lib org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
    + totalSize 10586
    +#### A masked pattern was here ####
    + serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
    +
    + input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat
    + output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat
    + properties:
    + bucket_count -1
    + columns key,value
    + columns.comments
    + columns.types string:string
    +#### A masked pattern was here ####
    + name default.list_bucketing_dynamic_part
    + partition_columns ds/hr
    + partition_columns.types string:string
    + serialization.ddl struct list_bucketing_dynamic_part { string key, string value}
    + serialization.format 1
    + serialization.lib org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
    +#### A masked pattern was here ####
    + serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
    + name: default.list_bucketing_dynamic_part
    + name: default.list_bucketing_dynamic_part
    + Processor Tree:
    + TableScan
    + alias: list_bucketing_dynamic_part
    + Statistics: Num rows: 16 Data size: 136 Basic stats: PARTIAL Column stats: NONE
    + GatherStats: false
    + Filter Operator
    + isSamplingPred: false
    + predicate: ((key = '484') and (value = 'val_484')) (type: boolean)
    + Statistics: Num rows: 4 Data size: 34 Basic stats: COMPLETE Column stats: NONE
    + Select Operator
    + expressions: '484' (type: string), 'val_484' (type: string), ds (type: string), hr (type: string)
    + outputColumnNames: _col0, _col1, _col2, _col3
    + Statistics: Num rows: 4 Data size: 34 Basic stats: COMPLETE Column stats: NONE
    + ListSink
    +
    +PREHOOK: query: select * from list_bucketing_dynamic_part where key = '484' and value = 'val_484'
    +PREHOOK: type: QUERY
    +PREHOOK: Input: default@list_bucketing_dynamic_part
    +PREHOOK: Input: default@list_bucketing_dynamic_part@ds=2008-04-08/hr=a1
    +PREHOOK: Input: default@list_bucketing_dynamic_part@ds=2008-04-08/hr=b1
    +#### A masked pattern was here ####
    +POSTHOOK: query: select * from list_bucketing_dynamic_part where key = '484' and value = 'val_484'
    +POSTHOOK: type: QUERY
    +POSTHOOK: Input: default@list_bucketing_dynamic_part
    +POSTHOOK: Input: default@list_bucketing_dynamic_part@ds=2008-04-08/hr=a1
    +POSTHOOK: Input: default@list_bucketing_dynamic_part@ds=2008-04-08/hr=b1
    +#### A masked pattern was here ####
    +484 val_484 2008-04-08 b1
    +484 val_484 2008-04-08 b1
    +PREHOOK: query: select * from srcpart where ds = '2008-04-08' and key = '484' and value = 'val_484' order by hr
    +PREHOOK: type: QUERY
    +PREHOOK: Input: default@srcpart
    +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11
    +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12
    +#### A masked pattern was here ####
    +POSTHOOK: query: select * from srcpart where ds = '2008-04-08' and key = '484' and value = 'val_484' order by hr
    +POSTHOOK: type: QUERY
    +POSTHOOK: Input: default@srcpart
    +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11
    +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12
    +#### A masked pattern was here ####
    +484 val_484 2008-04-08 11
    +484 val_484 2008-04-08 12
    +PREHOOK: query: -- clean up
    +drop table list_bucketing_dynamic_part
    +PREHOOK: type: DROPTABLE
    +PREHOOK: Input: default@list_bucketing_dynamic_part
    +PREHOOK: Output: default@list_bucketing_dynamic_part
    +POSTHOOK: query: -- clean up
    +drop table list_bucketing_dynamic_part
    +POSTHOOK: type: DROPTABLE
    +POSTHOOK: Input: default@list_bucketing_dynamic_part
    +POSTHOOK: Output: default@list_bucketing_dynamic_part

    Added: hive/trunk/ql/src/test/results/clientpositive/list_bucket_dml_9.q.java1.7.out
    URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/list_bucket_dml_9.q.java1.7.out?rev=1647912&view=auto
    ==============================================================================
    --- hive/trunk/ql/src/test/results/clientpositive/list_bucket_dml_9.q.java1.7.out (added)
    +++ hive/trunk/ql/src/test/results/clientpositive/list_bucket_dml_9.q.java1.7.out Thu Dec 25 18:06:30 2014
    @@ -0,0 +1,951 @@
    +PREHOOK: query: -- INCLUDE_HADOOP_MAJOR_VERSIONS(0.23)
    +-- SORT_QUERY_RESULTS
    +-- JAVA_VERSION_SPECIFIC_OUTPUT
    +
    +-- list bucketing DML: static partition. multiple skewed columns. merge.
    +-- ds=2008-04-08/hr=11/HIVE_DEFAULT_LIST_BUCKETING_DIR_NAME:
    +-- 5263 000000_0
    +-- 5263 000001_0
    +-- ds=2008-04-08/hr=11/key=103:
    +-- 99 000000_0
    +-- 99 000001_0
    +-- after merge
    +-- 142 000000_0
    +-- ds=2008-04-08/hr=11/key=484:
    +-- 87 000000_0
    +-- 87 000001_0
    +-- after merge
    +-- 118 000001_0
    +
    +-- create a skewed table
    +create table list_bucketing_static_part (key String, value String)
    + partitioned by (ds String, hr String)
    + skewed by (key) on ('484','103')
    + stored as DIRECTORIES
    + STORED AS RCFILE
    +PREHOOK: type: CREATETABLE
    +PREHOOK: Output: database:default
    +PREHOOK: Output: default@list_bucketing_static_part
    +POSTHOOK: query: -- INCLUDE_HADOOP_MAJOR_VERSIONS(0.23)
    +-- SORT_QUERY_RESULTS
    +-- JAVA_VERSION_SPECIFIC_OUTPUT
    +
    +-- list bucketing DML: static partition. multiple skewed columns. merge.
    +-- ds=2008-04-08/hr=11/HIVE_DEFAULT_LIST_BUCKETING_DIR_NAME:
    +-- 5263 000000_0
    +-- 5263 000001_0
    +-- ds=2008-04-08/hr=11/key=103:
    +-- 99 000000_0
    +-- 99 000001_0
    +-- after merge
    +-- 142 000000_0
    +-- ds=2008-04-08/hr=11/key=484:
    +-- 87 000000_0
    +-- 87 000001_0
    +-- after merge
    +-- 118 000001_0
    +
    +-- create a skewed table
    +create table list_bucketing_static_part (key String, value String)
    + partitioned by (ds String, hr String)
    + skewed by (key) on ('484','103')
    + stored as DIRECTORIES
    + STORED AS RCFILE
    +POSTHOOK: type: CREATETABLE
    +POSTHOOK: Output: database:default
    +POSTHOOK: Output: default@list_bucketing_static_part
    +PREHOOK: query: -- list bucketing DML without merge. use bucketize to generate a few small files.
    +explain extended
    +insert overwrite table list_bucketing_static_part partition (ds = '2008-04-08', hr = '11')
    +select key, value from srcpart where ds = '2008-04-08'
    +PREHOOK: type: QUERY
    +POSTHOOK: query: -- list bucketing DML without merge. use bucketize to generate a few small files.
    +explain extended
    +insert overwrite table list_bucketing_static_part partition (ds = '2008-04-08', hr = '11')
    +select key, value from srcpart where ds = '2008-04-08'
    +POSTHOOK: type: QUERY
    +ABSTRACT SYNTAX TREE:
    +
    +TOK_QUERY
    + TOK_FROM
    + TOK_TABREF
    + TOK_TABNAME
    + srcpart
    + TOK_INSERT
    + TOK_DESTINATION
    + TOK_TAB
    + TOK_TABNAME
    + list_bucketing_static_part
    + TOK_PARTSPEC
    + TOK_PARTVAL
    + ds
    + '2008-04-08'
    + TOK_PARTVAL
    + hr
    + '11'
    + TOK_SELECT
    + TOK_SELEXPR
    + TOK_TABLE_OR_COL
    + key
    + TOK_SELEXPR
    + TOK_TABLE_OR_COL
    + value
    + TOK_WHERE
    + =
    + TOK_TABLE_OR_COL
    + ds
    + '2008-04-08'
    +
    +
    +STAGE DEPENDENCIES:
    + Stage-1 is a root stage
    + Stage-0 depends on stages: Stage-1
    + Stage-2 depends on stages: Stage-0
    +
    +STAGE PLANS:
    + Stage: Stage-1
    + Map Reduce
    + Map Operator Tree:
    + TableScan
    + alias: srcpart
    + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE
    + GatherStats: false
    + Select Operator
    + expressions: key (type: string), value (type: string)
    + outputColumnNames: _col0, _col1
    + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE
    + File Output Operator
    + compressed: false
    + GlobalTableId: 1
    +#### A masked pattern was here ####
    + NumFilesPerFileSink: 1
    + Static Partition Specification: ds=2008-04-08/hr=11/
    + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE
    +#### A masked pattern was here ####
    + table:
    + input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat
    + output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat
    + properties:
    + bucket_count -1
    + columns key,value
    + columns.comments
    + columns.types string:string
    +#### A masked pattern was here ####
    + name default.list_bucketing_static_part
    + partition_columns ds/hr
    + partition_columns.types string:string
    + serialization.ddl struct list_bucketing_static_part { string key, string value}
    + serialization.format 1
    + serialization.lib org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
    +#### A masked pattern was here ####
    + serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
    + name: default.list_bucketing_static_part
    + TotalFiles: 1
    + GatherStats: true
    + MultiFileSpray: false
    + Path -> Alias:
    +#### A masked pattern was here ####
    + Path -> Partition:
    +#### A masked pattern was here ####
    + Partition
    + base file name: hr=11
    + input format: org.apache.hadoop.mapred.TextInputFormat
    + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
    + partition values:
    + ds 2008-04-08
    + hr 11
    + properties:
    + COLUMN_STATS_ACCURATE true
    + bucket_count -1
    + columns key,value
    + columns.comments 'default','default'
    + columns.types string:string
    +#### A masked pattern was here ####
    + name default.srcpart
    + numFiles 1
    + numRows 500
    + partition_columns ds/hr
    + partition_columns.types string:string
    + rawDataSize 5312
    + serialization.ddl struct srcpart { string key, string value}
    + serialization.format 1
    + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
    + totalSize 5812
    +#### A masked pattern was here ####
    + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
    +
    + input format: org.apache.hadoop.mapred.TextInputFormat
    + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
    + properties:
    + bucket_count -1
    + columns key,value
    + columns.comments 'default','default'
    + columns.types string:string
    +#### A masked pattern was here ####
    + name default.srcpart
    + partition_columns ds/hr
    + partition_columns.types string:string
    + serialization.ddl struct srcpart { string key, string value}
    + serialization.format 1
    + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
    +#### A masked pattern was here ####
    + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
    + name: default.srcpart
    + name: default.srcpart
    +#### A masked pattern was here ####
    + Partition
    + base file name: hr=12
    + input format: org.apache.hadoop.mapred.TextInputFormat
    + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
    + partition values:
    + ds 2008-04-08
    + hr 12
    + properties:
    + COLUMN_STATS_ACCURATE true
    + bucket_count -1
    + columns key,value
    + columns.comments 'default','default'
    + columns.types string:string
    +#### A masked pattern was here ####
    + name default.srcpart
    + numFiles 1
    + numRows 500
    + partition_columns ds/hr
    + partition_columns.types string:string
    + rawDataSize 5312
    + serialization.ddl struct srcpart { string key, string value}
    + serialization.format 1
    + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
    + totalSize 5812
    +#### A masked pattern was here ####
    + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
    +
    + input format: org.apache.hadoop.mapred.TextInputFormat
    + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
    + properties:
    + bucket_count -1
    + columns key,value
    + columns.comments 'default','default'
    + columns.types string:string
    +#### A masked pattern was here ####
    + name default.srcpart
    + partition_columns ds/hr
    + partition_columns.types string:string
    + serialization.ddl struct srcpart { string key, string value}
    + serialization.format 1
    + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
    +#### A masked pattern was here ####
    + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
    + name: default.srcpart
    + name: default.srcpart
    + Truncated Path -> Alias:
    + /srcpart/ds=2008-04-08/hr=11 [$hdt$_0:srcpart]
    + /srcpart/ds=2008-04-08/hr=12 [$hdt$_0:srcpart]
    +
    + Stage: Stage-0
    + Move Operator
    + tables:
    + partition:
    + ds 2008-04-08
    + hr 11
    + replace: true
    +#### A masked pattern was here ####
    + table:
    + input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat
    + output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat
    + properties:
    + bucket_count -1
    + columns key,value
    + columns.comments
    + columns.types string:string
    +#### A masked pattern was here ####
    + name default.list_bucketing_static_part
    + partition_columns ds/hr
    + partition_columns.types string:string
    + serialization.ddl struct list_bucketing_static_part { string key, string value}
    + serialization.format 1
    + serialization.lib org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
    +#### A masked pattern was here ####
    + serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
    + name: default.list_bucketing_static_part
    +
    + Stage: Stage-2
    + Stats-Aggr Operator
    +#### A masked pattern was here ####
    +
    +PREHOOK: query: insert overwrite table list_bucketing_static_part partition (ds = '2008-04-08', hr = '11')
    +select key, value from srcpart where ds = '2008-04-08'
    +PREHOOK: type: QUERY
    +PREHOOK: Input: default@srcpart
    +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11
    +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12
    +PREHOOK: Output: default@list_bucketing_static_part@ds=2008-04-08/hr=11
    +POSTHOOK: query: insert overwrite table list_bucketing_static_part partition (ds = '2008-04-08', hr = '11')
    +select key, value from srcpart where ds = '2008-04-08'
    +POSTHOOK: type: QUERY
    +POSTHOOK: Input: default@srcpart
    +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11
    +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12
    +POSTHOOK: Output: default@list_bucketing_static_part@ds=2008-04-08/hr=11
    +POSTHOOK: Lineage: list_bucketing_static_part PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ]
    +POSTHOOK: Lineage: list_bucketing_static_part PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ]
    +PREHOOK: query: -- check DML result
    +show partitions list_bucketing_static_part
    +PREHOOK: type: SHOWPARTITIONS
    +PREHOOK: Input: default@list_bucketing_static_part
    +POSTHOOK: query: -- check DML result
    +show partitions list_bucketing_static_part
    +POSTHOOK: type: SHOWPARTITIONS
    +POSTHOOK: Input: default@list_bucketing_static_part
    +ds=2008-04-08/hr=11
    +PREHOOK: query: desc formatted list_bucketing_static_part partition (ds='2008-04-08', hr='11')
    +PREHOOK: type: DESCTABLE
    +PREHOOK: Input: default@list_bucketing_static_part
    +POSTHOOK: query: desc formatted list_bucketing_static_part partition (ds='2008-04-08', hr='11')
    +POSTHOOK: type: DESCTABLE
    +POSTHOOK: Input: default@list_bucketing_static_part
    +# col_name data_type comment
    +
    +key string
    +value string
    +
    +# Partition Information
    +# col_name data_type comment
    +
    +ds string
    +hr string
    +
    +# Detailed Partition Information
    +Partition Value: [2008-04-08, 11]
    +Database: default
    +Table: list_bucketing_static_part
    +#### A masked pattern was here ####
    +Protect Mode: None
    +#### A masked pattern was here ####
    +Partition Parameters:
    + COLUMN_STATS_ACCURATE true
    + numFiles 6
    + numRows 1000
    + rawDataSize 9624
    + totalSize 10898
    +#### A masked pattern was here ####
    +
    +# Storage Information
    +SerDe Library: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
    +InputFormat: org.apache.hadoop.hive.ql.io.RCFileInputFormat
    +OutputFormat: org.apache.hadoop.hive.ql.io.RCFileOutputFormat
    +Compressed: No
    +Num Buckets: -1
    +Bucket Columns: []
    +Sort Columns: []
    +Stored As SubDirectories: Yes
    +Skewed Columns: [key]
    +Skewed Values: [[484], [103]]
    +#### A masked pattern was here ####
    +Skewed Value to Truncated Path: {[484]=/list_bucketing_static_part/ds=2008-04-08/hr=11/key=484, [103]=/list_bucketing_static_part/ds=2008-04-08/hr=11/key=103}
    +Storage Desc Params:
    + serialization.format 1
    +PREHOOK: query: -- list bucketing DML with merge. use bucketize to generate a few small files.
    +explain extended
    +insert overwrite table list_bucketing_static_part partition (ds = '2008-04-08', hr = '11')
    +select key, value from srcpart where ds = '2008-04-08'
    +PREHOOK: type: QUERY
    +POSTHOOK: query: -- list bucketing DML with merge. use bucketize to generate a few small files.
    +explain extended
    +insert overwrite table list_bucketing_static_part partition (ds = '2008-04-08', hr = '11')
    +select key, value from srcpart where ds = '2008-04-08'
    +POSTHOOK: type: QUERY
    +ABSTRACT SYNTAX TREE:
    +
    +TOK_QUERY
    + TOK_FROM
    + TOK_TABREF
    + TOK_TABNAME
    + srcpart
    + TOK_INSERT
    + TOK_DESTINATION
    + TOK_TAB
    + TOK_TABNAME
    + list_bucketing_static_part
    + TOK_PARTSPEC
    + TOK_PARTVAL
    + ds
    + '2008-04-08'
    + TOK_PARTVAL
    + hr
    + '11'
    + TOK_SELECT
    + TOK_SELEXPR
    + TOK_TABLE_OR_COL
    + key
    + TOK_SELEXPR
    + TOK_TABLE_OR_COL
    + value
    + TOK_WHERE
    + =
    + TOK_TABLE_OR_COL
    + ds
    + '2008-04-08'
    +
    +
    +STAGE DEPENDENCIES:
    + Stage-1 is a root stage
    + Stage-7 depends on stages: Stage-1 , consists of Stage-4, Stage-3, Stage-5
    + Stage-4
    + Stage-0 depends on stages: Stage-4, Stage-3, Stage-6
    + Stage-2 depends on stages: Stage-0
    + Stage-3
    + Stage-5
    + Stage-6 depends on stages: Stage-5
    +
    +STAGE PLANS:
    + Stage: Stage-1
    + Map Reduce
    + Map Operator Tree:
    + TableScan
    + alias: srcpart
    + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE
    + GatherStats: false
    + Select Operator
    + expressions: key (type: string), value (type: string)
    + outputColumnNames: _col0, _col1
    + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE
    + File Output Operator
    + compressed: false
    + GlobalTableId: 1
    +#### A masked pattern was here ####
    + NumFilesPerFileSink: 1
    + Static Partition Specification: ds=2008-04-08/hr=11/
    + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE
    +#### A masked pattern was here ####
    + table:
    + input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat
    + output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat
    + properties:
    + bucket_count -1
    + columns key,value
    + columns.comments
    + columns.types string:string
    +#### A masked pattern was here ####
    + name default.list_bucketing_static_part
    + partition_columns.types string:string
    + serialization.ddl struct list_bucketing_static_part { string key, string value}
    + serialization.format 1
    + serialization.lib org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
    +#### A masked pattern was here ####
    + serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
    + name: default.list_bucketing_static_part
    + TotalFiles: 1
    + GatherStats: true
    + MultiFileSpray: false
    + Path -> Alias:
    +#### A masked pattern was here ####
    + Path -> Partition:
    +#### A masked pattern was here ####
    + Partition
    + base file name: hr=11
    + input format: org.apache.hadoop.mapred.TextInputFormat
    + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
    + partition values:
    + ds 2008-04-08
    + hr 11
    + properties:
    + COLUMN_STATS_ACCURATE true
    + bucket_count -1
    + columns key,value
    + columns.comments 'default','default'
    + columns.types string:string
    +#### A masked pattern was here ####
    + name default.srcpart
    + numFiles 1
    + numRows 500
    + partition_columns ds/hr
    + partition_columns.types string:string
    + rawDataSize 5312
    + serialization.ddl struct srcpart { string key, string value}
    + serialization.format 1
    + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
    + totalSize 5812
    +#### A masked pattern was here ####
    + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
    +
    + input format: org.apache.hadoop.mapred.TextInputFormat
    + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
    + properties:
    + bucket_count -1
    + columns key,value
    + columns.comments 'default','default'
    + columns.types string:string
    +#### A masked pattern was here ####
    + name default.srcpart
    + partition_columns ds/hr
    + partition_columns.types string:string
    + serialization.ddl struct srcpart { string key, string value}
    + serialization.format 1
    + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
    +#### A masked pattern was here ####
    + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
    + name: default.srcpart
    + name: default.srcpart
    +#### A masked pattern was here ####
    + Partition
    + base file name: hr=12
    + input format: org.apache.hadoop.mapred.TextInputFormat
    + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
    + partition values:
    + ds 2008-04-08
    + hr 12
    + properties:
    + COLUMN_STATS_ACCURATE true
    + bucket_count -1
    + columns key,value
    + columns.comments 'default','default'
    + columns.types string:string
    +#### A masked pattern was here ####
    + name default.srcpart
    + numFiles 1
    + numRows 500
    + partition_columns ds/hr
    + partition_columns.types string:string
    + rawDataSize 5312
    + serialization.ddl struct srcpart { string key, string value}
    + serialization.format 1
    + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
    + totalSize 5812
    +#### A masked pattern was here ####
    + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
    +
    + input format: org.apache.hadoop.mapred.TextInputFormat
    + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
    + properties:
    + bucket_count -1
    + columns key,value
    + columns.comments 'default','default'
    + columns.types string:string
    +#### A masked pattern was here ####
    + name default.srcpart
    + partition_columns ds/hr
    + partition_columns.types string:string
    + serialization.ddl struct srcpart { string key, string value}
    + serialization.format 1
    + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
    +#### A masked pattern was here ####
    + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
    + name: default.srcpart
    + name: default.srcpart
    + Truncated Path -> Alias:
    + /srcpart/ds=2008-04-08/hr=11 [$hdt$_0:srcpart]
    + /srcpart/ds=2008-04-08/hr=12 [$hdt$_0:srcpart]
    +
    + Stage: Stage-7
    + Conditional Operator
    +
    + Stage: Stage-4
    + Move Operator
    + files:
    + hdfs directory: true
    +#### A masked pattern was here ####
    +
    + Stage: Stage-0
    + Move Operator
    + tables:
    + partition:
    + ds 2008-04-08
    + hr 11
    + replace: true
    +#### A masked pattern was here ####
    + table:
    + input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat
    + output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat
    + properties:
    + bucket_count -1
    + columns key,value
    + columns.comments
    + columns.types string:string
    +#### A masked pattern was here ####
    + name default.list_bucketing_static_part
    + partition_columns.types string:string
    + serialization.ddl struct list_bucketing_static_part { string key, string value}
    + serialization.format 1
    + serialization.lib org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
    +#### A masked pattern was here ####
    + serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
    + name: default.list_bucketing_static_part
    +
    + Stage: Stage-2
    + Stats-Aggr Operator
    +#### A masked pattern was here ####
    +
    + Stage: Stage-3
    + Merge File Operator
    + Map Operator Tree:
    + RCFile Merge Operator
    + merge level: block
    + Path -> Alias:
    +#### A masked pattern was here ####
    + Path -> Partition:
    +#### A masked pattern was here ####
    + Partition
    + input format: org.apache.hadoop.hive.ql.io.rcfile.merge.RCFileBlockMergeInputFormat
    + output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat
    + properties:
    + bucket_count -1
    + columns key,value
    + columns.comments
    + columns.types string:string
    +#### A masked pattern was here ####
    + name default.list_bucketing_static_part
    + partition_columns.types string:string
    + serialization.ddl struct list_bucketing_static_part { string key, string value}
    + serialization.format 1
    + serialization.lib org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
    +#### A masked pattern was here ####
    + serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
    +
    + input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat
    + output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat
    + properties:
    + bucket_count -1
    + columns key,value
    + columns.comments
    + columns.types string:string
    +#### A masked pattern was here ####
    + name default.list_bucketing_static_part
    + partition_columns.types string:string
    + serialization.ddl struct list_bucketing_static_part { string key, string value}
    + serialization.format 1
    + serialization.lib org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
    +#### A masked pattern was here ####
    + serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
    + name: default.list_bucketing_static_part
    + name: default.list_bucketing_static_part
    + input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat
    + Truncated Path -> Alias:
    +#### A masked pattern was here ####
    +
    + Stage: Stage-5
    + Merge File Operator
    + Map Operator Tree:
    + RCFile Merge Operator
    + merge level: block
    + Path -> Alias:
    +#### A masked pattern was here ####
    + Path -> Partition:
    +#### A masked pattern was here ####
    + Partition
    + input format: org.apache.hadoop.hive.ql.io.rcfile.merge.RCFileBlockMergeInputFormat
    + output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat
    + properties:
    + bucket_count -1
    + columns key,value
    + columns.comments
    + columns.types string:string
    +#### A masked pattern was here ####
    + name default.list_bucketing_static_part
    + partition_columns.types string:string
    + serialization.ddl struct list_bucketing_static_part { string key, string value}
    + serialization.format 1
    + serialization.lib org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
    +#### A masked pattern was here ####
    + serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
    +
    + input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat
    + output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat
    + properties:
    + bucket_count -1
    + columns key,value
    + columns.comments
    + columns.types string:string
    +#### A masked pattern was here ####
    + name default.list_bucketing_static_part
    + partition_columns.types string:string
    + serialization.ddl struct list_bucketing_static_part { string key, string value}
    + serialization.format 1
    + serialization.lib org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
    +#### A masked pattern was here ####
    + serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
    + name: default.list_bucketing_static_part
    + name: default.list_bucketing_static_part
    + input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat
    + Truncated Path -> Alias:
    +#### A masked pattern was here ####
    +
    + Stage: Stage-6
    + Move Operator
    + files:
    + hdfs directory: true
    +#### A masked pattern was here ####
    +
    +PREHOOK: query: insert overwrite table list_bucketing_static_part partition (ds = '2008-04-08', hr = '11')
    +select key, value from srcpart where ds = '2008-04-08'
    +PREHOOK: type: QUERY
    +PREHOOK: Input: default@srcpart
    +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11
    +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12
    +PREHOOK: Output: default@list_bucketing_static_part@ds=2008-04-08/hr=11
    +POSTHOOK: query: insert overwrite table list_bucketing_static_part partition (ds = '2008-04-08', hr = '11')
    +select key, value from srcpart where ds = '2008-04-08'
    +POSTHOOK: type: QUERY
    +POSTHOOK: Input: default@srcpart
    +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11
    +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12
    +POSTHOOK: Output: default@list_bucketing_static_part@ds=2008-04-08/hr=11
    +POSTHOOK: Lineage: list_bucketing_static_part PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ]
    +POSTHOOK: Lineage: list_bucketing_static_part PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ]
    +PREHOOK: query: -- check DML result
    +show partitions list_bucketing_static_part
    +PREHOOK: type: SHOWPARTITIONS
    +PREHOOK: Input: default@list_bucketing_static_part
    +POSTHOOK: query: -- check DML result
    +show partitions list_bucketing_static_part
    +POSTHOOK: type: SHOWPARTITIONS
    +POSTHOOK: Input: default@list_bucketing_static_part
    +ds=2008-04-08/hr=11
    +PREHOOK: query: desc formatted list_bucketing_static_part partition (ds='2008-04-08', hr='11')
    +PREHOOK: type: DESCTABLE
    +PREHOOK: Input: default@list_bucketing_static_part
    +POSTHOOK: query: desc formatted list_bucketing_static_part partition (ds='2008-04-08', hr='11')
    +POSTHOOK: type: DESCTABLE
    +POSTHOOK: Input: default@list_bucketing_static_part
    +# col_name data_type comment
    +
    +key string
    +value string
    +
    +# Partition Information
    +# col_name data_type comment
    +
    +ds string
    +hr string
    +
    +# Detailed Partition Information
    +Partition Value: [2008-04-08, 11]
    +Database: default
    +Table: list_bucketing_static_part
    +#### A masked pattern was here ####
    +Protect Mode: None
    +#### A masked pattern was here ####
    +Partition Parameters:
    + COLUMN_STATS_ACCURATE true
    + numFiles 4
    + numRows 1000
    + rawDataSize 9624
    + totalSize 10786
    +#### A masked pattern was here ####
    +
    +# Storage Information
    +SerDe Library: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
    +InputFormat: org.apache.hadoop.hive.ql.io.RCFileInputFormat
    +OutputFormat: org.apache.hadoop.hive.ql.io.RCFileOutputFormat
    +Compressed: No
    +Num Buckets: -1
    +Bucket Columns: []
    +Sort Columns: []
    +Stored As SubDirectories: Yes
    +Skewed Columns: [key]
    +Skewed Values: [[484], [103]]
    +#### A masked pattern was here ####
    +Skewed Value to Truncated Path: {[484]=/list_bucketing_static_part/ds=2008-04-08/hr=11/key=484, [103]=/list_bucketing_static_part/ds=2008-04-08/hr=11/key=103}
    +Storage Desc Params:
    + serialization.format 1
    +PREHOOK: query: select count(1) from srcpart where ds = '2008-04-08'
    +PREHOOK: type: QUERY
    +PREHOOK: Input: default@srcpart
    +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11
    +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12
    +#### A masked pattern was here ####
    +POSTHOOK: query: select count(1) from srcpart where ds = '2008-04-08'
    +POSTHOOK: type: QUERY
    +POSTHOOK: Input: default@srcpart
    +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11
    +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12
    +#### A masked pattern was here ####
    +1000
    +PREHOOK: query: select count(*) from list_bucketing_static_part
    +PREHOOK: type: QUERY
    +PREHOOK: Input: default@list_bucketing_static_part
    +PREHOOK: Input: default@list_bucketing_static_part@ds=2008-04-08/hr=11
    +#### A masked pattern was here ####
    +POSTHOOK: query: select count(*) from list_bucketing_static_part
    +POSTHOOK: type: QUERY
    +POSTHOOK: Input: default@list_bucketing_static_part
    +POSTHOOK: Input: default@list_bucketing_static_part@ds=2008-04-08/hr=11
    +#### A masked pattern was here ####
    +1000
    +PREHOOK: query: explain extended
    +select * from list_bucketing_static_part where ds = '2008-04-08' and hr = '11' and key = '484' and value = 'val_484'
    +PREHOOK: type: QUERY
    +POSTHOOK: query: explain extended
    +select * from list_bucketing_static_part where ds = '2008-04-08' and hr = '11' and key = '484' and value = 'val_484'
    +POSTHOOK: type: QUERY
    +ABSTRACT SYNTAX TREE:
    +
    +TOK_QUERY
    + TOK_FROM
    + TOK_TABREF
    + TOK_TABNAME
    + list_bucketing_static_part
    + TOK_INSERT
    + TOK_DESTINATION
    + TOK_DIR
    + TOK_TMP_FILE
    + TOK_SELECT
    + TOK_SELEXPR
    + TOK_ALLCOLREF
    + TOK_WHERE
    + and
    + and
    + and
    + =
    + TOK_TABLE_OR_COL
    + ds
    + '2008-04-08'
    + =
    + TOK_TABLE_OR_COL
    + hr
    + '11'
    + =
    + TOK_TABLE_OR_COL
    + key
    + '484'
    + =
    + TOK_TABLE_OR_COL
    + value
    + 'val_484'
    +
    +
    +STAGE DEPENDENCIES:
    + Stage-1 is a root stage
    + Stage-0 depends on stages: Stage-1
    +
    +STAGE PLANS:
    + Stage: Stage-1
    + Map Reduce
    + Map Operator Tree:
    + TableScan
    + alias: list_bucketing_static_part
    + Statistics: Num rows: 1000 Data size: 9624 Basic stats: COMPLETE Column stats: NONE
    + GatherStats: false
    + Filter Operator
    + isSamplingPred: false
    + predicate: ((key = '484') and (value = 'val_484')) (type: boolean)
    + Statistics: Num rows: 250 Data size: 2406 Basic stats: COMPLETE Column stats: NONE
    + Select Operator
    + expressions: '484' (type: string), 'val_484' (type: string), '2008-04-08' (type: string), '11' (type: string)
    + outputColumnNames: _col0, _col1, _col2, _col3
    + Statistics: Num rows: 250 Data size: 2406 Basic stats: COMPLETE Column stats: NONE
    + File Output Operator
    + compressed: false
    + GlobalTableId: 0
    +#### A masked pattern was here ####
    + NumFilesPerFileSink: 1
    + Statistics: Num rows: 250 Data size: 2406 Basic stats: COMPLETE Column stats: NONE
    +#### A masked pattern was here ####
    + table:
    + input format: org.apache.hadoop.mapred.TextInputFormat
    + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
    + properties:
    + columns _col0,_col1,_col2,_col3
    + columns.types string:string:string:string
    + escape.delim \
    + hive.serialization.extend.nesting.levels true
    + serialization.format 1
    + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
    + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
    + TotalFiles: 1
    + GatherStats: false
    + MultiFileSpray: false
    + Path -> Alias:
    +#### A masked pattern was here ####
    + Path -> Partition:
    +#### A masked pattern was here ####
    + Partition
    + base file name: key=484
    + input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat
    + output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat
    + partition values:
    + ds 2008-04-08
    + hr 11
    + properties:
    + COLUMN_STATS_ACCURATE true
    + bucket_count -1
    + columns key,value
    + columns.comments
    + columns.types string:string
    +#### A masked pattern was here ####
    + name default.list_bucketing_static_part
    + numFiles 4
    + numRows 1000
    + partition_columns ds/hr
    + partition_columns.types string:string
    + rawDataSize 9624
    + serialization.ddl struct list_bucketing_static_part { string key, string value}
    + serialization.format 1
    + serialization.lib org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
    + totalSize 10786
    +#### A masked pattern was here ####
    + serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
    +
    + input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat
    + output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat
    + properties:
    + bucket_count -1
    + columns key,value
    + columns.comments
    + columns.types string:string
    +#### A masked pattern was here ####
    + name default.list_bucketing_static_part
    + partition_columns ds/hr
    + partition_columns.types string:string
    + serialization.ddl struct list_bucketing_static_part { string key, string value}
    + serialization.format 1
    + serialization.lib org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
    +#### A masked pattern was here ####
    + serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
    + name: default.list_bucketing_static_part
    + name: default.list_bucketing_static_part
    + Truncated Path -> Alias:
    + /list_bucketing_static_part/ds=2008-04-08/hr=11/key=484 [list_bucketing_static_part]
    +
    + Stage: Stage-0
    + Fetch Operator
    + limit: -1
    + Processor Tree:
    + ListSink
    +
    +PREHOOK: query: select * from list_bucketing_static_part where ds = '2008-04-08' and hr = '11' and key = '484' and value = 'val_484'
    +PREHOOK: type: QUERY
    +PREHOOK: Input: default@list_bucketing_static_part
    +PREHOOK: Input: default@list_bucketing_static_part@ds=2008-04-08/hr=11
    +#### A masked pattern was here ####
    +POSTHOOK: query: select * from list_bucketing_static_part where ds = '2008-04-08' and hr = '11' and key = '484' and value = 'val_484'
    +POSTHOOK: type: QUERY
    +POSTHOOK: Input: default@list_bucketing_static_part
    +POSTHOOK: Input: default@list_bucketing_static_part@ds=2008-04-08/hr=11
    +#### A masked pattern was here ####
    +484 val_484 2008-04-08 11
    +484 val_484 2008-04-08 11
    +PREHOOK: query: select * from srcpart where ds = '2008-04-08' and key = '484' and value = 'val_484'
    +PREHOOK: type: QUERY
    +PREHOOK: Input: default@srcpart
    +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11
    +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12
    +#### A masked pattern was here ####
    +POSTHOOK: query: select * from srcpart where ds = '2008-04-08' and key = '484' and value = 'val_484'
    +POSTHOOK: type: QUERY
    +POSTHOOK: Input: default@srcpart
    +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11
    +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12
    +#### A masked pattern was here ####
    +484 val_484 2008-04-08 11
    +484 val_484 2008-04-08 12
    +PREHOOK: query: -- clean up
    +drop table list_bucketing_static_part
    +PREHOOK: type: DROPTABLE
    +PREHOOK: Input: default@list_bucketing_static_part
    +PREHOOK: Output: default@list_bucketing_static_part
    +POSTHOOK: query: -- clean up
    +drop table list_bucketing_static_part
    +POSTHOOK: type: DROPTABLE
    +POSTHOOK: Input: default@list_bucketing_static_part
    +POSTHOOK: Output: default@list_bucketing_static_part
  • Brock at Dec 25, 2014 at 6:06 pm
    Added: hive/trunk/ql/src/test/results/clientpositive/list_bucket_dml_2.q.java1.7.out
    URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/list_bucket_dml_2.q.java1.7.out?rev=1647912&view=auto
    ==============================================================================
    --- hive/trunk/ql/src/test/results/clientpositive/list_bucket_dml_2.q.java1.7.out (added)
    +++ hive/trunk/ql/src/test/results/clientpositive/list_bucket_dml_2.q.java1.7.out Thu Dec 25 18:06:30 2014
    @@ -0,0 +1,694 @@
    +PREHOOK: query: -- INCLUDE_HADOOP_MAJOR_VERSIONS(0.23)
    +-- SORT_QUERY_RESULTS
    +-- JAVA_VERSION_SPECIFIC_OUTPUT
    +
    +-- list bucketing DML: static partition. multiple skewed columns.
    +-- ds=2008-04-08/hr=11/HIVE_DEFAULT_LIST_BUCKETING_DIR_NAME:
    +-- 5263 000000_0
    +-- 5263 000001_0
    +-- ds=2008-04-08/hr=11/key=103/value=val_103:
    +-- 99 000000_0
    +-- 99 000001_0
    +-- ds=2008-04-08/hr=11/key=484/value=val_484:
    +-- 87 000000_0
    +-- 87 000001_0
    +
    +-- create a skewed table
    +create table list_bucketing_static_part (key String, value String)
    + partitioned by (ds String, hr String)
    + skewed by (key, value) on (('484','val_484'),('51','val_14'),('103','val_103'))
    + stored as DIRECTORIES
    + STORED AS RCFILE
    +PREHOOK: type: CREATETABLE
    +PREHOOK: Output: database:default
    +PREHOOK: Output: default@list_bucketing_static_part
    +POSTHOOK: query: -- INCLUDE_HADOOP_MAJOR_VERSIONS(0.23)
    +-- SORT_QUERY_RESULTS
    +-- JAVA_VERSION_SPECIFIC_OUTPUT
    +
    +-- list bucketing DML: static partition. multiple skewed columns.
    +-- ds=2008-04-08/hr=11/HIVE_DEFAULT_LIST_BUCKETING_DIR_NAME:
    +-- 5263 000000_0
    +-- 5263 000001_0
    +-- ds=2008-04-08/hr=11/key=103/value=val_103:
    +-- 99 000000_0
    +-- 99 000001_0
    +-- ds=2008-04-08/hr=11/key=484/value=val_484:
    +-- 87 000000_0
    +-- 87 000001_0
    +
    +-- create a skewed table
    +create table list_bucketing_static_part (key String, value String)
    + partitioned by (ds String, hr String)
    + skewed by (key, value) on (('484','val_484'),('51','val_14'),('103','val_103'))
    + stored as DIRECTORIES
    + STORED AS RCFILE
    +POSTHOOK: type: CREATETABLE
    +POSTHOOK: Output: database:default
    +POSTHOOK: Output: default@list_bucketing_static_part
    +PREHOOK: query: -- list bucketing DML without merge. use bucketize to generate a few small files.
    +explain extended
    +insert overwrite table list_bucketing_static_part partition (ds = '2008-04-08', hr = '11')
    +select key, value from srcpart where ds = '2008-04-08'
    +PREHOOK: type: QUERY
    +POSTHOOK: query: -- list bucketing DML without merge. use bucketize to generate a few small files.
    +explain extended
    +insert overwrite table list_bucketing_static_part partition (ds = '2008-04-08', hr = '11')
    +select key, value from srcpart where ds = '2008-04-08'
    +POSTHOOK: type: QUERY
    +ABSTRACT SYNTAX TREE:
    +
    +TOK_QUERY
    + TOK_FROM
    + TOK_TABREF
    + TOK_TABNAME
    + srcpart
    + TOK_INSERT
    + TOK_DESTINATION
    + TOK_TAB
    + TOK_TABNAME
    + list_bucketing_static_part
    + TOK_PARTSPEC
    + TOK_PARTVAL
    + ds
    + '2008-04-08'
    + TOK_PARTVAL
    + hr
    + '11'
    + TOK_SELECT
    + TOK_SELEXPR
    + TOK_TABLE_OR_COL
    + key
    + TOK_SELEXPR
    + TOK_TABLE_OR_COL
    + value
    + TOK_WHERE
    + =
    + TOK_TABLE_OR_COL
    + ds
    + '2008-04-08'
    +
    +
    +STAGE DEPENDENCIES:
    + Stage-1 is a root stage
    + Stage-0 depends on stages: Stage-1
    + Stage-2 depends on stages: Stage-0
    +
    +STAGE PLANS:
    + Stage: Stage-1
    + Map Reduce
    + Map Operator Tree:
    + TableScan
    + alias: srcpart
    + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE
    + GatherStats: false
    + Select Operator
    + expressions: key (type: string), value (type: string)
    + outputColumnNames: _col0, _col1
    + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE
    + File Output Operator
    + compressed: false
    + GlobalTableId: 1
    +#### A masked pattern was here ####
    + NumFilesPerFileSink: 1
    + Static Partition Specification: ds=2008-04-08/hr=11/
    + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE
    +#### A masked pattern was here ####
    + table:
    + input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat
    + output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat
    + properties:
    + bucket_count -1
    + columns key,value
    + columns.comments
    + columns.types string:string
    +#### A masked pattern was here ####
    + name default.list_bucketing_static_part
    + partition_columns ds/hr
    + partition_columns.types string:string
    + serialization.ddl struct list_bucketing_static_part { string key, string value}
    + serialization.format 1
    + serialization.lib org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
    +#### A masked pattern was here ####
    + serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
    + name: default.list_bucketing_static_part
    + TotalFiles: 1
    + GatherStats: true
    + MultiFileSpray: false
    + Path -> Alias:
    +#### A masked pattern was here ####
    + Path -> Partition:
    +#### A masked pattern was here ####
    + Partition
    + base file name: hr=11
    + input format: org.apache.hadoop.mapred.TextInputFormat
    + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
    + partition values:
    + ds 2008-04-08
    + hr 11
    + properties:
    + COLUMN_STATS_ACCURATE true
    + bucket_count -1
    + columns key,value
    + columns.comments 'default','default'
    + columns.types string:string
    +#### A masked pattern was here ####
    + name default.srcpart
    + numFiles 1
    + numRows 500
    + partition_columns ds/hr
    + partition_columns.types string:string
    + rawDataSize 5312
    + serialization.ddl struct srcpart { string key, string value}
    + serialization.format 1
    + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
    + totalSize 5812
    +#### A masked pattern was here ####
    + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
    +
    + input format: org.apache.hadoop.mapred.TextInputFormat
    + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
    + properties:
    + bucket_count -1
    + columns key,value
    + columns.comments 'default','default'
    + columns.types string:string
    +#### A masked pattern was here ####
    + name default.srcpart
    + partition_columns ds/hr
    + partition_columns.types string:string
    + serialization.ddl struct srcpart { string key, string value}
    + serialization.format 1
    + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
    +#### A masked pattern was here ####
    + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
    + name: default.srcpart
    + name: default.srcpart
    +#### A masked pattern was here ####
    + Partition
    + base file name: hr=12
    + input format: org.apache.hadoop.mapred.TextInputFormat
    + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
    + partition values:
    + ds 2008-04-08
    + hr 12
    + properties:
    + COLUMN_STATS_ACCURATE true
    + bucket_count -1
    + columns key,value
    + columns.comments 'default','default'
    + columns.types string:string
    +#### A masked pattern was here ####
    + name default.srcpart
    + numFiles 1
    + numRows 500
    + partition_columns ds/hr
    + partition_columns.types string:string
    + rawDataSize 5312
    + serialization.ddl struct srcpart { string key, string value}
    + serialization.format 1
    + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
    + totalSize 5812
    +#### A masked pattern was here ####
    + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
    +
    + input format: org.apache.hadoop.mapred.TextInputFormat
    + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
    + properties:
    + bucket_count -1
    + columns key,value
    + columns.comments 'default','default'
    + columns.types string:string
    +#### A masked pattern was here ####
    + name default.srcpart
    + partition_columns ds/hr
    + partition_columns.types string:string
    + serialization.ddl struct srcpart { string key, string value}
    + serialization.format 1
    + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
    +#### A masked pattern was here ####
    + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
    + name: default.srcpart
    + name: default.srcpart
    + Truncated Path -> Alias:
    + /srcpart/ds=2008-04-08/hr=11 [$hdt$_0:srcpart]
    + /srcpart/ds=2008-04-08/hr=12 [$hdt$_0:srcpart]
    +
    + Stage: Stage-0
    + Move Operator
    + tables:
    + partition:
    + ds 2008-04-08
    + hr 11
    + replace: true
    +#### A masked pattern was here ####
    + table:
    + input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat
    + output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat
    + properties:
    + bucket_count -1
    + columns key,value
    + columns.comments
    + columns.types string:string
    +#### A masked pattern was here ####
    + name default.list_bucketing_static_part
    + partition_columns ds/hr
    + partition_columns.types string:string
    + serialization.ddl struct list_bucketing_static_part { string key, string value}
    + serialization.format 1
    + serialization.lib org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
    +#### A masked pattern was here ####
    + serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
    + name: default.list_bucketing_static_part
    +
    + Stage: Stage-2
    + Stats-Aggr Operator
    +#### A masked pattern was here ####
    +
    +PREHOOK: query: insert overwrite table list_bucketing_static_part partition (ds = '2008-04-08', hr = '11')
    +select key, value from srcpart where ds = '2008-04-08'
    +PREHOOK: type: QUERY
    +PREHOOK: Input: default@srcpart
    +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11
    +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12
    +PREHOOK: Output: default@list_bucketing_static_part@ds=2008-04-08/hr=11
    +POSTHOOK: query: insert overwrite table list_bucketing_static_part partition (ds = '2008-04-08', hr = '11')
    +select key, value from srcpart where ds = '2008-04-08'
    +POSTHOOK: type: QUERY
    +POSTHOOK: Input: default@srcpart
    +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11
    +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12
    +POSTHOOK: Output: default@list_bucketing_static_part@ds=2008-04-08/hr=11
    +POSTHOOK: Lineage: list_bucketing_static_part PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ]
    +POSTHOOK: Lineage: list_bucketing_static_part PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ]
    +PREHOOK: query: -- check DML result
    +show partitions list_bucketing_static_part
    +PREHOOK: type: SHOWPARTITIONS
    +PREHOOK: Input: default@list_bucketing_static_part
    +POSTHOOK: query: -- check DML result
    +show partitions list_bucketing_static_part
    +POSTHOOK: type: SHOWPARTITIONS
    +POSTHOOK: Input: default@list_bucketing_static_part
    +ds=2008-04-08/hr=11
    +PREHOOK: query: desc formatted list_bucketing_static_part partition (ds='2008-04-08', hr='11')
    +PREHOOK: type: DESCTABLE
    +PREHOOK: Input: default@list_bucketing_static_part
    +POSTHOOK: query: desc formatted list_bucketing_static_part partition (ds='2008-04-08', hr='11')
    +POSTHOOK: type: DESCTABLE
    +POSTHOOK: Input: default@list_bucketing_static_part
    +# col_name data_type comment
    +
    +key string
    +value string
    +
    +# Partition Information
    +# col_name data_type comment
    +
    +ds string
    +hr string
    +
    +# Detailed Partition Information
    +Partition Value: [2008-04-08, 11]
    +Database: default
    +Table: list_bucketing_static_part
    +#### A masked pattern was here ####
    +Protect Mode: None
    +#### A masked pattern was here ####
    +Partition Parameters:
    + COLUMN_STATS_ACCURATE true
    + numFiles 6
    + numRows 1000
    + rawDataSize 9624
    + totalSize 10898
    +#### A masked pattern was here ####
    +
    +# Storage Information
    +SerDe Library: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
    +InputFormat: org.apache.hadoop.hive.ql.io.RCFileInputFormat
    +OutputFormat: org.apache.hadoop.hive.ql.io.RCFileOutputFormat
    +Compressed: No
    +Num Buckets: -1
    +Bucket Columns: []
    +Sort Columns: []
    +Stored As SubDirectories: Yes
    +Skewed Columns: [key, value]
    +Skewed Values: [[484, val_484], [51, val_14], [103, val_103]]
    +#### A masked pattern was here ####
    +Skewed Value to Truncated Path: {[103, val_103]=/list_bucketing_static_part/ds=2008-04-08/hr=11/key=103/value=val_103, [484, val_484]=/list_bucketing_static_part/ds=2008-04-08/hr=11/key=484/value=val_484}
    +Storage Desc Params:
    + serialization.format 1
    +PREHOOK: query: select count(1) from srcpart where ds = '2008-04-08'
    +PREHOOK: type: QUERY
    +PREHOOK: Input: default@srcpart
    +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11
    +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12
    +#### A masked pattern was here ####
    +POSTHOOK: query: select count(1) from srcpart where ds = '2008-04-08'
    +POSTHOOK: type: QUERY
    +POSTHOOK: Input: default@srcpart
    +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11
    +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12
    +#### A masked pattern was here ####
    +1000
    +PREHOOK: query: select count(*) from list_bucketing_static_part
    +PREHOOK: type: QUERY
    +PREHOOK: Input: default@list_bucketing_static_part
    +PREHOOK: Input: default@list_bucketing_static_part@ds=2008-04-08/hr=11
    +#### A masked pattern was here ####
    +POSTHOOK: query: select count(*) from list_bucketing_static_part
    +POSTHOOK: type: QUERY
    +POSTHOOK: Input: default@list_bucketing_static_part
    +POSTHOOK: Input: default@list_bucketing_static_part@ds=2008-04-08/hr=11
    +#### A masked pattern was here ####
    +1000
    +PREHOOK: query: explain extended
    +select * from list_bucketing_static_part where ds = '2008-04-08' and hr = '11' and key = '484' and value = 'val_484'
    +PREHOOK: type: QUERY
    +POSTHOOK: query: explain extended
    +select * from list_bucketing_static_part where ds = '2008-04-08' and hr = '11' and key = '484' and value = 'val_484'
    +POSTHOOK: type: QUERY
    +ABSTRACT SYNTAX TREE:
    +
    +TOK_QUERY
    + TOK_FROM
    + TOK_TABREF
    + TOK_TABNAME
    + list_bucketing_static_part
    + TOK_INSERT
    + TOK_DESTINATION
    + TOK_DIR
    + TOK_TMP_FILE
    + TOK_SELECT
    + TOK_SELEXPR
    + TOK_ALLCOLREF
    + TOK_WHERE
    + and
    + and
    + and
    + =
    + TOK_TABLE_OR_COL
    + ds
    + '2008-04-08'
    + =
    + TOK_TABLE_OR_COL
    + hr
    + '11'
    + =
    + TOK_TABLE_OR_COL
    + key
    + '484'
    + =
    + TOK_TABLE_OR_COL
    + value
    + 'val_484'
    +
    +
    +STAGE DEPENDENCIES:
    + Stage-1 is a root stage
    + Stage-0 depends on stages: Stage-1
    +
    +STAGE PLANS:
    + Stage: Stage-1
    + Map Reduce
    + Map Operator Tree:
    + TableScan
    + alias: list_bucketing_static_part
    + Statistics: Num rows: 1000 Data size: 9624 Basic stats: COMPLETE Column stats: NONE
    + GatherStats: false
    + Filter Operator
    + isSamplingPred: false
    + predicate: ((key = '484') and (value = 'val_484')) (type: boolean)
    + Statistics: Num rows: 250 Data size: 2406 Basic stats: COMPLETE Column stats: NONE
    + Select Operator
    + expressions: '484' (type: string), 'val_484' (type: string), '2008-04-08' (type: string), '11' (type: string)
    + outputColumnNames: _col0, _col1, _col2, _col3
    + Statistics: Num rows: 250 Data size: 2406 Basic stats: COMPLETE Column stats: NONE
    + File Output Operator
    + compressed: false
    + GlobalTableId: 0
    +#### A masked pattern was here ####
    + NumFilesPerFileSink: 1
    + Statistics: Num rows: 250 Data size: 2406 Basic stats: COMPLETE Column stats: NONE
    +#### A masked pattern was here ####
    + table:
    + input format: org.apache.hadoop.mapred.TextInputFormat
    + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
    + properties:
    + columns _col0,_col1,_col2,_col3
    + columns.types string:string:string:string
    + escape.delim \
    + hive.serialization.extend.nesting.levels true
    + serialization.format 1
    + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
    + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
    + TotalFiles: 1
    + GatherStats: false
    + MultiFileSpray: false
    + Path -> Alias:
    +#### A masked pattern was here ####
    + Path -> Partition:
    +#### A masked pattern was here ####
    + Partition
    + base file name: value=val_484
    + input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat
    + output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat
    + partition values:
    + ds 2008-04-08
    + hr 11
    + properties:
    + COLUMN_STATS_ACCURATE true
    + bucket_count -1
    + columns key,value
    + columns.comments
    + columns.types string:string
    +#### A masked pattern was here ####
    + name default.list_bucketing_static_part
    + numFiles 6
    + numRows 1000
    + partition_columns ds/hr
    + partition_columns.types string:string
    + rawDataSize 9624
    + serialization.ddl struct list_bucketing_static_part { string key, string value}
    + serialization.format 1
    + serialization.lib org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
    + totalSize 10898
    +#### A masked pattern was here ####
    + serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
    +
    + input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat
    + output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat
    + properties:
    + bucket_count -1
    + columns key,value
    + columns.comments
    + columns.types string:string
    +#### A masked pattern was here ####
    + name default.list_bucketing_static_part
    + partition_columns ds/hr
    + partition_columns.types string:string
    + serialization.ddl struct list_bucketing_static_part { string key, string value}
    + serialization.format 1
    + serialization.lib org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
    +#### A masked pattern was here ####
    + serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
    + name: default.list_bucketing_static_part
    + name: default.list_bucketing_static_part
    + Truncated Path -> Alias:
    + /list_bucketing_static_part/ds=2008-04-08/hr=11/key=484/value=val_484 [list_bucketing_static_part]
    +
    + Stage: Stage-0
    + Fetch Operator
    + limit: -1
    + Processor Tree:
    + ListSink
    +
    +PREHOOK: query: select * from list_bucketing_static_part where ds = '2008-04-08' and hr = '11' and key = '484' and value = 'val_484'
    +PREHOOK: type: QUERY
    +PREHOOK: Input: default@list_bucketing_static_part
    +PREHOOK: Input: default@list_bucketing_static_part@ds=2008-04-08/hr=11
    +#### A masked pattern was here ####
    +POSTHOOK: query: select * from list_bucketing_static_part where ds = '2008-04-08' and hr = '11' and key = '484' and value = 'val_484'
    +POSTHOOK: type: QUERY
    +POSTHOOK: Input: default@list_bucketing_static_part
    +POSTHOOK: Input: default@list_bucketing_static_part@ds=2008-04-08/hr=11
    +#### A masked pattern was here ####
    +484 val_484 2008-04-08 11
    +484 val_484 2008-04-08 11
    +PREHOOK: query: select * from srcpart where ds = '2008-04-08' and key = '484' and value = 'val_484'
    +PREHOOK: type: QUERY
    +PREHOOK: Input: default@srcpart
    +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11
    +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12
    +#### A masked pattern was here ####
    +POSTHOOK: query: select * from srcpart where ds = '2008-04-08' and key = '484' and value = 'val_484'
    +POSTHOOK: type: QUERY
    +POSTHOOK: Input: default@srcpart
    +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11
    +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12
    +#### A masked pattern was here ####
    +484 val_484 2008-04-08 11
    +484 val_484 2008-04-08 12
    +PREHOOK: query: -- 51 and val_51 in the table so skewed data for 51 and val_14 should be none
    +-- but query should succeed for 51 or 51 and val_14
    +select * from srcpart where ds = '2008-04-08' and key = '51'
    +PREHOOK: type: QUERY
    +PREHOOK: Input: default@srcpart
    +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11
    +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12
    +#### A masked pattern was here ####
    +POSTHOOK: query: -- 51 and val_51 in the table so skewed data for 51 and val_14 should be none
    +-- but query should succeed for 51 or 51 and val_14
    +select * from srcpart where ds = '2008-04-08' and key = '51'
    +POSTHOOK: type: QUERY
    +POSTHOOK: Input: default@srcpart
    +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11
    +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12
    +#### A masked pattern was here ####
    +51 val_51 2008-04-08 11
    +51 val_51 2008-04-08 11
    +51 val_51 2008-04-08 12
    +51 val_51 2008-04-08 12
    +PREHOOK: query: select * from list_bucketing_static_part where key = '51'
    +PREHOOK: type: QUERY
    +PREHOOK: Input: default@list_bucketing_static_part
    +PREHOOK: Input: default@list_bucketing_static_part@ds=2008-04-08/hr=11
    +#### A masked pattern was here ####
    +POSTHOOK: query: select * from list_bucketing_static_part where key = '51'
    +POSTHOOK: type: QUERY
    +POSTHOOK: Input: default@list_bucketing_static_part
    +POSTHOOK: Input: default@list_bucketing_static_part@ds=2008-04-08/hr=11
    +#### A masked pattern was here ####
    +51 val_51 2008-04-08 11
    +51 val_51 2008-04-08 11
    +51 val_51 2008-04-08 11
    +51 val_51 2008-04-08 11
    +PREHOOK: query: select * from srcpart where ds = '2008-04-08' and key = '51' and value = 'val_14'
    +PREHOOK: type: QUERY
    +PREHOOK: Input: default@srcpart
    +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11
    +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12
    +#### A masked pattern was here ####
    +POSTHOOK: query: select * from srcpart where ds = '2008-04-08' and key = '51' and value = 'val_14'
    +POSTHOOK: type: QUERY
    +POSTHOOK: Input: default@srcpart
    +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11
    +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12
    +#### A masked pattern was here ####
    +PREHOOK: query: select * from list_bucketing_static_part where key = '51' and value = 'val_14'
    +PREHOOK: type: QUERY
    +PREHOOK: Input: default@list_bucketing_static_part
    +PREHOOK: Input: default@list_bucketing_static_part@ds=2008-04-08/hr=11
    +#### A masked pattern was here ####
    +POSTHOOK: query: select * from list_bucketing_static_part where key = '51' and value = 'val_14'
    +POSTHOOK: type: QUERY
    +POSTHOOK: Input: default@list_bucketing_static_part
    +POSTHOOK: Input: default@list_bucketing_static_part@ds=2008-04-08/hr=11
    +#### A masked pattern was here ####
    +PREHOOK: query: -- queries with < <= > >= should work for skewed test although we don't benefit from pruning
    +select count(1) from srcpart where ds = '2008-04-08' and key < '51'
    +PREHOOK: type: QUERY
    +PREHOOK: Input: default@srcpart
    +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11
    +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12
    +#### A masked pattern was here ####
    +POSTHOOK: query: -- queries with < <= > >= should work for skewed test although we don't benefit from pruning
    +select count(1) from srcpart where ds = '2008-04-08' and key < '51'
    +POSTHOOK: type: QUERY
    +POSTHOOK: Input: default@srcpart
    +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11
    +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12
    +#### A masked pattern was here ####
    +910
    +PREHOOK: query: select count(1) from list_bucketing_static_part where key < '51'
    +PREHOOK: type: QUERY
    +PREHOOK: Input: default@list_bucketing_static_part
    +PREHOOK: Input: default@list_bucketing_static_part@ds=2008-04-08/hr=11
    +#### A masked pattern was here ####
    +POSTHOOK: query: select count(1) from list_bucketing_static_part where key < '51'
    +POSTHOOK: type: QUERY
    +POSTHOOK: Input: default@list_bucketing_static_part
    +POSTHOOK: Input: default@list_bucketing_static_part@ds=2008-04-08/hr=11
    +#### A masked pattern was here ####
    +910
    +PREHOOK: query: select count(1) from srcpart where ds = '2008-04-08' and key <= '51'
    +PREHOOK: type: QUERY
    +PREHOOK: Input: default@srcpart
    +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11
    +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12
    +#### A masked pattern was here ####
    +POSTHOOK: query: select count(1) from srcpart where ds = '2008-04-08' and key <= '51'
    +POSTHOOK: type: QUERY
    +POSTHOOK: Input: default@srcpart
    +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11
    +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12
    +#### A masked pattern was here ####
    +914
    +PREHOOK: query: select count(1) from list_bucketing_static_part where key <= '51'
    +PREHOOK: type: QUERY
    +PREHOOK: Input: default@list_bucketing_static_part
    +PREHOOK: Input: default@list_bucketing_static_part@ds=2008-04-08/hr=11
    +#### A masked pattern was here ####
    +POSTHOOK: query: select count(1) from list_bucketing_static_part where key <= '51'
    +POSTHOOK: type: QUERY
    +POSTHOOK: Input: default@list_bucketing_static_part
    +POSTHOOK: Input: default@list_bucketing_static_part@ds=2008-04-08/hr=11
    +#### A masked pattern was here ####
    +914
    +PREHOOK: query: select count(1) from srcpart where ds = '2008-04-08' and key > '51'
    +PREHOOK: type: QUERY
    +PREHOOK: Input: default@srcpart
    +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11
    +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12
    +#### A masked pattern was here ####
    +POSTHOOK: query: select count(1) from srcpart where ds = '2008-04-08' and key > '51'
    +POSTHOOK: type: QUERY
    +POSTHOOK: Input: default@srcpart
    +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11
    +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12
    +#### A masked pattern was here ####
    +86
    +PREHOOK: query: select count(1) from list_bucketing_static_part where key > '51'
    +PREHOOK: type: QUERY
    +PREHOOK: Input: default@list_bucketing_static_part
    +PREHOOK: Input: default@list_bucketing_static_part@ds=2008-04-08/hr=11
    +#### A masked pattern was here ####
    +POSTHOOK: query: select count(1) from list_bucketing_static_part where key > '51'
    +POSTHOOK: type: QUERY
    +POSTHOOK: Input: default@list_bucketing_static_part
    +POSTHOOK: Input: default@list_bucketing_static_part@ds=2008-04-08/hr=11
    +#### A masked pattern was here ####
    +86
    +PREHOOK: query: select count(1) from srcpart where ds = '2008-04-08' and key >= '51'
    +PREHOOK: type: QUERY
    +PREHOOK: Input: default@srcpart
    +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11
    +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12
    +#### A masked pattern was here ####
    +POSTHOOK: query: select count(1) from srcpart where ds = '2008-04-08' and key >= '51'
    +POSTHOOK: type: QUERY
    +POSTHOOK: Input: default@srcpart
    +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11
    +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12
    +#### A masked pattern was here ####
    +90
    +PREHOOK: query: select count(1) from list_bucketing_static_part where key >= '51'
    +PREHOOK: type: QUERY
    +PREHOOK: Input: default@list_bucketing_static_part
    +PREHOOK: Input: default@list_bucketing_static_part@ds=2008-04-08/hr=11
    +#### A masked pattern was here ####
    +POSTHOOK: query: select count(1) from list_bucketing_static_part where key >= '51'
    +POSTHOOK: type: QUERY
    +POSTHOOK: Input: default@list_bucketing_static_part
    +POSTHOOK: Input: default@list_bucketing_static_part@ds=2008-04-08/hr=11
    +#### A masked pattern was here ####
    +90
    +PREHOOK: query: -- clean up
    +drop table list_bucketing_static_part
    +PREHOOK: type: DROPTABLE
    +PREHOOK: Input: default@list_bucketing_static_part
    +PREHOOK: Output: default@list_bucketing_static_part
    +POSTHOOK: query: -- clean up
    +drop table list_bucketing_static_part
    +POSTHOOK: type: DROPTABLE
    +POSTHOOK: Input: default@list_bucketing_static_part
    +POSTHOOK: Output: default@list_bucketing_static_part

    Added: hive/trunk/ql/src/test/results/clientpositive/list_bucket_dml_2.q.java1.8.out
    URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/list_bucket_dml_2.q.java1.8.out?rev=1647912&view=auto
    ==============================================================================
    --- hive/trunk/ql/src/test/results/clientpositive/list_bucket_dml_2.q.java1.8.out (added)
    +++ hive/trunk/ql/src/test/results/clientpositive/list_bucket_dml_2.q.java1.8.out Thu Dec 25 18:06:30 2014
    @@ -0,0 +1,694 @@
    +PREHOOK: query: -- INCLUDE_HADOOP_MAJOR_VERSIONS(0.23)
    +-- SORT_QUERY_RESULTS
    +-- JAVA_VERSION_SPECIFIC_OUTPUT
    +
    +-- list bucketing DML: static partition. multiple skewed columns.
    +-- ds=2008-04-08/hr=11/HIVE_DEFAULT_LIST_BUCKETING_DIR_NAME:
    +-- 5263 000000_0
    +-- 5263 000001_0
    +-- ds=2008-04-08/hr=11/key=103/value=val_103:
    +-- 99 000000_0
    +-- 99 000001_0
    +-- ds=2008-04-08/hr=11/key=484/value=val_484:
    +-- 87 000000_0
    +-- 87 000001_0
    +
    +-- create a skewed table
    +create table list_bucketing_static_part (key String, value String)
    + partitioned by (ds String, hr String)
    + skewed by (key, value) on (('484','val_484'),('51','val_14'),('103','val_103'))
    + stored as DIRECTORIES
    + STORED AS RCFILE
    +PREHOOK: type: CREATETABLE
    +PREHOOK: Output: database:default
    +PREHOOK: Output: default@list_bucketing_static_part
    +POSTHOOK: query: -- INCLUDE_HADOOP_MAJOR_VERSIONS(0.23)
    +-- SORT_QUERY_RESULTS
    +-- JAVA_VERSION_SPECIFIC_OUTPUT
    +
    +-- list bucketing DML: static partition. multiple skewed columns.
    +-- ds=2008-04-08/hr=11/HIVE_DEFAULT_LIST_BUCKETING_DIR_NAME:
    +-- 5263 000000_0
    +-- 5263 000001_0
    +-- ds=2008-04-08/hr=11/key=103/value=val_103:
    +-- 99 000000_0
    +-- 99 000001_0
    +-- ds=2008-04-08/hr=11/key=484/value=val_484:
    +-- 87 000000_0
    +-- 87 000001_0
    +
    +-- create a skewed table
    +create table list_bucketing_static_part (key String, value String)
    + partitioned by (ds String, hr String)
    + skewed by (key, value) on (('484','val_484'),('51','val_14'),('103','val_103'))
    + stored as DIRECTORIES
    + STORED AS RCFILE
    +POSTHOOK: type: CREATETABLE
    +POSTHOOK: Output: database:default
    +POSTHOOK: Output: default@list_bucketing_static_part
    +PREHOOK: query: -- list bucketing DML without merge. use bucketize to generate a few small files.
    +explain extended
    +insert overwrite table list_bucketing_static_part partition (ds = '2008-04-08', hr = '11')
    +select key, value from srcpart where ds = '2008-04-08'
    +PREHOOK: type: QUERY
    +POSTHOOK: query: -- list bucketing DML without merge. use bucketize to generate a few small files.
    +explain extended
    +insert overwrite table list_bucketing_static_part partition (ds = '2008-04-08', hr = '11')
    +select key, value from srcpart where ds = '2008-04-08'
    +POSTHOOK: type: QUERY
    +ABSTRACT SYNTAX TREE:
    +
    +TOK_QUERY
    + TOK_FROM
    + TOK_TABREF
    + TOK_TABNAME
    + srcpart
    + TOK_INSERT
    + TOK_DESTINATION
    + TOK_TAB
    + TOK_TABNAME
    + list_bucketing_static_part
    + TOK_PARTSPEC
    + TOK_PARTVAL
    + ds
    + '2008-04-08'
    + TOK_PARTVAL
    + hr
    + '11'
    + TOK_SELECT
    + TOK_SELEXPR
    + TOK_TABLE_OR_COL
    + key
    + TOK_SELEXPR
    + TOK_TABLE_OR_COL
    + value
    + TOK_WHERE
    + =
    + TOK_TABLE_OR_COL
    + ds
    + '2008-04-08'
    +
    +
    +STAGE DEPENDENCIES:
    + Stage-1 is a root stage
    + Stage-0 depends on stages: Stage-1
    + Stage-2 depends on stages: Stage-0
    +
    +STAGE PLANS:
    + Stage: Stage-1
    + Map Reduce
    + Map Operator Tree:
    + TableScan
    + alias: srcpart
    + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE
    + GatherStats: false
    + Select Operator
    + expressions: key (type: string), value (type: string)
    + outputColumnNames: _col0, _col1
    + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE
    + File Output Operator
    + compressed: false
    + GlobalTableId: 1
    +#### A masked pattern was here ####
    + NumFilesPerFileSink: 1
    + Static Partition Specification: ds=2008-04-08/hr=11/
    + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE
    +#### A masked pattern was here ####
    + table:
    + input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat
    + output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat
    + properties:
    + bucket_count -1
    + columns key,value
    + columns.comments
    + columns.types string:string
    +#### A masked pattern was here ####
    + name default.list_bucketing_static_part
    + partition_columns ds/hr
    + partition_columns.types string:string
    + serialization.ddl struct list_bucketing_static_part { string key, string value}
    + serialization.format 1
    + serialization.lib org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
    +#### A masked pattern was here ####
    + serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
    + name: default.list_bucketing_static_part
    + TotalFiles: 1
    + GatherStats: true
    + MultiFileSpray: false
    + Path -> Alias:
    +#### A masked pattern was here ####
    + Path -> Partition:
    +#### A masked pattern was here ####
    + Partition
    + base file name: hr=11
    + input format: org.apache.hadoop.mapred.TextInputFormat
    + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
    + partition values:
    + ds 2008-04-08
    + hr 11
    + properties:
    + COLUMN_STATS_ACCURATE true
    + bucket_count -1
    + columns key,value
    + columns.comments 'default','default'
    + columns.types string:string
    +#### A masked pattern was here ####
    + name default.srcpart
    + numFiles 1
    + numRows 500
    + partition_columns ds/hr
    + partition_columns.types string:string
    + rawDataSize 5312
    + serialization.ddl struct srcpart { string key, string value}
    + serialization.format 1
    + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
    + totalSize 5812
    +#### A masked pattern was here ####
    + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
    +
    + input format: org.apache.hadoop.mapred.TextInputFormat
    + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
    + properties:
    + bucket_count -1
    + columns key,value
    + columns.comments 'default','default'
    + columns.types string:string
    +#### A masked pattern was here ####
    + name default.srcpart
    + partition_columns ds/hr
    + partition_columns.types string:string
    + serialization.ddl struct srcpart { string key, string value}
    + serialization.format 1
    + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
    +#### A masked pattern was here ####
    + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
    + name: default.srcpart
    + name: default.srcpart
    +#### A masked pattern was here ####
    + Partition
    + base file name: hr=12
    + input format: org.apache.hadoop.mapred.TextInputFormat
    + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
    + partition values:
    + ds 2008-04-08
    + hr 12
    + properties:
    + COLUMN_STATS_ACCURATE true
    + bucket_count -1
    + columns key,value
    + columns.comments 'default','default'
    + columns.types string:string
    +#### A masked pattern was here ####
    + name default.srcpart
    + numFiles 1
    + numRows 500
    + partition_columns ds/hr
    + partition_columns.types string:string
    + rawDataSize 5312
    + serialization.ddl struct srcpart { string key, string value}
    + serialization.format 1
    + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
    + totalSize 5812
    +#### A masked pattern was here ####
    + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
    +
    + input format: org.apache.hadoop.mapred.TextInputFormat
    + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
    + properties:
    + bucket_count -1
    + columns key,value
    + columns.comments 'default','default'
    + columns.types string:string
    +#### A masked pattern was here ####
    + name default.srcpart
    + partition_columns ds/hr
    + partition_columns.types string:string
    + serialization.ddl struct srcpart { string key, string value}
    + serialization.format 1
    + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
    +#### A masked pattern was here ####
    + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
    + name: default.srcpart
    + name: default.srcpart
    + Truncated Path -> Alias:
    + /srcpart/ds=2008-04-08/hr=11 [$hdt$_0:srcpart]
    + /srcpart/ds=2008-04-08/hr=12 [$hdt$_0:srcpart]
    +
    + Stage: Stage-0
    + Move Operator
    + tables:
    + partition:
    + ds 2008-04-08
    + hr 11
    + replace: true
    +#### A masked pattern was here ####
    + table:
    + input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat
    + output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat
    + properties:
    + bucket_count -1
    + columns key,value
    + columns.comments
    + columns.types string:string
    +#### A masked pattern was here ####
    + name default.list_bucketing_static_part
    + partition_columns ds/hr
    + partition_columns.types string:string
    + serialization.ddl struct list_bucketing_static_part { string key, string value}
    + serialization.format 1
    + serialization.lib org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
    +#### A masked pattern was here ####
    + serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
    + name: default.list_bucketing_static_part
    +
    + Stage: Stage-2
    + Stats-Aggr Operator
    +#### A masked pattern was here ####
    +
    +PREHOOK: query: insert overwrite table list_bucketing_static_part partition (ds = '2008-04-08', hr = '11')
    +select key, value from srcpart where ds = '2008-04-08'
    +PREHOOK: type: QUERY
    +PREHOOK: Input: default@srcpart
    +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11
    +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12
    +PREHOOK: Output: default@list_bucketing_static_part@ds=2008-04-08/hr=11
    +POSTHOOK: query: insert overwrite table list_bucketing_static_part partition (ds = '2008-04-08', hr = '11')
    +select key, value from srcpart where ds = '2008-04-08'
    +POSTHOOK: type: QUERY
    +POSTHOOK: Input: default@srcpart
    +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11
    +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12
    +POSTHOOK: Output: default@list_bucketing_static_part@ds=2008-04-08/hr=11
    +POSTHOOK: Lineage: list_bucketing_static_part PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(srcpart)srcpart.FieldSchema(name:key, type:string, comment:default), ]
    +POSTHOOK: Lineage: list_bucketing_static_part PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ]
    +PREHOOK: query: -- check DML result
    +show partitions list_bucketing_static_part
    +PREHOOK: type: SHOWPARTITIONS
    +PREHOOK: Input: default@list_bucketing_static_part
    +POSTHOOK: query: -- check DML result
    +show partitions list_bucketing_static_part
    +POSTHOOK: type: SHOWPARTITIONS
    +POSTHOOK: Input: default@list_bucketing_static_part
    +ds=2008-04-08/hr=11
    +PREHOOK: query: desc formatted list_bucketing_static_part partition (ds='2008-04-08', hr='11')
    +PREHOOK: type: DESCTABLE
    +PREHOOK: Input: default@list_bucketing_static_part
    +POSTHOOK: query: desc formatted list_bucketing_static_part partition (ds='2008-04-08', hr='11')
    +POSTHOOK: type: DESCTABLE
    +POSTHOOK: Input: default@list_bucketing_static_part
    +# col_name data_type comment
    +
    +key string
    +value string
    +
    +# Partition Information
    +# col_name data_type comment
    +
    +ds string
    +hr string
    +
    +# Detailed Partition Information
    +Partition Value: [2008-04-08, 11]
    +Database: default
    +Table: list_bucketing_static_part
    +#### A masked pattern was here ####
    +Protect Mode: None
    +#### A masked pattern was here ####
    +Partition Parameters:
    + COLUMN_STATS_ACCURATE true
    + numFiles 6
    + numRows 1000
    + rawDataSize 9624
    + totalSize 10898
    +#### A masked pattern was here ####
    +
    +# Storage Information
    +SerDe Library: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
    +InputFormat: org.apache.hadoop.hive.ql.io.RCFileInputFormat
    +OutputFormat: org.apache.hadoop.hive.ql.io.RCFileOutputFormat
    +Compressed: No
    +Num Buckets: -1
    +Bucket Columns: []
    +Sort Columns: []
    +Stored As SubDirectories: Yes
    +Skewed Columns: [key, value]
    +Skewed Values: [[484, val_484], [51, val_14], [103, val_103]]
    +#### A masked pattern was here ####
    +Skewed Value to Truncated Path: {[484, val_484]=/list_bucketing_static_part/ds=2008-04-08/hr=11/key=484/value=val_484, [103, val_103]=/list_bucketing_static_part/ds=2008-04-08/hr=11/key=103/value=val_103}
    +Storage Desc Params:
    + serialization.format 1
    +PREHOOK: query: select count(1) from srcpart where ds = '2008-04-08'
    +PREHOOK: type: QUERY
    +PREHOOK: Input: default@srcpart
    +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11
    +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12
    +#### A masked pattern was here ####
    +POSTHOOK: query: select count(1) from srcpart where ds = '2008-04-08'
    +POSTHOOK: type: QUERY
    +POSTHOOK: Input: default@srcpart
    +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11
    +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12
    +#### A masked pattern was here ####
    +1000
    +PREHOOK: query: select count(*) from list_bucketing_static_part
    +PREHOOK: type: QUERY
    +PREHOOK: Input: default@list_bucketing_static_part
    +PREHOOK: Input: default@list_bucketing_static_part@ds=2008-04-08/hr=11
    +#### A masked pattern was here ####
    +POSTHOOK: query: select count(*) from list_bucketing_static_part
    +POSTHOOK: type: QUERY
    +POSTHOOK: Input: default@list_bucketing_static_part
    +POSTHOOK: Input: default@list_bucketing_static_part@ds=2008-04-08/hr=11
    +#### A masked pattern was here ####
    +1000
    +PREHOOK: query: explain extended
    +select * from list_bucketing_static_part where ds = '2008-04-08' and hr = '11' and key = '484' and value = 'val_484'
    +PREHOOK: type: QUERY
    +POSTHOOK: query: explain extended
    +select * from list_bucketing_static_part where ds = '2008-04-08' and hr = '11' and key = '484' and value = 'val_484'
    +POSTHOOK: type: QUERY
    +ABSTRACT SYNTAX TREE:
    +
    +TOK_QUERY
    + TOK_FROM
    + TOK_TABREF
    + TOK_TABNAME
    + list_bucketing_static_part
    + TOK_INSERT
    + TOK_DESTINATION
    + TOK_DIR
    + TOK_TMP_FILE
    + TOK_SELECT
    + TOK_SELEXPR
    + TOK_ALLCOLREF
    + TOK_WHERE
    + and
    + and
    + and
    + =
    + TOK_TABLE_OR_COL
    + ds
    + '2008-04-08'
    + =
    + TOK_TABLE_OR_COL
    + hr
    + '11'
    + =
    + TOK_TABLE_OR_COL
    + key
    + '484'
    + =
    + TOK_TABLE_OR_COL
    + value
    + 'val_484'
    +
    +
    +STAGE DEPENDENCIES:
    + Stage-1 is a root stage
    + Stage-0 depends on stages: Stage-1
    +
    +STAGE PLANS:
    + Stage: Stage-1
    + Map Reduce
    + Map Operator Tree:
    + TableScan
    + alias: list_bucketing_static_part
    + Statistics: Num rows: 1000 Data size: 9624 Basic stats: COMPLETE Column stats: NONE
    + GatherStats: false
    + Filter Operator
    + isSamplingPred: false
    + predicate: ((key = '484') and (value = 'val_484')) (type: boolean)
    + Statistics: Num rows: 250 Data size: 2406 Basic stats: COMPLETE Column stats: NONE
    + Select Operator
    + expressions: '484' (type: string), 'val_484' (type: string), '2008-04-08' (type: string), '11' (type: string)
    + outputColumnNames: _col0, _col1, _col2, _col3
    + Statistics: Num rows: 250 Data size: 2406 Basic stats: COMPLETE Column stats: NONE
    + File Output Operator
    + compressed: false
    + GlobalTableId: 0
    +#### A masked pattern was here ####
    + NumFilesPerFileSink: 1
    + Statistics: Num rows: 250 Data size: 2406 Basic stats: COMPLETE Column stats: NONE
    +#### A masked pattern was here ####
    + table:
    + input format: org.apache.hadoop.mapred.TextInputFormat
    + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
    + properties:
    + columns _col0,_col1,_col2,_col3
    + columns.types string:string:string:string
    + escape.delim \
    + hive.serialization.extend.nesting.levels true
    + serialization.format 1
    + serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
    + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
    + TotalFiles: 1
    + GatherStats: false
    + MultiFileSpray: false
    + Path -> Alias:
    +#### A masked pattern was here ####
    + Path -> Partition:
    +#### A masked pattern was here ####
    + Partition
    + base file name: value=val_484
    + input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat
    + output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat
    + partition values:
    + ds 2008-04-08
    + hr 11
    + properties:
    + COLUMN_STATS_ACCURATE true
    + bucket_count -1
    + columns key,value
    + columns.comments
    + columns.types string:string
    +#### A masked pattern was here ####
    + name default.list_bucketing_static_part
    + numFiles 6
    + numRows 1000
    + partition_columns ds/hr
    + partition_columns.types string:string
    + rawDataSize 9624
    + serialization.ddl struct list_bucketing_static_part { string key, string value}
    + serialization.format 1
    + serialization.lib org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
    + totalSize 10898
    +#### A masked pattern was here ####
    + serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
    +
    + input format: org.apache.hadoop.hive.ql.io.RCFileInputFormat
    + output format: org.apache.hadoop.hive.ql.io.RCFileOutputFormat
    + properties:
    + bucket_count -1
    + columns key,value
    + columns.comments
    + columns.types string:string
    +#### A masked pattern was here ####
    + name default.list_bucketing_static_part
    + partition_columns ds/hr
    + partition_columns.types string:string
    + serialization.ddl struct list_bucketing_static_part { string key, string value}
    + serialization.format 1
    + serialization.lib org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
    +#### A masked pattern was here ####
    + serde: org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe
    + name: default.list_bucketing_static_part
    + name: default.list_bucketing_static_part
    + Truncated Path -> Alias:
    + /list_bucketing_static_part/ds=2008-04-08/hr=11/key=484/value=val_484 [list_bucketing_static_part]
    +
    + Stage: Stage-0
    + Fetch Operator
    + limit: -1
    + Processor Tree:
    + ListSink
    +
    +PREHOOK: query: select * from list_bucketing_static_part where ds = '2008-04-08' and hr = '11' and key = '484' and value = 'val_484'
    +PREHOOK: type: QUERY
    +PREHOOK: Input: default@list_bucketing_static_part
    +PREHOOK: Input: default@list_bucketing_static_part@ds=2008-04-08/hr=11
    +#### A masked pattern was here ####
    +POSTHOOK: query: select * from list_bucketing_static_part where ds = '2008-04-08' and hr = '11' and key = '484' and value = 'val_484'
    +POSTHOOK: type: QUERY
    +POSTHOOK: Input: default@list_bucketing_static_part
    +POSTHOOK: Input: default@list_bucketing_static_part@ds=2008-04-08/hr=11
    +#### A masked pattern was here ####
    +484 val_484 2008-04-08 11
    +484 val_484 2008-04-08 11
    +PREHOOK: query: select * from srcpart where ds = '2008-04-08' and key = '484' and value = 'val_484'
    +PREHOOK: type: QUERY
    +PREHOOK: Input: default@srcpart
    +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11
    +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12
    +#### A masked pattern was here ####
    +POSTHOOK: query: select * from srcpart where ds = '2008-04-08' and key = '484' and value = 'val_484'
    +POSTHOOK: type: QUERY
    +POSTHOOK: Input: default@srcpart
    +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11
    +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12
    +#### A masked pattern was here ####
    +484 val_484 2008-04-08 11
    +484 val_484 2008-04-08 12
    +PREHOOK: query: -- 51 and val_51 in the table so skewed data for 51 and val_14 should be none
    +-- but query should succeed for 51 or 51 and val_14
    +select * from srcpart where ds = '2008-04-08' and key = '51'
    +PREHOOK: type: QUERY
    +PREHOOK: Input: default@srcpart
    +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11
    +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12
    +#### A masked pattern was here ####
    +POSTHOOK: query: -- 51 and val_51 in the table so skewed data for 51 and val_14 should be none
    +-- but query should succeed for 51 or 51 and val_14
    +select * from srcpart where ds = '2008-04-08' and key = '51'
    +POSTHOOK: type: QUERY
    +POSTHOOK: Input: default@srcpart
    +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11
    +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12
    +#### A masked pattern was here ####
    +51 val_51 2008-04-08 11
    +51 val_51 2008-04-08 11
    +51 val_51 2008-04-08 12
    +51 val_51 2008-04-08 12
    +PREHOOK: query: select * from list_bucketing_static_part where key = '51'
    +PREHOOK: type: QUERY
    +PREHOOK: Input: default@list_bucketing_static_part
    +PREHOOK: Input: default@list_bucketing_static_part@ds=2008-04-08/hr=11
    +#### A masked pattern was here ####
    +POSTHOOK: query: select * from list_bucketing_static_part where key = '51'
    +POSTHOOK: type: QUERY
    +POSTHOOK: Input: default@list_bucketing_static_part
    +POSTHOOK: Input: default@list_bucketing_static_part@ds=2008-04-08/hr=11
    +#### A masked pattern was here ####
    +51 val_51 2008-04-08 11
    +51 val_51 2008-04-08 11
    +51 val_51 2008-04-08 11
    +51 val_51 2008-04-08 11
    +PREHOOK: query: select * from srcpart where ds = '2008-04-08' and key = '51' and value = 'val_14'
    +PREHOOK: type: QUERY
    +PREHOOK: Input: default@srcpart
    +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11
    +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12
    +#### A masked pattern was here ####
    +POSTHOOK: query: select * from srcpart where ds = '2008-04-08' and key = '51' and value = 'val_14'
    +POSTHOOK: type: QUERY
    +POSTHOOK: Input: default@srcpart
    +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11
    +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12
    +#### A masked pattern was here ####
    +PREHOOK: query: select * from list_bucketing_static_part where key = '51' and value = 'val_14'
    +PREHOOK: type: QUERY
    +PREHOOK: Input: default@list_bucketing_static_part
    +PREHOOK: Input: default@list_bucketing_static_part@ds=2008-04-08/hr=11
    +#### A masked pattern was here ####
    +POSTHOOK: query: select * from list_bucketing_static_part where key = '51' and value = 'val_14'
    +POSTHOOK: type: QUERY
    +POSTHOOK: Input: default@list_bucketing_static_part
    +POSTHOOK: Input: default@list_bucketing_static_part@ds=2008-04-08/hr=11
    +#### A masked pattern was here ####
    +PREHOOK: query: -- queries with < <= > >= should work for skewed test although we don't benefit from pruning
    +select count(1) from srcpart where ds = '2008-04-08' and key < '51'
    +PREHOOK: type: QUERY
    +PREHOOK: Input: default@srcpart
    +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11
    +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12
    +#### A masked pattern was here ####
    +POSTHOOK: query: -- queries with < <= > >= should work for skewed test although we don't benefit from pruning
    +select count(1) from srcpart where ds = '2008-04-08' and key < '51'
    +POSTHOOK: type: QUERY
    +POSTHOOK: Input: default@srcpart
    +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11
    +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12
    +#### A masked pattern was here ####
    +910
    +PREHOOK: query: select count(1) from list_bucketing_static_part where key < '51'
    +PREHOOK: type: QUERY
    +PREHOOK: Input: default@list_bucketing_static_part
    +PREHOOK: Input: default@list_bucketing_static_part@ds=2008-04-08/hr=11
    +#### A masked pattern was here ####
    +POSTHOOK: query: select count(1) from list_bucketing_static_part where key < '51'
    +POSTHOOK: type: QUERY
    +POSTHOOK: Input: default@list_bucketing_static_part
    +POSTHOOK: Input: default@list_bucketing_static_part@ds=2008-04-08/hr=11
    +#### A masked pattern was here ####
    +910
    +PREHOOK: query: select count(1) from srcpart where ds = '2008-04-08' and key <= '51'
    +PREHOOK: type: QUERY
    +PREHOOK: Input: default@srcpart
    +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11
    +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12
    +#### A masked pattern was here ####
    +POSTHOOK: query: select count(1) from srcpart where ds = '2008-04-08' and key <= '51'
    +POSTHOOK: type: QUERY
    +POSTHOOK: Input: default@srcpart
    +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11
    +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12
    +#### A masked pattern was here ####
    +914
    +PREHOOK: query: select count(1) from list_bucketing_static_part where key <= '51'
    +PREHOOK: type: QUERY
    +PREHOOK: Input: default@list_bucketing_static_part
    +PREHOOK: Input: default@list_bucketing_static_part@ds=2008-04-08/hr=11
    +#### A masked pattern was here ####
    +POSTHOOK: query: select count(1) from list_bucketing_static_part where key <= '51'
    +POSTHOOK: type: QUERY
    +POSTHOOK: Input: default@list_bucketing_static_part
    +POSTHOOK: Input: default@list_bucketing_static_part@ds=2008-04-08/hr=11
    +#### A masked pattern was here ####
    +914
    +PREHOOK: query: select count(1) from srcpart where ds = '2008-04-08' and key > '51'
    +PREHOOK: type: QUERY
    +PREHOOK: Input: default@srcpart
    +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11
    +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12
    +#### A masked pattern was here ####
    +POSTHOOK: query: select count(1) from srcpart where ds = '2008-04-08' and key > '51'
    +POSTHOOK: type: QUERY
    +POSTHOOK: Input: default@srcpart
    +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11
    +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12
    +#### A masked pattern was here ####
    +86
    +PREHOOK: query: select count(1) from list_bucketing_static_part where key > '51'
    +PREHOOK: type: QUERY
    +PREHOOK: Input: default@list_bucketing_static_part
    +PREHOOK: Input: default@list_bucketing_static_part@ds=2008-04-08/hr=11
    +#### A masked pattern was here ####
    +POSTHOOK: query: select count(1) from list_bucketing_static_part where key > '51'
    +POSTHOOK: type: QUERY
    +POSTHOOK: Input: default@list_bucketing_static_part
    +POSTHOOK: Input: default@list_bucketing_static_part@ds=2008-04-08/hr=11
    +#### A masked pattern was here ####
    +86
    +PREHOOK: query: select count(1) from srcpart where ds = '2008-04-08' and key >= '51'
    +PREHOOK: type: QUERY
    +PREHOOK: Input: default@srcpart
    +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11
    +PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12
    +#### A masked pattern was here ####
    +POSTHOOK: query: select count(1) from srcpart where ds = '2008-04-08' and key >= '51'
    +POSTHOOK: type: QUERY
    +POSTHOOK: Input: default@srcpart
    +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11
    +POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12
    +#### A masked pattern was here ####
    +90
    +PREHOOK: query: select count(1) from list_bucketing_static_part where key >= '51'
    +PREHOOK: type: QUERY
    +PREHOOK: Input: default@list_bucketing_static_part
    +PREHOOK: Input: default@list_bucketing_static_part@ds=2008-04-08/hr=11
    +#### A masked pattern was here ####
    +POSTHOOK: query: select count(1) from list_bucketing_static_part where key >= '51'
    +POSTHOOK: type: QUERY
    +POSTHOOK: Input: default@list_bucketing_static_part
    +POSTHOOK: Input: default@list_bucketing_static_part@ds=2008-04-08/hr=11
    +#### A masked pattern was here ####
    +90
    +PREHOOK: query: -- clean up
    +drop table list_bucketing_static_part
    +PREHOOK: type: DROPTABLE
    +PREHOOK: Input: default@list_bucketing_static_part
    +PREHOOK: Output: default@list_bucketing_static_part
    +POSTHOOK: query: -- clean up
    +drop table list_bucketing_static_part
    +POSTHOOK: type: DROPTABLE
    +POSTHOOK: Input: default@list_bucketing_static_part
    +POSTHOOK: Output: default@list_bucketing_static_part

Related Discussions

Discussion Navigation
viewthread | post
Discussion Overview
groupcommits @
categorieshive, hadoop
postedDec 25, '14 at 6:06p
activeDec 25, '14 at 6:06p
posts7
users1
websitehive.apache.org

1 user in discussion

Brock: 7 posts

People

Translate

site design / logo © 2021 Grokbase