FAQ
Repository: hive
Updated Branches:
   refs/heads/llap b97a07688 -> 88248fb6b


HIVE-11451: SemanticAnalyzer throws IndexOutOfBounds Exception (Aihua Xu, reviewed by Chao Sun)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/db46e6e8
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/db46e6e8
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/db46e6e8

Branch: refs/heads/llap
Commit: db46e6e8ccc4ee86fbd074dbc3d2e4fa7d88ce25
Parents: bd90fc3
Author: Aihua Xu <aihuaxu@gmail.com>
Authored: Thu Aug 13 10:55:30 2015 -0700
Committer: Chao Sun <sunchao@apache.org>
Committed: Thu Aug 13 11:24:31 2015 -0700

----------------------------------------------------------------------
  .../hadoop/hive/ql/parse/SemanticAnalyzer.java | 15 +++++++++------
  .../clientnegative/mismatch_columns_insertion.q | 4 ++++
  .../clientnegative/mismatch_columns_insertion.q.out | 9 +++++++++
  3 files changed, 22 insertions(+), 6 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/db46e6e8/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
index 5ea6f3f..b809a23 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
@@ -6724,6 +6724,14 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
        outColumnCnt += dpCtx.getNumDPCols();
      }

+ // The numbers of input columns and output columns should match for regular query
+ if (!updating() && !deleting() && inColumnCnt != outColumnCnt) {
+ String reason = "Table " + dest + " has " + outColumnCnt
+ + " columns, but query has " + inColumnCnt + " columns.";
+ throw new SemanticException(ErrorMsg.TARGET_TABLE_COLUMN_MISMATCH.getMsg(
+ qb.getParseInfo().getDestForClause(dest), reason));
+ }
+
      // Check column types
      boolean converted = false;
      int columnNumber = tableFields.size();
@@ -6830,12 +6838,7 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
          dpCtx.mapInputToDP(rowFields.subList(tableFields.size() + 1, rowFields.size()));
        }
      } else {
- if (inColumnCnt != outColumnCnt) {
- String reason = "Table " + dest + " has " + outColumnCnt
- + " columns, but query has " + inColumnCnt + " columns.";
- throw new SemanticException(ErrorMsg.TARGET_TABLE_COLUMN_MISMATCH.getMsg(
- qb.getParseInfo().getDestForClause(dest), reason));
- } else if (dynPart && dpCtx != null) {
+ if (dynPart && dpCtx != null) {
          // create the mapping from input ExprNode to dest table DP column
          dpCtx.mapInputToDP(rowFields.subList(tableFields.size(), rowFields.size()));
        }

http://git-wip-us.apache.org/repos/asf/hive/blob/db46e6e8/ql/src/test/queries/clientnegative/mismatch_columns_insertion.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientnegative/mismatch_columns_insertion.q b/ql/src/test/queries/clientnegative/mismatch_columns_insertion.q
new file mode 100644
index 0000000..dad1ec1
--- /dev/null
+++ b/ql/src/test/queries/clientnegative/mismatch_columns_insertion.q
@@ -0,0 +1,4 @@
+
+create table mismatch_columns(key string, value string);
+
+insert overwrite table mismatch_columns select key from srcpart where ds is not null;

http://git-wip-us.apache.org/repos/asf/hive/blob/db46e6e8/ql/src/test/results/clientnegative/mismatch_columns_insertion.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientnegative/mismatch_columns_insertion.q.out b/ql/src/test/results/clientnegative/mismatch_columns_insertion.q.out
new file mode 100644
index 0000000..831af12
--- /dev/null
+++ b/ql/src/test/results/clientnegative/mismatch_columns_insertion.q.out
@@ -0,0 +1,9 @@
+PREHOOK: query: create table mismatch_columns(key string, value string)
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@mismatch_columns
+POSTHOOK: query: create table mismatch_columns(key string, value string)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@mismatch_columns
+FAILED: SemanticException [Error 10044]: Line 3:23 Cannot insert into target table because column number/types are different 'mismatch_columns': Table insclause-0 has 2 columns, but query has 1 columns.

Search Discussions

  • Sershe at Aug 25, 2015 at 9:28 pm
    HIVE-11493: Predicate with integer column equals double evaluates to false (Pengcheng Xiong, reviewed by Hari Sankar Sivarama Subramaniyan)


    Project: http://git-wip-us.apache.org/repos/asf/hive/repo
    Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/b8f1ae11
    Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/b8f1ae11
    Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/b8f1ae11

    Branch: refs/heads/llap
    Commit: b8f1ae110616a8fe162f79140c785f76be76fc67
    Parents: 6e76291
    Author: Pengcheng Xiong <pxiong@apache.org>
    Authored: Thu Aug 13 21:01:54 2015 -0700
    Committer: Pengcheng Xiong <pxiong@apache.org>
    Committed: Thu Aug 13 21:02:40 2015 -0700

    ----------------------------------------------------------------------
      .../hive/ql/parse/TypeCheckProcFactory.java | 2 +-
      .../clientpositive/cast_tinyint_to_double.q | 7 ++++
      .../clientpositive/cast_tinyint_to_double.q.out | 38 ++++++++++++++++++++
      .../clientpositive/infer_const_type.q.out | 7 ++--
      .../clientpositive/spark/vectorization_0.q.out | 2 +-
      .../spark/vectorization_short_regress.q.out | 20 +++++------
      .../clientpositive/tez/vectorization_0.q.out | 2 +-
      .../tez/vectorization_short_regress.q.out | 20 +++++------
      .../clientpositive/vectorization_0.q.out | 2 +-
      .../vectorization_short_regress.q.out | 20 +++++------
      10 files changed, 84 insertions(+), 36 deletions(-)
    ----------------------------------------------------------------------


    http://git-wip-us.apache.org/repos/asf/hive/blob/b8f1ae11/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactory.java
    ----------------------------------------------------------------------
    diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactory.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactory.java
    index cd68f4e..ab5d006 100644
    --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactory.java
    +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactory.java
    @@ -1034,7 +1034,7 @@ public class TypeCheckProcFactory {
                    // we'll try again to convert it to double
                    // however, if we already tried this, or the column is NUMBER type and
                    // the operator is EQUAL, return false due to the type mismatch
    - if (triedDouble ||
    + if (triedDouble &&
                        (genericUDF instanceof GenericUDFOPEqual
                        && !columnType.equals(serdeConstants.STRING_TYPE_NAME))) {
                      return new ExprNodeConstantDesc(false);

    http://git-wip-us.apache.org/repos/asf/hive/blob/b8f1ae11/ql/src/test/queries/clientpositive/cast_tinyint_to_double.q
    ----------------------------------------------------------------------
    diff --git a/ql/src/test/queries/clientpositive/cast_tinyint_to_double.q b/ql/src/test/queries/clientpositive/cast_tinyint_to_double.q
    new file mode 100644
    index 0000000..59c5e89
    --- /dev/null
    +++ b/ql/src/test/queries/clientpositive/cast_tinyint_to_double.q
    @@ -0,0 +1,7 @@
    +drop table t;
    +CREATE TABLE t(c tinyint);
    +insert overwrite table t select 10 from src limit 1;
    +
    +select * from t where c = 10.0;
    +
    +select * from t where c = -10.0;
    \ No newline at end of file

    http://git-wip-us.apache.org/repos/asf/hive/blob/b8f1ae11/ql/src/test/results/clientpositive/cast_tinyint_to_double.q.out
    ----------------------------------------------------------------------
    diff --git a/ql/src/test/results/clientpositive/cast_tinyint_to_double.q.out b/ql/src/test/results/clientpositive/cast_tinyint_to_double.q.out
    new file mode 100644
    index 0000000..c29df65
    --- /dev/null
    +++ b/ql/src/test/results/clientpositive/cast_tinyint_to_double.q.out
    @@ -0,0 +1,38 @@
    +PREHOOK: query: drop table t
    +PREHOOK: type: DROPTABLE
    +POSTHOOK: query: drop table t
    +POSTHOOK: type: DROPTABLE
    +PREHOOK: query: CREATE TABLE t(c tinyint)
    +PREHOOK: type: CREATETABLE
    +PREHOOK: Output: database:default
    +PREHOOK: Output: default@t
    +POSTHOOK: query: CREATE TABLE t(c tinyint)
    +POSTHOOK: type: CREATETABLE
    +POSTHOOK: Output: database:default
    +POSTHOOK: Output: default@t
    +PREHOOK: query: insert overwrite table t select 10 from src limit 1
    +PREHOOK: type: QUERY
    +PREHOOK: Input: default@src
    +PREHOOK: Output: default@t
    +POSTHOOK: query: insert overwrite table t select 10 from src limit 1
    +POSTHOOK: type: QUERY
    +POSTHOOK: Input: default@src
    +POSTHOOK: Output: default@t
    +POSTHOOK: Lineage: t.c EXPRESSION []
    +PREHOOK: query: select * from t where c = 10.0
    +PREHOOK: type: QUERY
    +PREHOOK: Input: default@t
    +#### A masked pattern was here ####
    +POSTHOOK: query: select * from t where c = 10.0
    +POSTHOOK: type: QUERY
    +POSTHOOK: Input: default@t
    +#### A masked pattern was here ####
    +10
    +PREHOOK: query: select * from t where c = -10.0
    +PREHOOK: type: QUERY
    +PREHOOK: Input: default@t
    +#### A masked pattern was here ####
    +POSTHOOK: query: select * from t where c = -10.0
    +POSTHOOK: type: QUERY
    +POSTHOOK: Input: default@t
    +#### A masked pattern was here ####

    http://git-wip-us.apache.org/repos/asf/hive/blob/b8f1ae11/ql/src/test/results/clientpositive/infer_const_type.q.out
    ----------------------------------------------------------------------
    diff --git a/ql/src/test/results/clientpositive/infer_const_type.q.out b/ql/src/test/results/clientpositive/infer_const_type.q.out
    index 6368e4a..05c6a45 100644
    --- a/ql/src/test/results/clientpositive/infer_const_type.q.out
    +++ b/ql/src/test/results/clientpositive/infer_const_type.q.out
    @@ -102,6 +102,7 @@ POSTHOOK: type: QUERY
      POSTHOOK: Input: default@infertypes
      #### A masked pattern was here ####
      127 32767 12345 -12345 906.0 -307.0 1234
    +WARNING: Comparing a bigint and a double may result in a loss of precision.
      PREHOOK: query: -- all should return false as all numbers exceeed the largest number
      -- which could be represented by the corresponding type
      -- and string_col = long_const should return false
    @@ -136,7 +137,7 @@ STAGE PLANS:
                  alias: infertypes
                  Statistics: Num rows: 1 Data size: 117 Basic stats: COMPLETE Column stats: NONE
                  Filter Operator
    - predicate: false (type: boolean)
    + predicate: ((UDFToDouble(ti) = 128.0) or (UDFToDouble(si) = 32768.0) or (UDFToDouble(i) = 2.147483648E9) or (UDFToDouble(bi) = 9.223372036854776E18)) (type: boolean)
                    Statistics: Num rows: 1 Data size: 117 Basic stats: COMPLETE Column stats: NONE
                    Select Operator
                      expressions: ti (type: tinyint), si (type: smallint), i (type: int), bi (type: bigint), fl (type: float), db (type: double), str (type: string)
    @@ -156,6 +157,7 @@ STAGE PLANS:
            Processor Tree:
              ListSink

    +WARNING: Comparing a bigint and a double may result in a loss of precision.
      PREHOOK: query: SELECT * FROM infertypes WHERE
        ti = '128' OR
        si = 32768 OR
    @@ -200,7 +202,7 @@ STAGE PLANS:
                  alias: infertypes
                  Statistics: Num rows: 1 Data size: 117 Basic stats: COMPLETE Column stats: NONE
                  Filter Operator
    - predicate: false (type: boolean)
    + predicate: ((UDFToDouble(ti) = 127.0) or (UDFToDouble(si) = 327.0) or (UDFToDouble(i) = -100.0)) (type: boolean)
                    Statistics: Num rows: 1 Data size: 117 Basic stats: COMPLETE Column stats: NONE
                    Select Operator
                      expressions: ti (type: tinyint), si (type: smallint), i (type: int), bi (type: bigint), fl (type: float), db (type: double), str (type: string)
    @@ -234,6 +236,7 @@ POSTHOOK: query: SELECT * FROM infertypes WHERE
      POSTHOOK: type: QUERY
      POSTHOOK: Input: default@infertypes
      #### A masked pattern was here ####
    +127 32767 12345 -12345 906.0 -307.0 1234
      PREHOOK: query: EXPLAIN SELECT * FROM infertypes WHERE
        ti < '127.0' AND
        i > '100.0' AND

    http://git-wip-us.apache.org/repos/asf/hive/blob/b8f1ae11/ql/src/test/results/clientpositive/spark/vectorization_0.q.out
    ----------------------------------------------------------------------
    diff --git a/ql/src/test/results/clientpositive/spark/vectorization_0.q.out b/ql/src/test/results/clientpositive/spark/vectorization_0.q.out
    index 90bcc1b..3ad059c 100644
    --- a/ql/src/test/results/clientpositive/spark/vectorization_0.q.out
    +++ b/ql/src/test/results/clientpositive/spark/vectorization_0.q.out
    @@ -994,7 +994,7 @@ STAGE PLANS:
                        alias: alltypesorc
                        Statistics: Num rows: 12288 Data size: 377237 Basic stats: COMPLETE Column stats: NONE
                        Filter Operator
    - predicate: ((cstring2 like '%b%') or (79.553 <> UDFToDouble(cint)) or (UDFToDouble(cbigint) < cdouble)) (type: boolean)
    + predicate: ((cstring2 like '%b%') or (79.553 <> UDFToDouble(cint)) or (UDFToDouble(cbigint) < cdouble) or ((UDFToShort(ctinyint) >= csmallint) and (cboolean2 = 1) and (3569.0 = UDFToDouble(ctinyint)))) (type: boolean)
                          Statistics: Num rows: 12288 Data size: 377237 Basic stats: COMPLETE Column stats: NONE
                          Select Operator
                            expressions: cbigint (type: bigint), cfloat (type: float), ctinyint (type: tinyint)

    http://git-wip-us.apache.org/repos/asf/hive/blob/b8f1ae11/ql/src/test/results/clientpositive/spark/vectorization_short_regress.q.out
    ----------------------------------------------------------------------
    diff --git a/ql/src/test/results/clientpositive/spark/vectorization_short_regress.q.out b/ql/src/test/results/clientpositive/spark/vectorization_short_regress.q.out
    index 6bd1bb2..3d17aba 100644
    --- a/ql/src/test/results/clientpositive/spark/vectorization_short_regress.q.out
    +++ b/ql/src/test/results/clientpositive/spark/vectorization_short_regress.q.out
    @@ -2085,23 +2085,23 @@ STAGE PLANS:
                        alias: alltypesorc
                        Statistics: Num rows: 12288 Data size: 377237 Basic stats: COMPLETE Column stats: NONE
                        Filter Operator
    - predicate: ((cdouble > 2563.58) and (((cbigint >= UDFToLong(cint)) and (UDFToInteger(csmallint) < cint) and (UDFToDouble(cfloat) < -5638.15)) or ((cdouble <= UDFToDouble(cbigint)) and (-5638.15 > UDFToDouble(cbigint))))) (type: boolean)
    - Statistics: Num rows: 606 Data size: 18603 Basic stats: COMPLETE Column stats: NONE
    + predicate: ((cdouble > 2563.58) and (((cbigint >= UDFToLong(cint)) and (UDFToInteger(csmallint) < cint) and (UDFToDouble(cfloat) < -5638.15)) or (2563.58 = UDFToDouble(ctinyint)) or ((cdouble <= UDFToDouble(cbigint)) and (-5638.15 > UDFToDouble(cbigint))))) (type: boolean)
    + Statistics: Num rows: 2654 Data size: 81476 Basic stats: COMPLETE Column stats: NONE
                          Select Operator
                            expressions: cdouble (type: double), cfloat (type: float)
                            outputColumnNames: _col0, _col1
    - Statistics: Num rows: 606 Data size: 18603 Basic stats: COMPLETE Column stats: NONE
    + Statistics: Num rows: 2654 Data size: 81476 Basic stats: COMPLETE Column stats: NONE
                            Group By Operator
                              aggregations: var_samp(_col0), count(_col1), sum(_col1), var_pop(_col0), stddev_pop(_col0), sum(_col0)
                              keys: _col0 (type: double)
                              mode: hash
                              outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6
    - Statistics: Num rows: 606 Data size: 18603 Basic stats: COMPLETE Column stats: NONE
    + Statistics: Num rows: 2654 Data size: 81476 Basic stats: COMPLETE Column stats: NONE
                              Reduce Output Operator
                                key expressions: _col0 (type: double)
                                sort order: +
                                Map-reduce partition columns: _col0 (type: double)
    - Statistics: Num rows: 606 Data size: 18603 Basic stats: COMPLETE Column stats: NONE
    + Statistics: Num rows: 2654 Data size: 81476 Basic stats: COMPLETE Column stats: NONE
                                value expressions: _col1 (type: struct<count:bigint,sum:double,variance:double>), _col2 (type: bigint), _col3 (type: double), _col4 (type: struct<count:bigint,sum:double,variance:double>), _col5 (type: struct<count:bigint,sum:double,variance:double>), _col6 (type: double)
                  Execution mode: vectorized
              Reducer 2
    @@ -2111,25 +2111,25 @@ STAGE PLANS:
                      keys: KEY._col0 (type: double)
                      mode: mergepartial
                      outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6
    - Statistics: Num rows: 303 Data size: 9301 Basic stats: COMPLETE Column stats: NONE
    + Statistics: Num rows: 1327 Data size: 40738 Basic stats: COMPLETE Column stats: NONE
                      Select Operator
                        expressions: _col0 (type: double), _col1 (type: double), _col5 (type: double), (_col0 + _col1) (type: double), (_col0 * 762.0) (type: double), _col6 (type: double), (-863.257 % (_col0 * 762.0)) (type: double), (2563.58 * _col1) (type: double), (- _col1) (type: double), _col2 (type: bigint), ((2563.58 * _col1) + -5638.15) (type: double), ((- _col1) * ((2563.58 * _col1) + -5638.15)) (type: double), _col3 (type: double), _col4 (type: double), (_col0 - (- _col1)) (type: double)
                        outputColumnNames: _col0, _col1, _col10, _col11, _col12, _col13, _col14, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9
    - Statistics: Num rows: 303 Data size: 9301 Basic stats: COMPLETE Column stats: NONE
    + Statistics: Num rows: 1327 Data size: 40738 Basic stats: COMPLETE Column stats: NONE
                        Reduce Output Operator
                          key expressions: _col0 (type: double)
                          sort order: +
    - Statistics: Num rows: 303 Data size: 9301 Basic stats: COMPLETE Column stats: NONE
    + Statistics: Num rows: 1327 Data size: 40738 Basic stats: COMPLETE Column stats: NONE
                          value expressions: _col1 (type: double), _col2 (type: double), _col3 (type: double), _col4 (type: bigint), _col5 (type: double), _col6 (type: double), _col7 (type: double), _col8 (type: double), _col9 (type: double), _col10 (type: double), _col11 (type: double), _col12 (type: double), _col13 (type: double), _col14 (type: double)
              Reducer 3
                  Reduce Operator Tree:
                    Select Operator
                      expressions: KEY.reducesinkkey0 (type: double), VALUE._col0 (type: double), VALUE._col1 (type: double), VALUE._col2 (type: double), VALUE._col3 (type: bigint), VALUE._col4 (type: double), VALUE._col5 (type: double), VALUE._col6 (type: double), VALUE._col7 (type: double), VALUE._col8 (type: double), VALUE._col9 (type: double), VALUE._col10 (type: double), VALUE._col11 (type: double), VALUE._col12 (type: double), VALUE._col13 (type: double), VALUE._col12 (type: double)
                      outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15
    - Statistics: Num rows: 303 Data size: 9301 Basic stats: COMPLETE Column stats: NONE
    + Statistics: Num rows: 1327 Data size: 40738 Basic stats: COMPLETE Column stats: NONE
                      File Output Operator
                        compressed: false
    - Statistics: Num rows: 303 Data size: 9301 Basic stats: COMPLETE Column stats: NONE
    + Statistics: Num rows: 1327 Data size: 40738 Basic stats: COMPLETE Column stats: NONE
                        table:
                            input format: org.apache.hadoop.mapred.TextInputFormat
                            output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat

    http://git-wip-us.apache.org/repos/asf/hive/blob/b8f1ae11/ql/src/test/results/clientpositive/tez/vectorization_0.q.out
    ----------------------------------------------------------------------
    diff --git a/ql/src/test/results/clientpositive/tez/vectorization_0.q.out b/ql/src/test/results/clientpositive/tez/vectorization_0.q.out
    index cfbe9ce..18e042d 100644
    --- a/ql/src/test/results/clientpositive/tez/vectorization_0.q.out
    +++ b/ql/src/test/results/clientpositive/tez/vectorization_0.q.out
    @@ -994,7 +994,7 @@ STAGE PLANS:
                        alias: alltypesorc
                        Statistics: Num rows: 12288 Data size: 2641964 Basic stats: COMPLETE Column stats: NONE
                        Filter Operator
    - predicate: ((cstring2 like '%b%') or (79.553 <> UDFToDouble(cint)) or (UDFToDouble(cbigint) < cdouble)) (type: boolean)
    + predicate: ((cstring2 like '%b%') or (79.553 <> UDFToDouble(cint)) or (UDFToDouble(cbigint) < cdouble) or ((UDFToShort(ctinyint) >= csmallint) and (cboolean2 = 1) and (3569.0 = UDFToDouble(ctinyint)))) (type: boolean)
                          Statistics: Num rows: 12288 Data size: 2641964 Basic stats: COMPLETE Column stats: NONE
                          Select Operator
                            expressions: cbigint (type: bigint), cfloat (type: float), ctinyint (type: tinyint)

    http://git-wip-us.apache.org/repos/asf/hive/blob/b8f1ae11/ql/src/test/results/clientpositive/tez/vectorization_short_regress.q.out
    ----------------------------------------------------------------------
    diff --git a/ql/src/test/results/clientpositive/tez/vectorization_short_regress.q.out b/ql/src/test/results/clientpositive/tez/vectorization_short_regress.q.out
    index 5e0d42c..59b457a 100644
    --- a/ql/src/test/results/clientpositive/tez/vectorization_short_regress.q.out
    +++ b/ql/src/test/results/clientpositive/tez/vectorization_short_regress.q.out
    @@ -2085,23 +2085,23 @@ STAGE PLANS:
                        alias: alltypesorc
                        Statistics: Num rows: 12288 Data size: 2641964 Basic stats: COMPLETE Column stats: NONE
                        Filter Operator
    - predicate: ((cdouble > 2563.58) and (((cbigint >= UDFToLong(cint)) and (UDFToInteger(csmallint) < cint) and (UDFToDouble(cfloat) < -5638.15)) or ((cdouble <= UDFToDouble(cbigint)) and (-5638.15 > UDFToDouble(cbigint))))) (type: boolean)
    - Statistics: Num rows: 606 Data size: 130292 Basic stats: COMPLETE Column stats: NONE
    + predicate: ((cdouble > 2563.58) and (((cbigint >= UDFToLong(cint)) and (UDFToInteger(csmallint) < cint) and (UDFToDouble(cfloat) < -5638.15)) or (2563.58 = UDFToDouble(ctinyint)) or ((cdouble <= UDFToDouble(cbigint)) and (-5638.15 > UDFToDouble(cbigint))))) (type: boolean)
    + Statistics: Num rows: 2654 Data size: 570619 Basic stats: COMPLETE Column stats: NONE
                          Select Operator
                            expressions: cdouble (type: double), cfloat (type: float)
                            outputColumnNames: _col0, _col1
    - Statistics: Num rows: 606 Data size: 130292 Basic stats: COMPLETE Column stats: NONE
    + Statistics: Num rows: 2654 Data size: 570619 Basic stats: COMPLETE Column stats: NONE
                            Group By Operator
                              aggregations: var_samp(_col0), count(_col1), sum(_col1), var_pop(_col0), stddev_pop(_col0), sum(_col0)
                              keys: _col0 (type: double)
                              mode: hash
                              outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6
    - Statistics: Num rows: 606 Data size: 130292 Basic stats: COMPLETE Column stats: NONE
    + Statistics: Num rows: 2654 Data size: 570619 Basic stats: COMPLETE Column stats: NONE
                              Reduce Output Operator
                                key expressions: _col0 (type: double)
                                sort order: +
                                Map-reduce partition columns: _col0 (type: double)
    - Statistics: Num rows: 606 Data size: 130292 Basic stats: COMPLETE Column stats: NONE
    + Statistics: Num rows: 2654 Data size: 570619 Basic stats: COMPLETE Column stats: NONE
                                value expressions: _col1 (type: struct<count:bigint,sum:double,variance:double>), _col2 (type: bigint), _col3 (type: double), _col4 (type: struct<count:bigint,sum:double,variance:double>), _col5 (type: struct<count:bigint,sum:double,variance:double>), _col6 (type: double)
                  Execution mode: vectorized
              Reducer 2
    @@ -2111,25 +2111,25 @@ STAGE PLANS:
                      keys: KEY._col0 (type: double)
                      mode: mergepartial
                      outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6
    - Statistics: Num rows: 303 Data size: 65146 Basic stats: COMPLETE Column stats: NONE
    + Statistics: Num rows: 1327 Data size: 285309 Basic stats: COMPLETE Column stats: NONE
                      Select Operator
                        expressions: _col0 (type: double), _col1 (type: double), _col5 (type: double), (_col0 + _col1) (type: double), (_col0 * 762.0) (type: double), _col6 (type: double), (-863.257 % (_col0 * 762.0)) (type: double), (2563.58 * _col1) (type: double), (- _col1) (type: double), _col2 (type: bigint), ((2563.58 * _col1) + -5638.15) (type: double), ((- _col1) * ((2563.58 * _col1) + -5638.15)) (type: double), _col3 (type: double), _col4 (type: double), (_col0 - (- _col1)) (type: double)
                        outputColumnNames: _col0, _col1, _col10, _col11, _col12, _col13, _col14, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9
    - Statistics: Num rows: 303 Data size: 65146 Basic stats: COMPLETE Column stats: NONE
    + Statistics: Num rows: 1327 Data size: 285309 Basic stats: COMPLETE Column stats: NONE
                        Reduce Output Operator
                          key expressions: _col0 (type: double)
                          sort order: +
    - Statistics: Num rows: 303 Data size: 65146 Basic stats: COMPLETE Column stats: NONE
    + Statistics: Num rows: 1327 Data size: 285309 Basic stats: COMPLETE Column stats: NONE
                          value expressions: _col1 (type: double), _col2 (type: double), _col3 (type: double), _col4 (type: bigint), _col5 (type: double), _col6 (type: double), _col7 (type: double), _col8 (type: double), _col9 (type: double), _col10 (type: double), _col11 (type: double), _col12 (type: double), _col13 (type: double), _col14 (type: double)
              Reducer 3
                  Reduce Operator Tree:
                    Select Operator
                      expressions: KEY.reducesinkkey0 (type: double), VALUE._col0 (type: double), VALUE._col1 (type: double), VALUE._col2 (type: double), VALUE._col3 (type: bigint), VALUE._col4 (type: double), VALUE._col5 (type: double), VALUE._col6 (type: double), VALUE._col7 (type: double), VALUE._col8 (type: double), VALUE._col9 (type: double), VALUE._col10 (type: double), VALUE._col11 (type: double), VALUE._col12 (type: double), VALUE._col13 (type: double), VALUE._col12 (type: double)
                      outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15
    - Statistics: Num rows: 303 Data size: 65146 Basic stats: COMPLETE Column stats: NONE
    + Statistics: Num rows: 1327 Data size: 285309 Basic stats: COMPLETE Column stats: NONE
                      File Output Operator
                        compressed: false
    - Statistics: Num rows: 303 Data size: 65146 Basic stats: COMPLETE Column stats: NONE
    + Statistics: Num rows: 1327 Data size: 285309 Basic stats: COMPLETE Column stats: NONE
                        table:
                            input format: org.apache.hadoop.mapred.TextInputFormat
                            output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat

    http://git-wip-us.apache.org/repos/asf/hive/blob/b8f1ae11/ql/src/test/results/clientpositive/vectorization_0.q.out
    ----------------------------------------------------------------------
    diff --git a/ql/src/test/results/clientpositive/vectorization_0.q.out b/ql/src/test/results/clientpositive/vectorization_0.q.out
    index 3fab2ff..89163cd 100644
    --- a/ql/src/test/results/clientpositive/vectorization_0.q.out
    +++ b/ql/src/test/results/clientpositive/vectorization_0.q.out
    @@ -1010,7 +1010,7 @@ STAGE PLANS:
                  alias: alltypesorc
                  Statistics: Num rows: 12288 Data size: 2641964 Basic stats: COMPLETE Column stats: NONE
                  Filter Operator
    - predicate: ((cstring2 like '%b%') or (79.553 <> UDFToDouble(cint)) or (UDFToDouble(cbigint) < cdouble)) (type: boolean)
    + predicate: ((cstring2 like '%b%') or (79.553 <> UDFToDouble(cint)) or (UDFToDouble(cbigint) < cdouble) or ((UDFToShort(ctinyint) >= csmallint) and (cboolean2 = 1) and (3569.0 = UDFToDouble(ctinyint)))) (type: boolean)
                    Statistics: Num rows: 12288 Data size: 2641964 Basic stats: COMPLETE Column stats: NONE
                    Select Operator
                      expressions: cbigint (type: bigint), cfloat (type: float), ctinyint (type: tinyint)

    http://git-wip-us.apache.org/repos/asf/hive/blob/b8f1ae11/ql/src/test/results/clientpositive/vectorization_short_regress.q.out
    ----------------------------------------------------------------------
    diff --git a/ql/src/test/results/clientpositive/vectorization_short_regress.q.out b/ql/src/test/results/clientpositive/vectorization_short_regress.q.out
    index f2cb3ec..728f628 100644
    --- a/ql/src/test/results/clientpositive/vectorization_short_regress.q.out
    +++ b/ql/src/test/results/clientpositive/vectorization_short_regress.q.out
    @@ -2031,23 +2031,23 @@ STAGE PLANS:
                  alias: alltypesorc
                  Statistics: Num rows: 12288 Data size: 2641964 Basic stats: COMPLETE Column stats: NONE
                  Filter Operator
    - predicate: ((cdouble > 2563.58) and (((cbigint >= UDFToLong(cint)) and (UDFToInteger(csmallint) < cint) and (UDFToDouble(cfloat) < -5638.15)) or ((cdouble <= UDFToDouble(cbigint)) and (-5638.15 > UDFToDouble(cbigint))))) (type: boolean)
    - Statistics: Num rows: 606 Data size: 130292 Basic stats: COMPLETE Column stats: NONE
    + predicate: ((cdouble > 2563.58) and (((cbigint >= UDFToLong(cint)) and (UDFToInteger(csmallint) < cint) and (UDFToDouble(cfloat) < -5638.15)) or (2563.58 = UDFToDouble(ctinyint)) or ((cdouble <= UDFToDouble(cbigint)) and (-5638.15 > UDFToDouble(cbigint))))) (type: boolean)
    + Statistics: Num rows: 2654 Data size: 570619 Basic stats: COMPLETE Column stats: NONE
                    Select Operator
                      expressions: cdouble (type: double), cfloat (type: float)
                      outputColumnNames: _col0, _col1
    - Statistics: Num rows: 606 Data size: 130292 Basic stats: COMPLETE Column stats: NONE
    + Statistics: Num rows: 2654 Data size: 570619 Basic stats: COMPLETE Column stats: NONE
                      Group By Operator
                        aggregations: var_samp(_col0), count(_col1), sum(_col1), var_pop(_col0), stddev_pop(_col0), sum(_col0)
                        keys: _col0 (type: double)
                        mode: hash
                        outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6
    - Statistics: Num rows: 606 Data size: 130292 Basic stats: COMPLETE Column stats: NONE
    + Statistics: Num rows: 2654 Data size: 570619 Basic stats: COMPLETE Column stats: NONE
                        Reduce Output Operator
                          key expressions: _col0 (type: double)
                          sort order: +
                          Map-reduce partition columns: _col0 (type: double)
    - Statistics: Num rows: 606 Data size: 130292 Basic stats: COMPLETE Column stats: NONE
    + Statistics: Num rows: 2654 Data size: 570619 Basic stats: COMPLETE Column stats: NONE
                          value expressions: _col1 (type: struct<count:bigint,sum:double,variance:double>), _col2 (type: bigint), _col3 (type: double), _col4 (type: struct<count:bigint,sum:double,variance:double>), _col5 (type: struct<count:bigint,sum:double,variance:double>), _col6 (type: double)
            Execution mode: vectorized
            Reduce Operator Tree:
    @@ -2056,11 +2056,11 @@ STAGE PLANS:
                keys: KEY._col0 (type: double)
                mode: mergepartial
                outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6
    - Statistics: Num rows: 303 Data size: 65146 Basic stats: COMPLETE Column stats: NONE
    + Statistics: Num rows: 1327 Data size: 285309 Basic stats: COMPLETE Column stats: NONE
                Select Operator
                  expressions: _col0 (type: double), _col1 (type: double), _col5 (type: double), (_col0 + _col1) (type: double), (_col0 * 762.0) (type: double), _col6 (type: double), (-863.257 % (_col0 * 762.0)) (type: double), (2563.58 * _col1) (type: double), (- _col1) (type: double), _col2 (type: bigint), ((2563.58 * _col1) + -5638.15) (type: double), ((- _col1) * ((2563.58 * _col1) + -5638.15)) (type: double), _col3 (type: double), _col4 (type: double), (_col0 - (- _col1)) (type: double)
                  outputColumnNames: _col0, _col1, _col10, _col11, _col12, _col13, _col14, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9
    - Statistics: Num rows: 303 Data size: 65146 Basic stats: COMPLETE Column stats: NONE
    + Statistics: Num rows: 1327 Data size: 285309 Basic stats: COMPLETE Column stats: NONE
                  File Output Operator
                    compressed: false
                    table:
    @@ -2075,16 +2075,16 @@ STAGE PLANS:
                  Reduce Output Operator
                    key expressions: _col0 (type: double)
                    sort order: +
    - Statistics: Num rows: 303 Data size: 65146 Basic stats: COMPLETE Column stats: NONE
    + Statistics: Num rows: 1327 Data size: 285309 Basic stats: COMPLETE Column stats: NONE
                    value expressions: _col1 (type: double), _col2 (type: double), _col3 (type: double), _col4 (type: bigint), _col5 (type: double), _col6 (type: double), _col7 (type: double), _col8 (type: double), _col9 (type: double), _col10 (type: double), _col11 (type: double), _col12 (type: double), _col13 (type: double), _col14 (type: double)
            Reduce Operator Tree:
              Select Operator
                expressions: KEY.reducesinkkey0 (type: double), VALUE._col0 (type: double), VALUE._col1 (type: double), VALUE._col2 (type: double), VALUE._col3 (type: bigint), VALUE._col4 (type: double), VALUE._col5 (type: double), VALUE._col6 (type: double), VALUE._col7 (type: double), VALUE._col8 (type: double), VALUE._col9 (type: double), VALUE._col10 (type: double), VALUE._col11 (type: double), VALUE._col12 (type: double), VALUE._col13 (type: double), VALUE._col12 (type: double)
                outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15
    - Statistics: Num rows: 303 Data size: 65146 Basic stats: COMPLETE Column stats: NONE
    + Statistics: Num rows: 1327 Data size: 285309 Basic stats: COMPLETE Column stats: NONE
                File Output Operator
                  compressed: false
    - Statistics: Num rows: 303 Data size: 65146 Basic stats: COMPLETE Column stats: NONE
    + Statistics: Num rows: 1327 Data size: 285309 Basic stats: COMPLETE Column stats: NONE
                  table:
                      input format: org.apache.hadoop.mapred.TextInputFormat
                      output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
  • Sershe at Aug 25, 2015 at 9:28 pm
    HIVE-11541: ORC: Split Strategy should depend on global file count, not per-partition (Gopal V reviewed by Prasanth Jayachandran)


    Project: http://git-wip-us.apache.org/repos/asf/hive/repo
    Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/f26b2569
    Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/f26b2569
    Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/f26b2569

    Branch: refs/heads/llap
    Commit: f26b2569198fbeceaf17a5a77c59eccf5175935c
    Parents: db46e6e
    Author: Prasanth Jayachandran <j.prasanth.j@gmail.com>
    Authored: Thu Aug 13 12:35:29 2015 -0700
    Committer: Prasanth Jayachandran <j.prasanth.j@gmail.com>
    Committed: Thu Aug 13 12:35:29 2015 -0700

    ----------------------------------------------------------------------
      ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java | 4 ++--
      1 file changed, 2 insertions(+), 2 deletions(-)
    ----------------------------------------------------------------------


    http://git-wip-us.apache.org/repos/asf/hive/blob/f26b2569/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java
    ----------------------------------------------------------------------
    diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java
    index 4e6dd7a..fe2eccd 100644
    --- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java
    +++ b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java
    @@ -483,7 +483,6 @@ public class OrcInputFormat implements InputFormat<NullWritable, OrcStruct>,
          }

          private FileInfo verifyCachedFileInfo(FileStatus file) {
    - context.numFilesCounter.incrementAndGet();
            FileInfo fileInfo = Context.footerCache.getIfPresent(file.getPath());
            if (fileInfo != null) {
              if (isDebugEnabled) {
    @@ -671,6 +670,7 @@ public class OrcInputFormat implements InputFormat<NullWritable, OrcStruct>,

              int numFiles = children.size();
              long avgFileSize = totalFileSize / numFiles;
    + int totalFiles = context.numFilesCounter.addAndGet(numFiles);
              switch(context.splitStrategyKind) {
                case BI:
                  // BI strategy requested through config
    @@ -684,7 +684,7 @@ public class OrcInputFormat implements InputFormat<NullWritable, OrcStruct>,
                  break;
                default:
                  // HYBRID strategy
    - if (avgFileSize > context.maxSize || numFiles <= context.minSplits) {
    + if (avgFileSize > context.maxSize || totalFiles <= context.minSplits) {
                    splitStrategy = new ETLSplitStrategy(context, fs, dir, children, isOriginal, deltas,
                        covered);
                  } else {
  • Sershe at Aug 25, 2015 at 9:28 pm
    HIVE-11467 : WriteBuffers rounding wbSize to next power of 2 may cause OOM (Wei Zheng, reviewed by Sergey Shelukhin)


    Project: http://git-wip-us.apache.org/repos/asf/hive/repo
    Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/433ea9cd
    Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/433ea9cd
    Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/433ea9cd

    Branch: refs/heads/llap
    Commit: 433ea9cda59a986689cf99953826512e4ff07e4d
    Parents: 1a75644
    Author: Sergey Shelukhin <sershe@apache.org>
    Authored: Thu Aug 13 14:46:07 2015 -0700
    Committer: Sergey Shelukhin <sershe@apache.org>
    Committed: Thu Aug 13 14:46:07 2015 -0700

    ----------------------------------------------------------------------
      .../hive/ql/exec/persistence/HybridHashTableContainer.java | 4 ++++
      serde/src/java/org/apache/hadoop/hive/serde2/WriteBuffers.java | 2 +-
      2 files changed, 5 insertions(+), 1 deletion(-)
    ----------------------------------------------------------------------


    http://git-wip-us.apache.org/repos/asf/hive/blob/433ea9cd/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/HybridHashTableContainer.java
    ----------------------------------------------------------------------
    diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/HybridHashTableContainer.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/HybridHashTableContainer.java
    index ad1246d..ff64f52 100644
    --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/HybridHashTableContainer.java
    +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/HybridHashTableContainer.java
    @@ -290,6 +290,10 @@ public class HybridHashTableContainer
            }
          }

    + // Round to power of 2 here, as is required by WriteBuffers
    + writeBufferSize = Integer.bitCount(writeBufferSize) == 1 ?
    + writeBufferSize : Integer.highestOneBit(writeBufferSize);
    +
          // Cap WriteBufferSize to avoid large preallocations
          writeBufferSize = writeBufferSize < minWbSize ? minWbSize : Math.min(maxWbSize, writeBufferSize);
          LOG.info("Write buffer size: " + writeBufferSize);

    http://git-wip-us.apache.org/repos/asf/hive/blob/433ea9cd/serde/src/java/org/apache/hadoop/hive/serde2/WriteBuffers.java
    ----------------------------------------------------------------------
    diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/WriteBuffers.java b/serde/src/java/org/apache/hadoop/hive/serde2/WriteBuffers.java
    index acb51f9..05d9359 100644
    --- a/serde/src/java/org/apache/hadoop/hive/serde2/WriteBuffers.java
    +++ b/serde/src/java/org/apache/hadoop/hive/serde2/WriteBuffers.java
    @@ -56,7 +56,7 @@ public final class WriteBuffers implements RandomAccessOutput {


        public WriteBuffers(int wbSize, long maxSize) {
    - this.wbSize = Integer.bitCount(wbSize) == 1 ? wbSize : (Integer.highestOneBit(wbSize) << 1);
    + this.wbSize = Integer.bitCount(wbSize) == 1 ? wbSize : Integer.highestOneBit(wbSize);
          this.wbSizeLog2 = 31 - Integer.numberOfLeadingZeros(this.wbSize);
          this.offsetMask = this.wbSize - 1;
          this.maxSize = maxSize;
  • Sershe at Aug 25, 2015 at 9:28 pm
    HIVE-11464: lineage info missing if there are multiple outputs (Jimmy)


    Project: http://git-wip-us.apache.org/repos/asf/hive/repo
    Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/1a75644d
    Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/1a75644d
    Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/1a75644d

    Branch: refs/heads/llap
    Commit: 1a75644d68c8c61fbafb4058fe45b7823492491c
    Parents: f26b256
    Author: Jimmy Xiang <jxiang@cloudera.com>
    Authored: Wed Aug 5 08:02:50 2015 -0700
    Committer: Jimmy Xiang <jxiang@cloudera.com>
    Committed: Thu Aug 13 13:44:03 2015 -0700

    ----------------------------------------------------------------------
      .../java/org/apache/hadoop/hive/ql/Driver.java | 8 ++--
      .../hadoop/hive/ql/hooks/LineageInfo.java | 9 ++--
      .../hadoop/hive/ql/hooks/LineageLogger.java | 44 +++++++++++++-------
      .../ql/optimizer/lineage/ExprProcFactory.java | 9 ++--
      .../hive/ql/optimizer/lineage/LineageCtx.java | 34 +++++++++++----
      .../ql/optimizer/lineage/OpProcFactory.java | 10 ++---
      ql/src/test/queries/clientpositive/lineage3.q | 15 +++++++
      .../test/results/clientpositive/lineage3.q.out | 32 +++++++++++++-
      8 files changed, 118 insertions(+), 43 deletions(-)
    ----------------------------------------------------------------------


    http://git-wip-us.apache.org/repos/asf/hive/blob/1a75644d/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
    ----------------------------------------------------------------------
    diff --git a/ql/src/java/org/apache/hadoop/hive/ql/Driver.java b/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
    index e7b7b55..c0c1b2e 100644
    --- a/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
    +++ b/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
    @@ -441,8 +441,11 @@ public class Driver implements CommandProcessor {
            // to avoid returning sensitive data
            String queryStr = HookUtils.redactLogString(conf, command);

    + // get the output schema
    + schema = getSchema(sem, conf);
    +
            plan = new QueryPlan(queryStr, sem, perfLogger.getStartTime(PerfLogger.DRIVER_RUN), queryId,
    - SessionState.get().getHiveOperation(), getSchema(sem, conf));
    + SessionState.get().getHiveOperation(), schema);

            conf.setVar(HiveConf.ConfVars.HIVEQUERYSTRING, queryStr);

    @@ -454,9 +457,6 @@ public class Driver implements CommandProcessor {
              plan.getFetchTask().initialize(conf, plan, null);
            }

    - // get the output schema
    - schema = getSchema(sem, conf);
    -
            //do the authorization check
            if (!sem.skipAuthorization() &&
                HiveConf.getBoolVar(conf, HiveConf.ConfVars.HIVE_AUTHORIZATION_ENABLED)) {

    http://git-wip-us.apache.org/repos/asf/hive/blob/1a75644d/ql/src/java/org/apache/hadoop/hive/ql/hooks/LineageInfo.java
    ----------------------------------------------------------------------
    diff --git a/ql/src/java/org/apache/hadoop/hive/ql/hooks/LineageInfo.java b/ql/src/java/org/apache/hadoop/hive/ql/hooks/LineageInfo.java
    index fe0841e..2806c54 100644
    --- a/ql/src/java/org/apache/hadoop/hive/ql/hooks/LineageInfo.java
    +++ b/ql/src/java/org/apache/hadoop/hive/ql/hooks/LineageInfo.java
    @@ -22,7 +22,6 @@ import java.io.Serializable;
      import java.util.Collections;
      import java.util.LinkedHashMap;
      import java.util.LinkedHashSet;
    -import java.util.List;
      import java.util.Map;
      import java.util.Set;

    @@ -375,9 +374,9 @@ public class LineageInfo implements Serializable {
          private String expr;

          /**
    - * The list of base columns that the particular column depends on.
    + * The set of base columns that the particular column depends on.
           */
    - private List<BaseColumnInfo> baseCols;
    + private Set<BaseColumnInfo> baseCols;

          /**
           * @return the type
    @@ -410,14 +409,14 @@ public class LineageInfo implements Serializable {
          /**
           * @return the baseCols
           */
    - public List<BaseColumnInfo> getBaseCols() {
    + public Set<BaseColumnInfo> getBaseCols() {
            return baseCols;
          }

          /**
           * @param baseCols the baseCols to set
           */
    - public void setBaseCols(List<BaseColumnInfo> baseCols) {
    + public void setBaseCols(Set<BaseColumnInfo> baseCols) {
            this.baseCols = baseCols;
          }


    http://git-wip-us.apache.org/repos/asf/hive/blob/1a75644d/ql/src/java/org/apache/hadoop/hive/ql/hooks/LineageLogger.java
    ----------------------------------------------------------------------
    diff --git a/ql/src/java/org/apache/hadoop/hive/ql/hooks/LineageLogger.java b/ql/src/java/org/apache/hadoop/hive/ql/hooks/LineageLogger.java
    index d615372..3c6ce94 100644
    --- a/ql/src/java/org/apache/hadoop/hive/ql/hooks/LineageLogger.java
    +++ b/ql/src/java/org/apache/hadoop/hive/ql/hooks/LineageLogger.java
    @@ -33,6 +33,7 @@ import org.apache.commons.io.output.StringBuilderWriter;
      import org.apache.commons.lang.StringUtils;
      import org.apache.commons.logging.Log;
      import org.apache.commons.logging.LogFactory;
    +import org.apache.hadoop.hive.common.ObjectPair;
      import org.apache.hadoop.hive.conf.HiveConf;
      import org.apache.hadoop.hive.metastore.api.FieldSchema;
      import org.apache.hadoop.hive.metastore.api.Table;
    @@ -147,6 +148,7 @@ public class LineageLogger implements ExecuteWithHookContext {
                // Don't emit user/timestamp info in test mode,
                // so that the test golden output file is fixed.
                long queryTime = plan.getQueryStartTime().longValue();
    + if (queryTime == 0) queryTime = System.currentTimeMillis();
                writer.name("user").value(hookContext.getUgi().getUserName());
                writer.name("timestamp").value(queryTime/1000);
                writer.name("jobIds");
    @@ -209,23 +211,28 @@ public class LineageLogger implements ExecuteWithHookContext {
         * For each target column, find out its sources based on the dependency index.
         */
        private List<Edge> getEdges(QueryPlan plan, Index index) {
    - List<FieldSchema> fieldSchemas = plan.getResultSchema().getFieldSchemas();
    - int fields = fieldSchemas == null ? 0 : fieldSchemas.size();
    - SelectOperator finalSelOp = index.getFinalSelectOp();
    + LinkedHashMap<String, ObjectPair<SelectOperator,
    + org.apache.hadoop.hive.ql.metadata.Table>> finalSelOps = index.getFinalSelectOps();
    + Set<Vertex> allTargets = new LinkedHashSet<Vertex>();
    + Map<String, Vertex> allSources = new LinkedHashMap<String, Vertex>();
          List<Edge> edges = new ArrayList<Edge>();
    - if (finalSelOp != null && fields > 0) {
    - Map<ColumnInfo, Dependency> colMap = index.getDependencies(finalSelOp);
    - List<Dependency> dependencies = colMap != null ? Lists.newArrayList(colMap.values()) : null;
    - if (dependencies == null || dependencies.size() != fields) {
    - log("Result schema has " + fields
    - + " fields, but we don't get as many dependencies");
    + for (ObjectPair<SelectOperator,
    + org.apache.hadoop.hive.ql.metadata.Table> pair: finalSelOps.values()) {
    + List<FieldSchema> fieldSchemas = plan.getResultSchema().getFieldSchemas();
    + SelectOperator finalSelOp = pair.getFirst();
    + org.apache.hadoop.hive.ql.metadata.Table t = pair.getSecond();
    + String destTableName = null;
    + List<String> colNames = null;
    + if (t != null) {
    + destTableName = t.getDbName() + "." + t.getTableName();
    + fieldSchemas = t.getCols();
            } else {
    - String destTableName = null;
    - List<String> colNames = null;
              // Based on the plan outputs, find out the target table name and column names.
              for (WriteEntity output : plan.getOutputs()) {
    - if (output.getType() == Entity.Type.TABLE) {
    - org.apache.hadoop.hive.ql.metadata.Table t = output.getTable();
    + Entity.Type entityType = output.getType();
    + if (entityType == Entity.Type.TABLE
    + || entityType == Entity.Type.PARTITION) {
    + t = output.getTable();
                  destTableName = t.getDbName() + "." + t.getTableName();
                  List<FieldSchema> cols = t.getCols();
                  if (cols != null && !cols.isEmpty()) {
    @@ -234,10 +241,15 @@ public class LineageLogger implements ExecuteWithHookContext {
                  break;
                }
              }
    -
    + }
    + int fields = fieldSchemas.size();
    + Map<ColumnInfo, Dependency> colMap = index.getDependencies(finalSelOp);
    + List<Dependency> dependencies = colMap != null ? Lists.newArrayList(colMap.values()) : null;
    + if (dependencies == null || dependencies.size() != fields) {
    + log("Result schema has " + fields
    + + " fields, but we don't get as many dependencies");
    + } else {
              // Go through each target column, generate the lineage edges.
    - Set<Vertex> allTargets = new LinkedHashSet<Vertex>();
    - Map<String, Vertex> allSources = new LinkedHashMap<String, Vertex>();
              for (int i = 0; i < fields; i++) {
                Vertex target = new Vertex(
                  getTargetFieldName(i, destTableName, colNames, fieldSchemas));

    http://git-wip-us.apache.org/repos/asf/hive/blob/1a75644d/ql/src/java/org/apache/hadoop/hive/ql/optimizer/lineage/ExprProcFactory.java
    ----------------------------------------------------------------------
    diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/lineage/ExprProcFactory.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/lineage/ExprProcFactory.java
    index 455a525..38040e3 100644
    --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/lineage/ExprProcFactory.java
    +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/lineage/ExprProcFactory.java
    @@ -24,6 +24,7 @@ import java.util.LinkedHashMap;
      import java.util.LinkedHashSet;
      import java.util.List;
      import java.util.Map;
    +import java.util.Set;
      import java.util.Stack;

      import org.apache.hadoop.hive.metastore.api.FieldSchema;
    @@ -124,7 +125,7 @@ public class ExprProcFactory {
              bci_set.addAll(child_dep.getBaseCols());
            }

    - dep.setBaseCols(new ArrayList<BaseColumnInfo>(bci_set));
    + dep.setBaseCols(bci_set);
            dep.setType(new_type);

            return dep;
    @@ -146,7 +147,7 @@ public class ExprProcFactory {
            // Create a dependency that has no basecols
            Dependency dep = new Dependency();
            dep.setType(LineageInfo.DependencyType.SIMPLE);
    - dep.setBaseCols(new ArrayList<BaseColumnInfo>());
    + dep.setBaseCols(new LinkedHashSet<BaseColumnInfo>());

            return dep;
          }
    @@ -218,9 +219,9 @@ public class ExprProcFactory {
            Dependency dep = lctx.getIndex().getDependency(inpOp, internalName);
            if ((tabAlias == null || tabAlias.startsWith("_") || tabAlias.startsWith("$"))
                && (dep != null && dep.getType() == DependencyType.SIMPLE)) {
    - List<BaseColumnInfo> baseCols = dep.getBaseCols();
    + Set<BaseColumnInfo> baseCols = dep.getBaseCols();
              if (baseCols != null && !baseCols.isEmpty()) {
    - BaseColumnInfo baseCol = baseCols.get(0);
    + BaseColumnInfo baseCol = baseCols.iterator().next();
                tabAlias = baseCol.getTabAlias().getAlias();
                alias = baseCol.getColumn().getName();
              }

    http://git-wip-us.apache.org/repos/asf/hive/blob/1a75644d/ql/src/java/org/apache/hadoop/hive/ql/optimizer/lineage/LineageCtx.java
    ----------------------------------------------------------------------
    diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/lineage/LineageCtx.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/lineage/LineageCtx.java
    index d26d8da..c33d775 100644
    --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/lineage/LineageCtx.java
    +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/lineage/LineageCtx.java
    @@ -25,7 +25,9 @@ import java.util.LinkedHashSet;
      import java.util.Map;
      import java.util.Set;

    +import org.apache.hadoop.hive.common.ObjectPair;
      import org.apache.hadoop.hive.ql.exec.ColumnInfo;
    +import org.apache.hadoop.hive.ql.exec.FileSinkOperator;
      import org.apache.hadoop.hive.ql.exec.Operator;
      import org.apache.hadoop.hive.ql.exec.SelectOperator;
      import org.apache.hadoop.hive.ql.hooks.LineageInfo;
    @@ -33,6 +35,7 @@ import org.apache.hadoop.hive.ql.hooks.LineageInfo.BaseColumnInfo;
      import org.apache.hadoop.hive.ql.hooks.LineageInfo.Dependency;
      import org.apache.hadoop.hive.ql.hooks.LineageInfo.Predicate;
      import org.apache.hadoop.hive.ql.lib.NodeProcessorCtx;
    +import org.apache.hadoop.hive.ql.metadata.Table;
      import org.apache.hadoop.hive.ql.parse.ParseContext;
      import org.apache.hadoop.hive.ql.plan.OperatorDesc;

    @@ -59,7 +62,11 @@ public class LineageCtx implements NodeProcessorCtx {
           */
          private final Map<Operator<? extends OperatorDesc>, Set<Predicate>> condMap;

    - private SelectOperator finalSelectOp;
    + /**
    + * A map from a final select operator id to the select operator
    + * and the corresponding target table in case an insert into query.
    + */
    + private LinkedHashMap<String, ObjectPair<SelectOperator, Table>> finalSelectOps;

          /**
           * Constructor.
    @@ -69,6 +76,8 @@ public class LineageCtx implements NodeProcessorCtx {
              new LinkedHashMap<Operator<? extends OperatorDesc>,
                                LinkedHashMap<ColumnInfo, Dependency>>();
            condMap = new HashMap<Operator<? extends OperatorDesc>, Set<Predicate>>();
    + finalSelectOps =
    + new LinkedHashMap<String, ObjectPair<SelectOperator, Table>>();
          }

          /**
    @@ -146,7 +155,7 @@ public class LineageCtx implements NodeProcessorCtx {
              old_dep.setType(new_type);
              Set<BaseColumnInfo> bci_set = new LinkedHashSet<BaseColumnInfo>(old_dep.getBaseCols());
              bci_set.addAll(dep.getBaseCols());
    - old_dep.setBaseCols(new ArrayList<BaseColumnInfo>(bci_set));
    + old_dep.setBaseCols(bci_set);
              // TODO: Fix the expressions later.
              old_dep.setExpr(null);
            }
    @@ -179,16 +188,27 @@ public class LineageCtx implements NodeProcessorCtx {
            return condMap.get(op);
          }

    - public void setFinalSelectOp(SelectOperator sop) {
    - finalSelectOp = sop;
    + public void addFinalSelectOp(
    + SelectOperator sop, Operator<? extends OperatorDesc> sinkOp) {
    + String operatorId = sop.getOperatorId();
    + if (!finalSelectOps.containsKey(operatorId)) {
    + Table table = null;
    + if (sinkOp instanceof FileSinkOperator) {
    + FileSinkOperator fso = (FileSinkOperator) sinkOp;
    + table = fso.getConf().getTable();
    + }
    + finalSelectOps.put(operatorId,
    + new ObjectPair<SelectOperator, Table>(sop, table));
    + }
          }

    - public SelectOperator getFinalSelectOp() {
    - return finalSelectOp;
    + public LinkedHashMap<String,
    + ObjectPair<SelectOperator, Table>> getFinalSelectOps() {
    + return finalSelectOps;
          }

          public void clear() {
    - finalSelectOp = null;
    + finalSelectOps.clear();
            depMap.clear();
          }
        }

    http://git-wip-us.apache.org/repos/asf/hive/blob/1a75644d/ql/src/java/org/apache/hadoop/hive/ql/optimizer/lineage/OpProcFactory.java
    ----------------------------------------------------------------------
    diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/lineage/OpProcFactory.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/lineage/OpProcFactory.java
    index f670db8..5c5d0d6 100644
    --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/lineage/OpProcFactory.java
    +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/lineage/OpProcFactory.java
    @@ -120,7 +120,7 @@ public class OpProcFactory {
            }

            dep.setType(new_type);
    - dep.setBaseCols(new ArrayList<BaseColumnInfo>(col_set));
    + dep.setBaseCols(col_set);

            boolean isScript = op instanceof ScriptOperator;

    @@ -186,7 +186,7 @@ public class OpProcFactory {

              // Populate the dependency
              dep.setType(LineageInfo.DependencyType.SIMPLE);
    - dep.setBaseCols(new ArrayList<BaseColumnInfo>());
    + dep.setBaseCols(new LinkedHashSet<BaseColumnInfo>());
              dep.getBaseCols().add(bci);

              // Put the dependency in the map
    @@ -396,7 +396,7 @@ public class OpProcFactory {
            }
            if (op == null || (op.getChildOperators().isEmpty()
                && op instanceof FileSinkOperator)) {
    - lctx.getIndex().setFinalSelectOp(sop);
    + lctx.getIndex().addFinalSelectOp(sop, op);
            }

            return null;
    @@ -450,7 +450,7 @@ public class OpProcFactory {
                  new_type = LineageCtx.getNewDependencyType(expr_dep.getType(), new_type);
                  bci_set.addAll(expr_dep.getBaseCols());
                  if (expr_dep.getType() == LineageInfo.DependencyType.SIMPLE) {
    - BaseColumnInfo col = expr_dep.getBaseCols().get(0);
    + BaseColumnInfo col = expr_dep.getBaseCols().iterator().next();
                    Table t = col.getTabAlias().getTable();
                    if (t != null) {
                      sb.append(t.getDbName()).append(".").append(t.getTableName()).append(".");
    @@ -514,7 +514,7 @@ public class OpProcFactory {
                }
              }

    - dep.setBaseCols(new ArrayList<BaseColumnInfo>(bci_set));
    + dep.setBaseCols(bci_set);
              dep.setType(new_type);
              lctx.getIndex().putDependency(gop, col_infos.get(cnt++), dep);
            }

    http://git-wip-us.apache.org/repos/asf/hive/blob/1a75644d/ql/src/test/queries/clientpositive/lineage3.q
    ----------------------------------------------------------------------
    diff --git a/ql/src/test/queries/clientpositive/lineage3.q b/ql/src/test/queries/clientpositive/lineage3.q
    index 53fff0f..c24ff7d 100644
    --- a/ql/src/test/queries/clientpositive/lineage3.q
    +++ b/ql/src/test/queries/clientpositive/lineage3.q
    @@ -1,5 +1,20 @@
      set hive.exec.post.hooks=org.apache.hadoop.hive.ql.hooks.LineageLogger;

    +drop table if exists d1;
    +create table d1(a int);
    +
    +from (select a.ctinyint x, b.cstring1 y
    +from alltypesorc a join alltypesorc b on a.cint = b.cbigint) t
    +insert into table d1 select x + length(y);
    +
    +drop table if exists d2;
    +create table d2(b varchar(128));
    +
    +from (select a.ctinyint x, b.cstring1 y
    +from alltypesorc a join alltypesorc b on a.cint = b.cbigint) t
    +insert into table d1 select x where y is null
    +insert into table d2 select y where x > 0;
    +
      drop table if exists t;
      create table t as
      select * from

    http://git-wip-us.apache.org/repos/asf/hive/blob/1a75644d/ql/src/test/results/clientpositive/lineage3.q.out
    ----------------------------------------------------------------------
    diff --git a/ql/src/test/results/clientpositive/lineage3.q.out b/ql/src/test/results/clientpositive/lineage3.q.out
    index 75d88f8..b6b4e0b 100644
    --- a/ql/src/test/results/clientpositive/lineage3.q.out
    +++ b/ql/src/test/results/clientpositive/lineage3.q.out
    @@ -1,3 +1,31 @@
    +PREHOOK: query: drop table if exists d1
    +PREHOOK: type: DROPTABLE
    +PREHOOK: query: create table d1(a int)
    +PREHOOK: type: CREATETABLE
    +PREHOOK: Output: database:default
    +PREHOOK: Output: default@d1
    +PREHOOK: query: from (select a.ctinyint x, b.cstring1 y
    +from alltypesorc a join alltypesorc b on a.cint = b.cbigint) t
    +insert into table d1 select x + length(y)
    +PREHOOK: type: QUERY
    +PREHOOK: Input: default@alltypesorc
    +PREHOOK: Output: default@d1
    +{"version":"1.0","engine":"mr","hash":"4c9b7b8d89403cef78668f15d393e542","queryText":"from (select a.ctinyint x, b.cstring1 y\nfrom alltypesorc a join alltypesorc b on a.cint = b.cbigint) t\ninsert into table d1 select x + length(y)","edges":[{"sources":[1,2],"targets":[0],"expression":"(UDFToInteger(a.ctinyint) + length(a.cstring1))","edgeType":"PROJECTION"},{"sources":[3,4],"targets":[0],"expression":"(UDFToLong(a.cint) = a.cbigint)","edgeType":"PREDICATE"}],"vertices":[{"id":0,"vertexType":"COLUMN","vertexId":"default.d1.a"},{"id":1,"vertexType":"COLUMN","vertexId":"default.alltypesorc.ctinyint"},{"id":2,"vertexType":"COLUMN","vertexId":"default.alltypesorc.cstring1"},{"id":3,"vertexType":"COLUMN","vertexId":"default.alltypesorc.cint"},{"id":4,"vertexType":"COLUMN","vertexId":"default.alltypesorc.cbigint"}]}
    +PREHOOK: query: drop table if exists d2
    +PREHOOK: type: DROPTABLE
    +PREHOOK: query: create table d2(b varchar(128))
    +PREHOOK: type: CREATETABLE
    +PREHOOK: Output: database:default
    +PREHOOK: Output: default@d2
    +PREHOOK: query: from (select a.ctinyint x, b.cstring1 y
    +from alltypesorc a join alltypesorc b on a.cint = b.cbigint) t
    +insert into table d1 select x where y is null
    +insert into table d2 select y where x > 0
    +PREHOOK: type: QUERY
    +PREHOOK: Input: default@alltypesorc
    +PREHOOK: Output: default@d1
    +PREHOOK: Output: default@d2
    +{"version":"1.0","engine":"mr","hash":"8703e4091ebd4c96afd3cac83e3a2957","queryText":"from (select a.ctinyint x, b.cstring1 y\nfrom alltypesorc a join alltypesorc b on a.cint = b.cbigint) t\ninsert into table d1 select x where y is null\ninsert into table d2 select y where x > 0","edges":[{"sources":[2],"targets":[0],"expression":"UDFToInteger(x)","edgeType":"PROJECTION"},{"sources":[3],"targets":[0,1],"expression":"t.y is null","edgeType":"PREDICATE"},{"sources":[4,5],"targets":[0,1],"expression":"(UDFToLong(a.cint) = b.cbigint)","edgeType":"PREDICATE"},{"sources":[3],"targets":[1],"expression":"CAST( y AS varchar(128))","edgeType":"PROJECTION"},{"sources":[2],"targets":[0,1],"expression":"(t.x > 0)","edgeType":"PREDICATE"}],"vertices":[{"id":0,"vertexType":"COLUMN","vertexId":"default.d1.a"},{"id":1,"vertexType":"COLUMN","vertexId":"default.d2.b"},{"id":2,"vertexType":"COLUMN","vertexId":"default.alltypesorc.ctinyint"},{"id":3,"vertexType":"COLUMN","vertexId":"default.alltypesorc.
      cstring1"},{"id":4,"vertexType":"COLUMN","vertexId":"default.alltypesorc.cint"},{"id":5,"vertexType":"COLUMN","vertexId":"default.alltypesorc.cbigint"}]}
      PREHOOK: query: drop table if exists t
      PREHOOK: type: DROPTABLE
      PREHOOK: query: create table t as
    @@ -23,7 +51,7 @@ where cint is not null and cint < 0 order by cint, cs limit 5
      PREHOOK: type: QUERY
      PREHOOK: Input: default@alltypesorc
      PREHOOK: Output: default@dest_l1@ds=today
    -{"version":"1.0","engine":"mr","hash":"2b5891d094ff74e23ec6acf5b4990f45","queryText":"insert into table dest_l1 partition (ds='today')\nselect cint, cast(cstring1 as varchar(128)) as cs\nfrom alltypesorc\nwhere cint is not null and cint < 0 order by cint, cs limit 5","edges":[{"sources":[2],"targets":[0],"edgeType":"PROJECTION"},{"sources":[3],"targets":[1],"expression":"CAST( alltypesorc.cstring1 AS varchar(128))","edgeType":"PROJECTION"},{"sources":[2],"targets":[0,1],"expression":"(alltypesorc.cint is not null and (alltypesorc.cint < 0))","edgeType":"PREDICATE"}],"vertices":[{"id":0,"vertexType":"COLUMN","vertexId":"cint"},{"id":1,"vertexType":"COLUMN","vertexId":"cs"},{"id":2,"vertexType":"COLUMN","vertexId":"default.alltypesorc.cint"},{"id":3,"vertexType":"COLUMN","vertexId":"default.alltypesorc.cstring1"}]}
    +{"version":"1.0","engine":"mr","hash":"2b5891d094ff74e23ec6acf5b4990f45","queryText":"insert into table dest_l1 partition (ds='today')\nselect cint, cast(cstring1 as varchar(128)) as cs\nfrom alltypesorc\nwhere cint is not null and cint < 0 order by cint, cs limit 5","edges":[{"sources":[2],"targets":[0],"edgeType":"PROJECTION"},{"sources":[3],"targets":[1],"expression":"CAST( alltypesorc.cstring1 AS varchar(128))","edgeType":"PROJECTION"},{"sources":[2],"targets":[0,1],"expression":"(alltypesorc.cint is not null and (alltypesorc.cint < 0))","edgeType":"PREDICATE"}],"vertices":[{"id":0,"vertexType":"COLUMN","vertexId":"default.dest_l1.a"},{"id":1,"vertexType":"COLUMN","vertexId":"default.dest_l1.b"},{"id":2,"vertexType":"COLUMN","vertexId":"default.alltypesorc.cint"},{"id":3,"vertexType":"COLUMN","vertexId":"default.alltypesorc.cstring1"}]}
      PREHOOK: query: insert into table dest_l1 partition (ds='tomorrow')
      select min(cint), cast(min(cstring1) as varchar(128)) as cs
      from alltypesorc
    @@ -33,7 +61,7 @@ having min(cbigint) > 10
      PREHOOK: type: QUERY
      PREHOOK: Input: default@alltypesorc
      PREHOOK: Output: default@dest_l1@ds=tomorrow
    -{"version":"1.0","engine":"mr","hash":"4ad6338a8abfe3fe0342198fcbd1f11d","queryText":"insert into table dest_l1 partition (ds='tomorrow')\nselect min(cint), cast(min(cstring1) as varchar(128)) as cs\nfrom alltypesorc\nwhere cint is not null and cboolean1 = true\ngroup by csmallint\nhaving min(cbigint) > 10","edges":[{"sources":[2],"targets":[0],"expression":"min(default.alltypesorc.cint)","edgeType":"PROJECTION"},{"sources":[3],"targets":[1],"expression":"CAST( min(default.alltypesorc.cstring1) AS varchar(128))","edgeType":"PROJECTION"},{"sources":[2,4],"targets":[0,1],"expression":"(alltypesorc.cint is not null and (alltypesorc.cboolean1 = true))","edgeType":"PREDICATE"},{"sources":[5],"targets":[0,1],"expression":"(min(default.alltypesorc.cbigint) > 10)","edgeType":"PREDICATE"}],"vertices":[{"id":0,"vertexType":"COLUMN","vertexId":"c0"},{"id":1,"vertexType":"COLUMN","vertexId":"cs"},{"id":2,"vertexType":"COLUMN","vertexId":"default.alltypesorc.cint"},{"id":3,"vertexType":"COLUMN",
      "vertexId":"default.alltypesorc.cstring1"},{"id":4,"vertexType":"COLUMN","vertexId":"default.alltypesorc.cboolean1"},{"id":5,"vertexType":"COLUMN","vertexId":"default.alltypesorc.cbigint"}]}
    +{"version":"1.0","engine":"mr","hash":"4ad6338a8abfe3fe0342198fcbd1f11d","queryText":"insert into table dest_l1 partition (ds='tomorrow')\nselect min(cint), cast(min(cstring1) as varchar(128)) as cs\nfrom alltypesorc\nwhere cint is not null and cboolean1 = true\ngroup by csmallint\nhaving min(cbigint) > 10","edges":[{"sources":[2],"targets":[0],"expression":"min(default.alltypesorc.cint)","edgeType":"PROJECTION"},{"sources":[3],"targets":[1],"expression":"CAST( min(default.alltypesorc.cstring1) AS varchar(128))","edgeType":"PROJECTION"},{"sources":[2,4],"targets":[0,1],"expression":"(alltypesorc.cint is not null and (alltypesorc.cboolean1 = true))","edgeType":"PREDICATE"},{"sources":[5],"targets":[0,1],"expression":"(min(default.alltypesorc.cbigint) > 10)","edgeType":"PREDICATE"}],"vertices":[{"id":0,"vertexType":"COLUMN","vertexId":"default.dest_l1.a"},{"id":1,"vertexType":"COLUMN","vertexId":"default.dest_l1.b"},{"id":2,"vertexType":"COLUMN","vertexId":"default.alltypesorc.cint"},
      {"id":3,"vertexType":"COLUMN","vertexId":"default.alltypesorc.cstring1"},{"id":4,"vertexType":"COLUMN","vertexId":"default.alltypesorc.cboolean1"},{"id":5,"vertexType":"COLUMN","vertexId":"default.alltypesorc.cbigint"}]}
      PREHOOK: query: select cint, rank() over(order by cint) from alltypesorc
      where cint > 10 and cint < 10000 limit 10
      PREHOOK: type: QUERY
  • Sershe at Aug 25, 2015 at 9:28 pm
    HIVE-11538 : Add an option to skip init script while running tests (Ashutosh Chauhan via Sergey Shelukhin)


    Project: http://git-wip-us.apache.org/repos/asf/hive/repo
    Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/6e762919
    Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/6e762919
    Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/6e762919

    Branch: refs/heads/llap
    Commit: 6e7629193ed7b8714bd5ae5ab48bd10c2cbd85cf
    Parents: 433ea9c
    Author: Ashutosh Chauhan <hashutosh@apache.org>
    Authored: Thu Aug 13 20:42:33 2015 -0700
    Committer: Ashutosh Chauhan <hashutosh@apache.org>
    Committed: Thu Aug 13 20:43:46 2015 -0700

    ----------------------------------------------------------------------
      itests/qtest/pom.xml | 26 ++++----
      .../org/apache/hadoop/hive/ql/QTestUtil.java | 62 ++++++++++----------
      2 files changed, 44 insertions(+), 44 deletions(-)
    ----------------------------------------------------------------------


    http://git-wip-us.apache.org/repos/asf/hive/blob/6e762919/itests/qtest/pom.xml
    ----------------------------------------------------------------------
    diff --git a/itests/qtest/pom.xml b/itests/qtest/pom.xml
    index 44d30da..0588233 100644
    --- a/itests/qtest/pom.xml
    +++ b/itests/qtest/pom.xml
    @@ -30,7 +30,7 @@

        <properties>
          <hive.path.to.root>../..</hive.path.to.root>
    -
    + <initScript>q_test_init.sql</initScript>
          <qfile></qfile>
          <qfile_regex></qfile_regex>
          <run_disabled>false</run_disabled>
    @@ -420,7 +420,7 @@
                        logFile="${project.build.directory}/testparseneggen.log"
                        hadoopVersion="${active.hadoop.version}"
                        logDirectory="${project.build.directory}/qfile-results/negative/"
    - initScript="q_test_init.sql"
    + initScript="${initScript}"
                        cleanupScript="q_test_cleanup.sql"/>

                      <!-- Cli -->
    @@ -437,7 +437,7 @@
                        logFile="${project.build.directory}/testclidrivergen.log"
                        logDirectory="${project.build.directory}/qfile-results/clientpositive/"
                        hadoopVersion="${active.hadoop.version}"
    - initScript="q_test_init.sql"
    + initScript="${initScript}"
                        cleanupScript="q_test_cleanup.sql"/>

                      <!-- Negative Cli -->
    @@ -454,7 +454,7 @@
                        logFile="${project.build.directory}/testnegativeclidrivergen.log"
                        logDirectory="${project.build.directory}/qfile-results/clientnegative/"
                        hadoopVersion="${active.hadoop.version}"
    - initScript="q_test_init.sql"
    + initScript="${initScript}"
                        cleanupScript="q_test_cleanup.sql"/>

                      <!-- Compare Cli -->
    @@ -470,7 +470,7 @@
                        logFile="${project.build.directory}/testcompareclidrivergen.log"
                        logDirectory="${project.build.directory}/qfile-results/clientcompare/"
                        hadoopVersion="${active.hadoop.version}"
    - initScript="q_test_init.sql"
    + initScript="${initScript}"
                        cleanupScript="q_test_cleanup.sql"/>

                      <!-- Minimr -->
    @@ -487,7 +487,7 @@
                        logFile="${project.build.directory}/testminimrclidrivergen.log"
                        logDirectory="${project.build.directory}/qfile-results/clientpositive/"
                        hadoopVersion="${active.hadoop.version}"
    - initScript="q_test_init.sql"
    + initScript="${initScript}"
                        cleanupScript="q_test_cleanup.sql"/>

                      <if>
    @@ -508,7 +508,7 @@
                                    logFile="${project.build.directory}/testminitezclidrivergen.log"
                                    logDirectory="${project.build.directory}/qfile-results/clientpositive/"
                                    hadoopVersion="${active.hadoop.version}"
    - initScript="q_test_init.sql"
    + initScript="${initScript}"
                                    cleanupScript="q_test_cleanup.sql"/>

                          <qtestgen hiveRootDirectory="${basedir}/${hive.path.to.root}/"
    @@ -547,7 +547,7 @@
                        logFile="${project.build.directory}/testnegativeminimrclidrivergen.log"
                        logDirectory="${project.build.directory}/qfile-results/clientnegative/"
                        hadoopVersion="${hadoopVersion}"
    - initScript="q_test_init.sql"
    + initScript="${initScript}"
                        cleanupScript="q_test_cleanup.sql"/>

                      <!-- HBase Positive -->
    @@ -561,7 +561,7 @@
                        resultsDirectory="${basedir}/${hive.path.to.root}/hbase-handler/src/test/results/positive/" className="TestHBaseCliDriver"
                        logFile="${project.build.directory}/testhbaseclidrivergen.log"
                        logDirectory="${project.build.directory}/qfile-results/hbase-handler/positive/"
    - initScript="q_test_init.sql"
    + initScript="${initScript}"
                        cleanupScript="q_test_cleanup.sql"/>

                      <!-- HBase Minimr -->
    @@ -575,7 +575,7 @@
                        resultsDirectory="${basedir}/${hive.path.to.root}/hbase-handler/src/test/results/positive/" className="TestHBaseMinimrCliDriver"
                        logFile="${project.build.directory}/testhbaseminimrclidrivergen.log"
                        logDirectory="${project.build.directory}/qfile-results/hbase-handler/minimrpositive/"
    - initScript="q_test_init.sql"
    + initScript="${initScript}"
                        cleanupScript="q_test_cleanup.sql"/>

                      <!-- HBase Negative -->
    @@ -589,7 +589,7 @@
                        resultsDirectory="${basedir}/${hive.path.to.root}/hbase-handler/src/test/results/negative/" className="TestHBaseNegativeCliDriver"
                        logFile="${project.build.directory}/testhbasenegativeclidrivergen.log"
                        logDirectory="${project.build.directory}/qfile-results/hbase-handler/negative"
    - initScript="q_test_init.sql"
    + initScript="${initScript}"
                        cleanupScript="q_test_cleanup.sql"/>

                      <!-- Beeline -->
    @@ -626,7 +626,7 @@
                        logFile="${project.build.directory}/testcontribclidrivergen.log"
                        logDirectory="${project.build.directory}/qfile-results/contribclientpositive"
                        hadoopVersion="${hadoopVersion}"
    - initScript="q_test_init.sql"
    + initScript="${initScript}"
                        cleanupScript="q_test_cleanup.sql"/>

                      <qtestgen hiveRootDirectory="${basedir}/${hive.path.to.root}/"
    @@ -639,7 +639,7 @@
                        resultsDirectory="${basedir}/${hive.path.to.root}/contrib/src/test/results/clientnegative/" className="TestContribNegativeCliDriver"
                        logFile="${project.build.directory}/testcontribnegclidrivergen.log"
                        logDirectory="${project.build.directory}/qfile-results/contribclientnegative"
    - initScript="q_test_init.sql"
    + initScript="${initScript}"
                        cleanupScript="q_test_cleanup.sql"/>



    http://git-wip-us.apache.org/repos/asf/hive/blob/6e762919/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
    ----------------------------------------------------------------------
    diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
    index 39d5d9e..3fae0ba 100644
    --- a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
    +++ b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
    @@ -121,8 +121,8 @@ public class QTestUtil {

        private static final Log LOG = LogFactory.getLog("QTestUtil");
        private static final String QTEST_LEAVE_FILES = "QTEST_LEAVE_FILES";
    - private final String defaultInitScript = "q_test_init.sql";
    - private final String defaultCleanupScript = "q_test_cleanup.sql";
    + private final static String defaultInitScript = "q_test_init.sql";
    + private final static String defaultCleanupScript = "q_test_cleanup.sql";
        private final String[] testOnlyCommands = new String[]{"crypto"};

        private String testWarehouse;
    @@ -149,7 +149,6 @@ public class QTestUtil {
        private HadoopShims.MiniMrShim mr = null;
        private HadoopShims.MiniDFSShim dfs = null;
        private HadoopShims.HdfsEncryptionShim hes = null;
    - private boolean miniMr = false;
        private String hadoopVer = null;
        private QTestSetup setup = null;
        private SparkSession sparkSession = null;
    @@ -209,7 +208,7 @@ public class QTestUtil {
                continue;
              }

    - if (file.isDir()) {
    + if (file.isDirectory()) {
                if (!destFs.exists(local_path)) {
                  destFs.mkdirs(local_path);
                }
    @@ -410,14 +409,9 @@ public class QTestUtil {
          if (scriptsDir == null) {
            scriptsDir = new File(".").getAbsolutePath() + "/data/scripts";
          }
    - if (initScript.isEmpty()) {
    - initScript = defaultInitScript;
    - }
    - if (cleanupScript.isEmpty()) {
    - cleanupScript = defaultCleanupScript;
    - }
    - this.initScript = scriptsDir + "/" + initScript;
    - this.cleanupScript = scriptsDir + "/" + cleanupScript;
    +
    + this.initScript = scriptsDir + File.separator + initScript;
    + this.cleanupScript = scriptsDir + File.separator + cleanupScript;

          overWrite = "true".equalsIgnoreCase(System.getProperty("test.output.overwrite"));

    @@ -705,7 +699,7 @@ public class QTestUtil {
            FileSystem fileSystem = p.getFileSystem(conf);
            if (fileSystem.exists(p)) {
              for (FileStatus status : fileSystem.listStatus(p)) {
    - if (status.isDir() && !srcTables.contains(status.getPath().getName())) {
    + if (status.isDirectory() && !srcTables.contains(status.getPath().getName())) {
                  fileSystem.delete(status.getPath(), true);
                }
              }
    @@ -755,16 +749,19 @@ public class QTestUtil {
          clearTablesCreatedDuringTests();
          clearKeysCreatedInTests();

    - SessionState.get().getConf().setBoolean("hive.test.shutdown.phase", true);
    -
    - String cleanupCommands = readEntireFileIntoString(new File(cleanupScript));
    - LOG.info("Cleanup (" + cleanupScript + "):\n" + cleanupCommands);
    - if(cliDriver == null) {
    - cliDriver = new CliDriver();
    + File cleanupFile = new File(cleanupScript);
    + if (cleanupFile.isFile()) {
    + String cleanupCommands = readEntireFileIntoString(cleanupFile);
    + LOG.info("Cleanup (" + cleanupScript + "):\n" + cleanupCommands);
    + if(cliDriver == null) {
    + cliDriver = new CliDriver();
    + }
    + SessionState.get().getConf().setBoolean("hive.test.shutdown.phase", true);
    + cliDriver.processLine(cleanupCommands);
    + SessionState.get().getConf().setBoolean("hive.test.shutdown.phase", false);
    + } else {
    + LOG.info("No cleanup script detected. Skipping.");
          }
    - cliDriver.processLine(cleanupCommands);
    -
    - SessionState.get().getConf().setBoolean("hive.test.shutdown.phase", false);

          // delete any contents in the warehouse dir
          Path p = new Path(testWarehouse);
    @@ -809,14 +806,21 @@ public class QTestUtil {
          if(!isSessionStateStarted) {
            startSessionState();
          }
    - conf.setBoolean("hive.test.init.phase", true);

    - String initCommands = readEntireFileIntoString(new File(this.initScript));
    - LOG.info("Initial setup (" + initScript + "):\n" + initCommands);
          if(cliDriver == null) {
            cliDriver = new CliDriver();
          }
          cliDriver.processLine("set test.data.dir=" + testFiles + ";");
    + File scriptFile = new File(this.initScript);
    + if (!scriptFile.isFile()) {
    + LOG.info("No init script detected. Skipping");
    + return;
    + }
    + conf.setBoolean("hive.test.init.phase", true);
    +
    + String initCommands = readEntireFileIntoString(scriptFile);
    + LOG.info("Initial setup (" + initScript + "):\n" + initCommands);
    +
          cliDriver.processLine(initCommands);

          conf.setBoolean("hive.test.init.phase", false);
    @@ -912,6 +916,7 @@ public class QTestUtil {

        private CliSessionState createSessionState() {
         return new CliSessionState(conf) {
    + @Override
            public void setSparkSession(SparkSession sparkSession) {
              super.setSparkSession(sparkSession);
              if (sparkSession != null) {
    @@ -1136,11 +1141,6 @@ public class QTestUtil {
          return commands;
        }

    - private boolean isComment(final String line) {
    - String lineTrimmed = line.trim();
    - return lineTrimmed.startsWith("#") || lineTrimmed.startsWith("--");
    - }
    -
        public boolean shouldBeSkipped(String tname) {
          return qSkipSet.contains(tname);
        }
    @@ -1816,7 +1816,7 @@ public class QTestUtil {
        {
          QTestUtil[] qt = new QTestUtil[qfiles.length];
          for (int i = 0; i < qfiles.length; i++) {
    - qt[i] = new QTestUtil(resDir, logDir, MiniClusterType.none, null, "0.20", "", "");
    + qt[i] = new QTestUtil(resDir, logDir, MiniClusterType.none, null, "0.20", defaultInitScript, defaultCleanupScript);
            qt[i].addFile(qfiles[i]);
            qt[i].clearTestSideEffects();
          }
  • Sershe at Aug 25, 2015 at 9:28 pm
    HIVE-11534: Improve validateTableCols error message (Mohit via Xuefu)


    Project: http://git-wip-us.apache.org/repos/asf/hive/repo
    Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/a4849cb2
    Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/a4849cb2
    Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/a4849cb2

    Branch: refs/heads/llap
    Commit: a4849cb2c5b7d1c542881e12bf0e48f73917d96c
    Parents: b8f1ae1
    Author: Xuefu Zhang <xzhang@cloudera.com>
    Authored: Fri Aug 14 06:48:19 2015 -0700
    Committer: Xuefu Zhang <xzhang@cloudera.com>
    Committed: Fri Aug 14 06:48:19 2015 -0700

    ----------------------------------------------------------------------
      .../hive/metastore/TestHiveMetaStore.java | 66 ++++++++++++++++++++
      .../hadoop/hive/metastore/ObjectStore.java | 7 ++-
      2 files changed, 71 insertions(+), 2 deletions(-)
    ----------------------------------------------------------------------


    http://git-wip-us.apache.org/repos/asf/hive/blob/a4849cb2/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java
    ----------------------------------------------------------------------
    diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java
    index 5a344bb..160667d 100644
    --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java
    +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java
    @@ -2966,4 +2966,70 @@ public abstract class TestHiveMetaStore extends TestCase {
          };
          return hookLoader;
        }
    +
    + public void testValidateTableCols() throws Throwable {
    +
    + try {
    + String dbName = "compdb";
    + String tblName = "comptbl";
    +
    + client.dropTable(dbName, tblName);
    + silentDropDatabase(dbName);
    + Database db = new Database();
    + db.setName(dbName);
    + db.setDescription("Validate Table Columns test");
    + client.createDatabase(db);
    +
    + ArrayList<FieldSchema> cols = new ArrayList<FieldSchema>(2);
    + cols.add(new FieldSchema("name", serdeConstants.STRING_TYPE_NAME, ""));
    + cols.add(new FieldSchema("income", serdeConstants.INT_TYPE_NAME, ""));
    +
    + Table tbl = new Table();
    + tbl.setDbName(dbName);
    + tbl.setTableName(tblName);
    + StorageDescriptor sd = new StorageDescriptor();
    + tbl.setSd(sd);
    + sd.setCols(cols);
    + sd.setCompressed(false);
    + sd.setSerdeInfo(new SerDeInfo());
    + sd.getSerdeInfo().setName(tbl.getTableName());
    + sd.getSerdeInfo().setParameters(new HashMap<String, String>());
    + sd.getSerdeInfo().getParameters()
    + .put(serdeConstants.SERIALIZATION_FORMAT, "1");
    + sd.getSerdeInfo().setSerializationLib(LazySimpleSerDe.class.getName());
    + sd.setInputFormat(HiveInputFormat.class.getName());
    + sd.setOutputFormat(HiveOutputFormat.class.getName());
    + sd.setSortCols(new ArrayList<Order>());
    +
    + client.createTable(tbl);
    + if (isThriftClient) {
    + tbl = client.getTable(dbName, tblName);
    + }
    +
    + List<String> expectedCols = Lists.newArrayList();
    + expectedCols.add("name");
    + ObjectStore objStore = new ObjectStore();
    + try {
    + objStore.validateTableCols(tbl, expectedCols);
    + } catch (MetaException ex) {
    + throw new RuntimeException(ex);
    + }
    +
    + expectedCols.add("doesntExist");
    + boolean exceptionFound = false;
    + try {
    + objStore.validateTableCols(tbl, expectedCols);
    + } catch (MetaException ex) {
    + assertEquals(ex.getMessage(),
    + "Column doesntExist doesn't exist in table comptbl in database compdb");
    + exceptionFound = true;
    + }
    + assertTrue(exceptionFound);
    +
    + } catch (Exception e) {
    + System.err.println(StringUtils.stringifyException(e));
    + System.err.println("testValidateTableCols() failed.");
    + throw e;
    + }
    + }
      }

    http://git-wip-us.apache.org/repos/asf/hive/blob/a4849cb2/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java
    ----------------------------------------------------------------------
    diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java b/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java
    index a37fbde..d165fc8 100644
    --- a/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java
    +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java
    @@ -51,6 +51,7 @@ import javax.jdo.Transaction;
      import javax.jdo.datastore.DataStoreCache;
      import javax.jdo.identity.IntIdentity;

    +import com.google.common.annotations.VisibleForTesting;
      import org.antlr.runtime.CommonTokenStream;
      import org.antlr.runtime.RecognitionException;
      import org.apache.commons.logging.Log;
    @@ -6261,7 +6262,8 @@ public class ObjectStore implements RawStore, Configurable {
          }
        }

    - private void validateTableCols(Table table, List<String> colNames) throws MetaException {
    + @VisibleForTesting
    + public void validateTableCols(Table table, List<String> colNames) throws MetaException {
          List<FieldSchema> colList = table.getSd().getCols();
          for (String colName : colNames) {
            boolean foundCol = false;
    @@ -6272,7 +6274,8 @@ public class ObjectStore implements RawStore, Configurable {
              }
            }
            if (!foundCol) {
    - throw new MetaException("Column " + colName + " doesn't exist.");
    + throw new MetaException("Column " + colName + " doesn't exist in table "
    + + table.getTableName() + " in database " + table.getDbName());
            }
          }
        }
  • Sershe at Aug 25, 2015 at 9:28 pm
    http://git-wip-us.apache.org/repos/asf/hive/blob/c93d6c77/itests/pom.xml
    ----------------------------------------------------------------------
    diff --git a/itests/pom.xml b/itests/pom.xml
    index f156cc4..acce713 100644
    --- a/itests/pom.xml
    +++ b/itests/pom.xml
    @@ -94,7 +94,7 @@
                            }
                            mkdir -p $DOWNLOAD_DIR
                            download "http://d3jw87u4immizc.cloudfront.net/spark-tarball/spark-${spark.version}-bin-hadoop2-without-hive.tgz" "spark"
    - cp -f $HIVE_ROOT/data/conf/spark/log4j.properties $BASE_DIR/spark/conf/
    + cp -f $HIVE_ROOT/data/conf/spark/log4j2.xml $BASE_DIR/spark/conf/
                            sed '/package /d' ${basedir}/${hive.path.to.root}/contrib/src/java/org/apache/hadoop/hive/contrib/udf/example/UDFExampleAdd.java > /tmp/UDFExampleAdd.java
                            javac -cp ${settings.localRepository}/org/apache/hive/hive-exec/${project.version}/hive-exec-${project.version}.jar /tmp/UDFExampleAdd.java -d /tmp
                            jar -cf /tmp/udfexampleadd-1.0.jar -C /tmp UDFExampleAdd.class

    http://git-wip-us.apache.org/repos/asf/hive/blob/c93d6c77/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
    ----------------------------------------------------------------------
    diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
    index b33cb58..65117c4 100644
    --- a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
    +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
    @@ -5931,7 +5931,7 @@ public class HiveMetaStore extends ThriftHiveMetastore {

          // If the log4j.configuration property hasn't already been explicitly set,
          // use Hive's default log4j configuration
    - if (System.getProperty("log4j.configuration") == null) {
    + if (System.getProperty("log4j.configurationFile") == null) {
            // NOTE: It is critical to do this here so that log4j is reinitialized
            // before any of the other core hive classes are loaded
            try {

    http://git-wip-us.apache.org/repos/asf/hive/blob/c93d6c77/metastore/src/test/org/apache/hadoop/hive/metastore/txn/TestCompactionTxnHandler.java
    ----------------------------------------------------------------------
    diff --git a/metastore/src/test/org/apache/hadoop/hive/metastore/txn/TestCompactionTxnHandler.java b/metastore/src/test/org/apache/hadoop/hive/metastore/txn/TestCompactionTxnHandler.java
    index ad99427..df42f1a 100644
    --- a/metastore/src/test/org/apache/hadoop/hive/metastore/txn/TestCompactionTxnHandler.java
    +++ b/metastore/src/test/org/apache/hadoop/hive/metastore/txn/TestCompactionTxnHandler.java
    @@ -17,15 +17,11 @@
       */
      package org.apache.hadoop.hive.metastore.txn;

    -import org.apache.commons.logging.Log;
    -import org.apache.commons.logging.LogFactory;
    -import org.apache.hadoop.hive.conf.HiveConf;
    -import org.apache.hadoop.hive.metastore.api.*;
    -import org.apache.log4j.Level;
    -import org.apache.log4j.LogManager;
    -import org.junit.After;
    -import org.junit.Before;
    -import org.junit.Test;
    +import static junit.framework.Assert.assertEquals;
    +import static junit.framework.Assert.assertNotNull;
    +import static junit.framework.Assert.assertNull;
    +import static junit.framework.Assert.assertTrue;
    +import static junit.framework.Assert.fail;

      import java.util.ArrayList;
      import java.util.Arrays;
    @@ -34,7 +30,29 @@ import java.util.Set;
      import java.util.SortedSet;
      import java.util.TreeSet;

    -import static junit.framework.Assert.*;
    +import org.apache.hadoop.hive.conf.HiveConf;
    +import org.apache.hadoop.hive.metastore.api.AbortTxnRequest;
    +import org.apache.hadoop.hive.metastore.api.AddDynamicPartitions;
    +import org.apache.hadoop.hive.metastore.api.CommitTxnRequest;
    +import org.apache.hadoop.hive.metastore.api.CompactionRequest;
    +import org.apache.hadoop.hive.metastore.api.CompactionType;
    +import org.apache.hadoop.hive.metastore.api.GetOpenTxnsResponse;
    +import org.apache.hadoop.hive.metastore.api.LockComponent;
    +import org.apache.hadoop.hive.metastore.api.LockLevel;
    +import org.apache.hadoop.hive.metastore.api.LockRequest;
    +import org.apache.hadoop.hive.metastore.api.LockResponse;
    +import org.apache.hadoop.hive.metastore.api.LockState;
    +import org.apache.hadoop.hive.metastore.api.LockType;
    +import org.apache.hadoop.hive.metastore.api.MetaException;
    +import org.apache.hadoop.hive.metastore.api.OpenTxnRequest;
    +import org.apache.hadoop.hive.metastore.api.OpenTxnsResponse;
    +import org.apache.hadoop.hive.metastore.api.ShowCompactRequest;
    +import org.apache.hadoop.hive.metastore.api.ShowCompactResponse;
    +import org.apache.hadoop.hive.metastore.api.ShowCompactResponseElement;
    +import org.apache.hadoop.hive.metastore.api.UnlockRequest;
    +import org.junit.After;
    +import org.junit.Before;
    +import org.junit.Test;

      /**
       * Tests for TxnHandler.
    @@ -43,11 +61,9 @@ public class TestCompactionTxnHandler {

        private HiveConf conf = new HiveConf();
        private CompactionTxnHandler txnHandler;
    - static final private Log LOG = LogFactory.getLog(TestCompactionTxnHandler.class);

        public TestCompactionTxnHandler() throws Exception {
          TxnDbUtil.setConfValues(conf);
    - LogManager.getLogger(TxnHandler.class.getName()).setLevel(Level.DEBUG);
          tearDown();
        }


    http://git-wip-us.apache.org/repos/asf/hive/blob/c93d6c77/metastore/src/test/org/apache/hadoop/hive/metastore/txn/TestTxnHandler.java
    ----------------------------------------------------------------------
    diff --git a/metastore/src/test/org/apache/hadoop/hive/metastore/txn/TestTxnHandler.java b/metastore/src/test/org/apache/hadoop/hive/metastore/txn/TestTxnHandler.java
    index f478184..6dc0bd3 100644
    --- a/metastore/src/test/org/apache/hadoop/hive/metastore/txn/TestTxnHandler.java
    +++ b/metastore/src/test/org/apache/hadoop/hive/metastore/txn/TestTxnHandler.java
    @@ -17,16 +17,11 @@
       */
      package org.apache.hadoop.hive.metastore.txn;

    -import org.apache.commons.logging.Log;
    -import org.apache.commons.logging.LogFactory;
    -import org.apache.hadoop.hive.conf.HiveConf;
    -import org.apache.hadoop.hive.metastore.api.*;
    -import org.apache.log4j.Level;
    -import org.apache.log4j.LogManager;
    -import org.junit.After;
    -import org.junit.Before;
    -import org.junit.Ignore;
    -import org.junit.Test;
    +import static junit.framework.Assert.assertEquals;
    +import static junit.framework.Assert.assertFalse;
    +import static junit.framework.Assert.assertNull;
    +import static junit.framework.Assert.assertTrue;
    +import static junit.framework.Assert.fail;

      import java.sql.Connection;
      import java.sql.SQLException;
    @@ -36,21 +31,66 @@ import java.util.List;
      import java.util.concurrent.TimeUnit;
      import java.util.concurrent.atomic.AtomicBoolean;

    -import static junit.framework.Assert.*;
    +import org.apache.hadoop.hive.conf.HiveConf;
    +import org.apache.hadoop.hive.metastore.api.AbortTxnRequest;
    +import org.apache.hadoop.hive.metastore.api.CheckLockRequest;
    +import org.apache.hadoop.hive.metastore.api.CommitTxnRequest;
    +import org.apache.hadoop.hive.metastore.api.CompactionRequest;
    +import org.apache.hadoop.hive.metastore.api.CompactionType;
    +import org.apache.hadoop.hive.metastore.api.GetOpenTxnsInfoResponse;
    +import org.apache.hadoop.hive.metastore.api.GetOpenTxnsResponse;
    +import org.apache.hadoop.hive.metastore.api.HeartbeatRequest;
    +import org.apache.hadoop.hive.metastore.api.HeartbeatTxnRangeRequest;
    +import org.apache.hadoop.hive.metastore.api.HeartbeatTxnRangeResponse;
    +import org.apache.hadoop.hive.metastore.api.LockComponent;
    +import org.apache.hadoop.hive.metastore.api.LockLevel;
    +import org.apache.hadoop.hive.metastore.api.LockRequest;
    +import org.apache.hadoop.hive.metastore.api.LockResponse;
    +import org.apache.hadoop.hive.metastore.api.LockState;
    +import org.apache.hadoop.hive.metastore.api.LockType;
    +import org.apache.hadoop.hive.metastore.api.MetaException;
    +import org.apache.hadoop.hive.metastore.api.NoSuchLockException;
    +import org.apache.hadoop.hive.metastore.api.NoSuchTxnException;
    +import org.apache.hadoop.hive.metastore.api.OpenTxnRequest;
    +import org.apache.hadoop.hive.metastore.api.OpenTxnsResponse;
    +import org.apache.hadoop.hive.metastore.api.ShowCompactRequest;
    +import org.apache.hadoop.hive.metastore.api.ShowCompactResponse;
    +import org.apache.hadoop.hive.metastore.api.ShowCompactResponseElement;
    +import org.apache.hadoop.hive.metastore.api.ShowLocksRequest;
    +import org.apache.hadoop.hive.metastore.api.ShowLocksResponse;
    +import org.apache.hadoop.hive.metastore.api.ShowLocksResponseElement;
    +import org.apache.hadoop.hive.metastore.api.TxnAbortedException;
    +import org.apache.hadoop.hive.metastore.api.TxnInfo;
    +import org.apache.hadoop.hive.metastore.api.TxnOpenException;
    +import org.apache.hadoop.hive.metastore.api.TxnState;
    +import org.apache.hadoop.hive.metastore.api.UnlockRequest;
    +import org.apache.logging.log4j.Level;
    +import org.apache.logging.log4j.LogManager;
    +import org.apache.logging.log4j.core.LoggerContext;
    +import org.apache.logging.log4j.core.config.Configuration;
    +import org.junit.After;
    +import org.junit.Before;
    +import org.junit.Ignore;
    +import org.junit.Test;
    +import org.slf4j.Logger;
    +import org.slf4j.LoggerFactory;

      /**
       * Tests for TxnHandler.
       */
      public class TestTxnHandler {
        static final private String CLASS_NAME = TxnHandler.class.getName();
    - static final private Log LOG = LogFactory.getLog(CLASS_NAME);
    + private static final Logger LOG = LoggerFactory.getLogger(CLASS_NAME);

        private HiveConf conf = new HiveConf();
        private TxnHandler txnHandler;

        public TestTxnHandler() throws Exception {
          TxnDbUtil.setConfValues(conf);
    - LogManager.getLogger(TxnHandler.class.getName()).setLevel(Level.DEBUG);
    + LoggerContext ctx = (LoggerContext) LogManager.getContext(false);
    + Configuration conf = ctx.getConfiguration();
    + conf.getLoggerConfig(CLASS_NAME).setLevel(Level.DEBUG);
    + ctx.updateLoggers(conf);
          tearDown();
        }


    http://git-wip-us.apache.org/repos/asf/hive/blob/c93d6c77/packaging/src/main/assembly/bin.xml
    ----------------------------------------------------------------------
    diff --git a/packaging/src/main/assembly/bin.xml b/packaging/src/main/assembly/bin.xml
    index 63253c5..0fa6af8 100644
    --- a/packaging/src/main/assembly/bin.xml
    +++ b/packaging/src/main/assembly/bin.xml
    @@ -275,7 +275,7 @@
            <fileMode>644</fileMode>
            <includes>
              <include>webhcat-default.xml</include>
    - <include>webhcat-log4j.properties</include>
    + <include>webhcat-log4j2.xml</include>
            </includes>
            <filtered>true</filtered>
            <outputDirectory>hcatalog/etc/webhcat</outputDirectory>
    @@ -323,19 +323,19 @@

        <files>
          <file>
    - <source>${project.parent.basedir}/common/src/main/resources/hive-log4j.properties</source>
    + <source>${project.parent.basedir}/common/src/main/resources/hive-log4j2.xml</source>
            <outputDirectory>conf</outputDirectory>
    - <destName>hive-log4j.properties.template</destName>
    + <destName>hive-log4j2.xml.template</destName>
          </file>
          <file>
    - <source>${project.parent.basedir}/ql/src/main/resources/hive-exec-log4j.properties</source>
    + <source>${project.parent.basedir}/ql/src/main/resources/hive-exec-log4j2.xml</source>
            <outputDirectory>conf</outputDirectory>
    - <destName>hive-exec-log4j.properties.template</destName>
    + <destName>hive-exec-log4j2.xml.template</destName>
          </file>
          <file>
    - <source>${project.parent.basedir}/beeline/src/main/resources/beeline-log4j.properties</source>
    + <source>${project.parent.basedir}/beeline/src/main/resources/beeline-log4j2.xml</source>
            <outputDirectory>conf</outputDirectory>
    - <destName>beeline-log4j.properties.template</destName>
    + <destName>beeline-log4j2.xml.template</destName>
          </file>
          <file>
            <source>${project.parent.basedir}/hcatalog/README.txt</source>

    http://git-wip-us.apache.org/repos/asf/hive/blob/c93d6c77/pom.xml
    ----------------------------------------------------------------------
    diff --git a/pom.xml b/pom.xml
    index 0383e01..15c2805 100644
    --- a/pom.xml
    +++ b/pom.xml
    @@ -148,8 +148,7 @@
          <kryo.version>2.22</kryo.version>
          <libfb303.version>0.9.2</libfb303.version>
          <libthrift.version>0.9.2</libthrift.version>
    - <log4j.version>1.2.16</log4j.version>
    - <log4j-extras.version>1.2.17</log4j-extras.version>
    + <log4j2.version>2.3</log4j2.version>
          <opencsv.version>2.3</opencsv.version>
          <mockito-all.version>1.9.5</mockito-all.version>
          <mina.version>2.0.0-M5</mina.version>
    @@ -366,14 +365,24 @@
              <version>${junit.version}</version>
            </dependency>
            <dependency>
    - <groupId>log4j</groupId>
    - <artifactId>log4j</artifactId>
    - <version>${log4j.version}</version>
    + <groupId>org.apache.logging.log4j</groupId>
    + <artifactId>log4j-1.2-api</artifactId>
    + <version>${log4j2.version}</version>
            </dependency>
            <dependency>
    - <groupId>log4j</groupId>
    - <artifactId>apache-log4j-extras</artifactId>
    - <version>${log4j-extras.version}</version>
    + <groupId>org.apache.logging.log4j</groupId>
    + <artifactId>log4j-web</artifactId>
    + <version>${log4j2.version}</version>
    + </dependency>
    + <dependency>
    + <groupId>org.apache.logging.log4j</groupId>
    + <artifactId>log4j-slf4j-impl</artifactId>
    + <version>${log4j2.version}</version>
    + </dependency>
    + <dependency>
    + <groupId>org.apache.logging.log4j</groupId>
    + <artifactId>log4j-jcl</artifactId>
    + <version>${log4j2.version}</version>
            </dependency>
            <dependency>
              <groupId>org.antlr</groupId>
    @@ -584,11 +593,6 @@
              <version>${slf4j.version}</version>
            </dependency>
            <dependency>
    - <groupId>org.slf4j</groupId>
    - <artifactId>slf4j-log4j12</artifactId>
    - <version>${slf4j.version}</version>
    - </dependency>
    - <dependency>
              <groupId>xerces</groupId>
              <artifactId>xercesImpl</artifactId>
              <version>${xerces.version}</version>
    @@ -604,11 +608,6 @@
            <artifactId>slf4j-api</artifactId>
            <version>${slf4j.version}</version>
          </dependency>
    - <dependency>
    - <groupId>org.slf4j</groupId>
    - <artifactId>slf4j-log4j12</artifactId>
    - <version>${slf4j.version}</version>
    - </dependency>
        </dependencies>

        <build>
    @@ -872,7 +871,7 @@
                  <!-- required for hive-exec jar path and tests which reference a jar -->
                  <maven.local.repository>${maven.repo.local}</maven.local.repository>
                  <mapred.job.tracker>local</mapred.job.tracker>
    - <log4j.configuration>${test.log4j.scheme}${test.tmp.dir}/conf/hive-log4j.properties</log4j.configuration>
    + <log4j.configurationFile>${test.log4j.scheme}${test.tmp.dir}/conf/hive-log4j2.xml</log4j.configurationFile>
                  <log4j.debug>true</log4j.debug>
                  <!-- don't diry up /tmp -->
                  <java.io.tmpdir>${test.tmp.dir}</java.io.tmpdir>

    http://git-wip-us.apache.org/repos/asf/hive/blob/c93d6c77/ql/pom.xml
    ----------------------------------------------------------------------
    diff --git a/ql/pom.xml b/ql/pom.xml
    index e7a8e7b..36b3433 100644
    --- a/ql/pom.xml
    +++ b/ql/pom.xml
    @@ -112,14 +112,19 @@
            <version>${javolution.version}</version>
          </dependency>
          <dependency>
    - <groupId>log4j</groupId>
    - <artifactId>log4j</artifactId>
    - <version>${log4j.version}</version>
    + <groupId>org.apache.logging.log4j</groupId>
    + <artifactId>log4j-1.2-api</artifactId>
    + <version>${log4j2.version}</version>
          </dependency>
          <dependency>
    - <groupId>log4j</groupId>
    - <artifactId>apache-log4j-extras</artifactId>
    - <version>${log4j-extras.version}</version>
    + <groupId>org.apache.logging.log4j</groupId>
    + <artifactId>log4j-slf4j-impl</artifactId>
    + <version>${log4j2.version}</version>
    + </dependency>
    + <dependency>
    + <groupId>org.apache.logging.log4j</groupId>
    + <artifactId>log4j-jcl</artifactId>
    + <version>${log4j2.version}</version>
          </dependency>
          <dependency>
            <groupId>org.antlr</groupId>

    http://git-wip-us.apache.org/repos/asf/hive/blob/c93d6c77/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecDriver.java
    ----------------------------------------------------------------------
    diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecDriver.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecDriver.java
    index a2cf712..82345ee 100644
    --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecDriver.java
    +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecDriver.java
    @@ -27,7 +27,6 @@ import java.lang.management.MemoryMXBean;
      import java.util.ArrayList;
      import java.util.Collection;
      import java.util.Collections;
    -import java.util.Enumeration;
      import java.util.List;
      import java.util.Properties;

    @@ -57,13 +56,14 @@ import org.apache.hadoop.hive.ql.exec.PartitionKeySampler;
      import org.apache.hadoop.hive.ql.exec.TableScanOperator;
      import org.apache.hadoop.hive.ql.exec.Task;
      import org.apache.hadoop.hive.ql.exec.Utilities;
    -import org.apache.hadoop.hive.ql.exec.tez.TezSessionState;
      import org.apache.hadoop.hive.ql.exec.tez.TezSessionPoolManager;
    +import org.apache.hadoop.hive.ql.exec.tez.TezSessionState;
      import org.apache.hadoop.hive.ql.io.BucketizedHiveInputFormat;
      import org.apache.hadoop.hive.ql.io.HiveFileFormatUtils;
      import org.apache.hadoop.hive.ql.io.HiveKey;
      import org.apache.hadoop.hive.ql.io.HiveOutputFormatImpl;
      import org.apache.hadoop.hive.ql.io.IOPrepareCache;
    +import org.apache.hadoop.hive.ql.log.NullAppender;
      import org.apache.hadoop.hive.ql.metadata.HiveException;
      import org.apache.hadoop.hive.ql.plan.FetchWork;
      import org.apache.hadoop.hive.ql.plan.MapWork;
    @@ -88,11 +88,12 @@ import org.apache.hadoop.mapred.JobConf;
      import org.apache.hadoop.mapred.Partitioner;
      import org.apache.hadoop.mapred.RunningJob;
      import org.apache.hadoop.security.UserGroupInformation;
    -import org.apache.log4j.Appender;
    -import org.apache.log4j.BasicConfigurator;
    -import org.apache.log4j.FileAppender;
    -import org.apache.log4j.LogManager;
    -import org.apache.log4j.varia.NullAppender;
    +import org.apache.logging.log4j.Level;
    +import org.apache.logging.log4j.LogManager;
    +import org.apache.logging.log4j.Logger;
    +import org.apache.logging.log4j.core.Appender;
    +import org.apache.logging.log4j.core.appender.FileAppender;
    +import org.apache.logging.log4j.core.appender.RollingFileAppender;

      /**
       * ExecDriver is the central class in co-ordinating execution of any map-reduce task.
    @@ -687,8 +688,10 @@ public class ExecDriver extends Task<MapredWork> implements Serializable, Hadoop
          if (noLog) {
            // If started from main(), and noLog is on, we should not output
            // any logs. To turn the log on, please set -Dtest.silent=false
    - BasicConfigurator.resetConfiguration();
    - BasicConfigurator.configure(new NullAppender());
    + Logger logger = org.apache.logging.log4j.LogManager.getRootLogger();
    + NullAppender appender = NullAppender.createNullAppender();
    + appender.addToLogger(logger.getName(), Level.ERROR);
    + appender.start();
          } else {
            setupChildLog4j(conf);
          }
    @@ -703,10 +706,12 @@ public class ExecDriver extends Task<MapredWork> implements Serializable, Hadoop

          // print out the location of the log file for the user so
          // that it's easy to find reason for local mode execution failures
    - for (Appender appender : Collections.list((Enumeration<Appender>) LogManager.getRootLogger()
    - .getAllAppenders())) {
    + for (Appender appender : ((org.apache.logging.log4j.core.Logger) LogManager.getRootLogger())
    + .getAppenders().values()) {
            if (appender instanceof FileAppender) {
    - console.printInfo("Execution log at: " + ((FileAppender) appender).getFile());
    + console.printInfo("Execution log at: " + ((FileAppender) appender).getFileName());
    + } else if (appender instanceof RollingFileAppender) {
    + console.printInfo("Execution log at: " + ((RollingFileAppender) appender).getFileName());
            }
          }


    http://git-wip-us.apache.org/repos/asf/hive/blob/c93d6c77/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/HadoopJobExecHelper.java
    ----------------------------------------------------------------------
    diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/HadoopJobExecHelper.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/HadoopJobExecHelper.java
    index 6a6593c..44dfe3e 100644
    --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/HadoopJobExecHelper.java
    +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/HadoopJobExecHelper.java
    @@ -24,7 +24,6 @@ import java.text.SimpleDateFormat;
      import java.util.ArrayList;
      import java.util.Calendar;
      import java.util.Collections;
    -import java.util.Enumeration;
      import java.util.HashMap;
      import java.util.LinkedList;
      import java.util.List;
    @@ -33,8 +32,6 @@ import java.util.concurrent.TimeUnit;

      import org.apache.commons.logging.Log;
      import org.apache.commons.logging.LogFactory;
    -import org.apache.hadoop.fs.Path;
    -import org.apache.hadoop.hive.common.JavaUtils;
      import org.apache.hadoop.hive.conf.HiveConf;
      import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
      import org.apache.hadoop.hive.ql.MapRedStats;
    @@ -59,9 +56,11 @@ import org.apache.hadoop.mapred.JobStatus;
      import org.apache.hadoop.mapred.RunningJob;
      import org.apache.hadoop.mapred.TaskCompletionEvent;
      import org.apache.hadoop.mapred.TaskReport;
    -import org.apache.log4j.Appender;
    -import org.apache.log4j.FileAppender;
    -import org.apache.log4j.LogManager;
    +import org.apache.logging.log4j.LogManager;
    +import org.apache.logging.log4j.core.Appender;
    +import org.apache.logging.log4j.core.Logger;
    +import org.apache.logging.log4j.core.appender.FileAppender;
    +import org.apache.logging.log4j.core.appender.RollingFileAppender;

      public class HadoopJobExecHelper {

    @@ -492,10 +491,11 @@ public class HadoopJobExecHelper {
          sb.append("Logs:\n");
          console.printError(sb.toString());

    - for (Appender a : Collections.list((Enumeration<Appender>)
    - LogManager.getRootLogger().getAllAppenders())) {
    - if (a instanceof FileAppender) {
    - console.printError((new Path(((FileAppender)a).getFile())).toUri().getPath());
    + for (Appender appender : ((Logger) LogManager.getRootLogger()).getAppenders().values()) {
    + if (appender instanceof FileAppender) {
    + console.printError(((FileAppender) appender).getFileName());
    + } else if (appender instanceof RollingFileAppender) {
    + console.printError(((RollingFileAppender) appender).getFileName());
            }
          }
        }

    http://git-wip-us.apache.org/repos/asf/hive/blob/c93d6c77/ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/stats/PartialScanTask.java
    ----------------------------------------------------------------------
    diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/stats/PartialScanTask.java b/ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/stats/PartialScanTask.java
    index 3cb9e9c..cee0878 100644
    --- a/ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/stats/PartialScanTask.java
    +++ b/ql/src/java/org/apache/hadoop/hive/ql/io/rcfile/stats/PartialScanTask.java
    @@ -21,8 +21,6 @@ package org.apache.hadoop.hive.ql.io.rcfile.stats;
      import java.io.IOException;
      import java.io.Serializable;
      import java.util.ArrayList;
    -import java.util.Collections;
    -import java.util.Enumeration;
      import java.util.List;

      import org.apache.commons.lang.StringUtils;
    @@ -59,9 +57,11 @@ import org.apache.hadoop.mapred.InputFormat;
      import org.apache.hadoop.mapred.JobClient;
      import org.apache.hadoop.mapred.JobConf;
      import org.apache.hadoop.mapred.RunningJob;
    -import org.apache.log4j.Appender;
    -import org.apache.log4j.FileAppender;
    -import org.apache.log4j.LogManager;
    +import org.apache.logging.log4j.LogManager;
    +import org.apache.logging.log4j.core.Appender;
    +import org.apache.logging.log4j.core.Logger;
    +import org.apache.logging.log4j.core.appender.FileAppender;
    +import org.apache.logging.log4j.core.appender.RollingFileAppender;

      /**
       * PartialScanTask.
    @@ -335,15 +335,15 @@ public class PartialScanTask extends Task<PartialScanWork> implements

          // print out the location of the log file for the user so
          // that it's easy to find reason for local mode execution failures
    - for (Appender appender : Collections
    - .list((Enumeration<Appender>) LogManager.getRootLogger()
    - .getAllAppenders())) {
    + for (Appender appender : ((Logger) LogManager.getRootLogger()).getAppenders().values()) {
            if (appender instanceof FileAppender) {
    - console.printInfo("Execution log at: "
    - + ((FileAppender) appender).getFile());
    + console.printInfo("Execution log at: " + ((FileAppender) appender).getFileName());
    + } else if (appender instanceof RollingFileAppender) {
    + console.printInfo("Execution log at: " + ((RollingFileAppender) appender).getFileName());
            }
          }

    +
          PartialScanWork mergeWork = new PartialScanWork(inputPaths);
          DriverContext driverCxt = new DriverContext();
          PartialScanTask taskExec = new PartialScanTask();

    http://git-wip-us.apache.org/repos/asf/hive/blob/c93d6c77/ql/src/java/org/apache/hadoop/hive/ql/log/HiveEventCounter.java
    ----------------------------------------------------------------------
    diff --git a/ql/src/java/org/apache/hadoop/hive/ql/log/HiveEventCounter.java b/ql/src/java/org/apache/hadoop/hive/ql/log/HiveEventCounter.java
    new file mode 100644
    index 0000000..46662c4
    --- /dev/null
    +++ b/ql/src/java/org/apache/hadoop/hive/ql/log/HiveEventCounter.java
    @@ -0,0 +1,135 @@
    +/**
    + * Licensed to the Apache Software Foundation (ASF) under one
    + * or more contributor license agreements. See the NOTICE file
    + * distributed with this work for additional information
    + * regarding copyright ownership. The ASF licenses this file
    + * to you under the Apache License, Version 2.0 (the
    + * "License"); you may not use this file except in compliance
    + * with the License. You may obtain a copy of the License at
    + * <p/>
    + * http://www.apache.org/licenses/LICENSE-2.0
    + * <p/>
    + * Unless required by applicable law or agreed to in writing, software
    + * distributed under the License is distributed on an "AS IS" BASIS,
    + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    + * See the License for the specific language governing permissions and
    + * limitations under the License.
    + */
    +package org.apache.hadoop.hive.ql.log;
    +
    +import java.io.Serializable;
    +import java.util.concurrent.atomic.AtomicLongArray;
    +
    +import org.apache.hadoop.hive.common.classification.InterfaceAudience;
    +import org.apache.logging.log4j.Level;
    +import org.apache.logging.log4j.LogManager;
    +import org.apache.logging.log4j.core.Filter;
    +import org.apache.logging.log4j.core.Layout;
    +import org.apache.logging.log4j.core.LogEvent;
    +import org.apache.logging.log4j.core.LoggerContext;
    +import org.apache.logging.log4j.core.appender.AbstractAppender;
    +import org.apache.logging.log4j.core.config.Configuration;
    +import org.apache.logging.log4j.core.config.LoggerConfig;
    +import org.apache.logging.log4j.core.config.plugins.Plugin;
    +import org.apache.logging.log4j.core.config.plugins.PluginAttribute;
    +import org.apache.logging.log4j.core.config.plugins.PluginElement;
    +import org.apache.logging.log4j.core.config.plugins.PluginFactory;
    +import org.apache.logging.log4j.core.layout.PatternLayout;
    +
    +import com.google.common.annotations.VisibleForTesting;
    +
    +/**
    + * A log4J2 Appender that simply counts logging events in four levels:
    + * fatal, error, warn and info. The class name is used in log4j2.xml
    + */
    +@Plugin(name = "HiveEventCounter", category = "Core", elementType = "appender", printObject = true)
    +public class HiveEventCounter extends AbstractAppender {
    + private static LoggerContext context = (LoggerContext) LogManager.getContext(false);
    + private static Configuration configuration = context.getConfiguration();
    + private static final String APPENDER_NAME = "HiveEventCounter";
    + private static final int FATAL = 0;
    + private static final int ERROR = 1;
    + private static final int WARN = 2;
    + private static final int INFO = 3;
    +
    + private static class EventCounts {
    + private final AtomicLongArray counts = new AtomicLongArray(4);
    +
    + private void incr(int i) {
    + counts.incrementAndGet(i);
    + }
    +
    + private long get(int i) {
    + return counts.get(i);
    + }
    + }
    +
    + private static EventCounts counts = new EventCounts();
    +
    + protected HiveEventCounter(String name, Filter filter,
    + Layout<? extends Serializable> layout, boolean ignoreExceptions) {
    + super(name, filter, layout, ignoreExceptions);
    + }
    +
    + @PluginFactory
    + public static HiveEventCounter createInstance(@PluginAttribute("name") String name,
    + @PluginAttribute("ignoreExceptions") boolean ignoreExceptions,
    + @PluginElement("Layout") Layout layout,
    + @PluginElement("Filters") Filter filter) {
    + if (name == null) {
    + name = APPENDER_NAME;
    + }
    +
    + if (layout == null) {
    + layout = PatternLayout.createDefaultLayout();
    + }
    + return new HiveEventCounter(name, filter, layout, ignoreExceptions);
    + }
    +
    + @InterfaceAudience.Private
    + public static long getFatal() {
    + return counts.get(FATAL);
    + }
    +
    + @InterfaceAudience.Private
    + public static long getError() {
    + return counts.get(ERROR);
    + }
    +
    + @InterfaceAudience.Private
    + public static long getWarn() {
    + return counts.get(WARN);
    + }
    +
    + @InterfaceAudience.Private
    + public static long getInfo() {
    + return counts.get(INFO);
    + }
    +
    + @VisibleForTesting
    + public void addToLogger(String loggerName, Level level) {
    + LoggerConfig loggerConfig = configuration.getLoggerConfig(loggerName);
    + loggerConfig.addAppender(this, level, null);
    + context.updateLoggers();
    + }
    +
    + @VisibleForTesting
    + public void removeFromLogger(String loggerName) {
    + LoggerConfig loggerConfig = configuration.getLoggerConfig(loggerName);
    + loggerConfig.removeAppender(APPENDER_NAME);
    + context.updateLoggers();
    + }
    +
    + public void append(LogEvent event) {
    + Level level = event.getLevel();
    + if (level.equals(Level.INFO)) {
    + counts.incr(INFO);
    + } else if (level.equals(Level.WARN)) {
    + counts.incr(WARN);
    + } else if (level.equals(Level.ERROR)) {
    + counts.incr(ERROR);
    + } else if (level.equals(Level.FATAL)) {
    + counts.incr(FATAL);
    + }
    + }
    +}

    http://git-wip-us.apache.org/repos/asf/hive/blob/c93d6c77/ql/src/java/org/apache/hadoop/hive/ql/log/NullAppender.java
    ----------------------------------------------------------------------
    diff --git a/ql/src/java/org/apache/hadoop/hive/ql/log/NullAppender.java b/ql/src/java/org/apache/hadoop/hive/ql/log/NullAppender.java
    new file mode 100644
    index 0000000..c4cb7dd
    --- /dev/null
    +++ b/ql/src/java/org/apache/hadoop/hive/ql/log/NullAppender.java
    @@ -0,0 +1,63 @@
    +/**
    + * Licensed to the Apache Software Foundation (ASF) under one
    + * or more contributor license agreements. See the NOTICE file
    + * distributed with this work for additional information
    + * regarding copyright ownership. The ASF licenses this file
    + * to you under the Apache License, Version 2.0 (the
    + * "License"); you may not use this file except in compliance
    + * with the License. You may obtain a copy of the License at
    + * <p/>
    + * http://www.apache.org/licenses/LICENSE-2.0
    + * <p/>
    + * Unless required by applicable law or agreed to in writing, software
    + * distributed under the License is distributed on an "AS IS" BASIS,
    + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    + * See the License for the specific language governing permissions and
    + * limitations under the License.
    + */
    +package org.apache.hadoop.hive.ql.log;
    +
    +import java.io.Serializable;
    +
    +import org.apache.logging.log4j.Level;
    +import org.apache.logging.log4j.LogManager;
    +import org.apache.logging.log4j.core.Filter;
    +import org.apache.logging.log4j.core.Layout;
    +import org.apache.logging.log4j.core.LogEvent;
    +import org.apache.logging.log4j.core.LoggerContext;
    +import org.apache.logging.log4j.core.appender.AbstractAppender;
    +import org.apache.logging.log4j.core.config.Configuration;
    +import org.apache.logging.log4j.core.config.LoggerConfig;
    +import org.apache.logging.log4j.core.config.plugins.Plugin;
    +import org.apache.logging.log4j.core.config.plugins.PluginFactory;
    +import org.apache.logging.log4j.core.layout.PatternLayout;
    +
    +/**
    + * A NullAppender merely exists, it never outputs a message to any device.
    + */
    +@Plugin(name = "NullAppender", category = "Core", elementType = "appender", printObject = false)
    +public class NullAppender extends AbstractAppender {
    +
    + private static LoggerContext context = (LoggerContext) LogManager.getContext(false);
    + private static Configuration configuration = context.getConfiguration();
    +
    + protected NullAppender(String name, Filter filter,
    + Layout<? extends Serializable> layout, boolean ignoreExceptions) {
    + super(name, filter, layout, ignoreExceptions);
    + }
    +
    + @PluginFactory
    + public static NullAppender createNullAppender() {
    + return new NullAppender("NullAppender", null, PatternLayout.createDefaultLayout(), true);
    + }
    +
    + public void addToLogger(String loggerName, Level level) {
    + LoggerConfig loggerConfig = configuration.getLoggerConfig(loggerName);
    + loggerConfig.addAppender(this, level, null);
    + context.updateLoggers();
    + }
    +
    + public void append(LogEvent event) {
    + // no-op
    + }
    +}

    http://git-wip-us.apache.org/repos/asf/hive/blob/c93d6c77/ql/src/java/org/apache/hadoop/hive/ql/log/PidDailyRollingFileAppender.java
    ----------------------------------------------------------------------
    diff --git a/ql/src/java/org/apache/hadoop/hive/ql/log/PidDailyRollingFileAppender.java b/ql/src/java/org/apache/hadoop/hive/ql/log/PidDailyRollingFileAppender.java
    deleted file mode 100644
    index 6a59d4a..0000000
    --- a/ql/src/java/org/apache/hadoop/hive/ql/log/PidDailyRollingFileAppender.java
    +++ /dev/null
    @@ -1,33 +0,0 @@
    -/**
    - * Licensed to the Apache Software Foundation (ASF) under one
    - * or more contributor license agreements. See the NOTICE file
    - * distributed with this work for additional information
    - * regarding copyright ownership. The ASF licenses this file
    - * to you under the Apache License, Version 2.0 (the
    - * "License"); you may not use this file except in compliance
    - * with the License. You may obtain a copy of the License at
    - *
    - * http://www.apache.org/licenses/LICENSE-2.0
    - *
    - * Unless required by applicable law or agreed to in writing, software
    - * distributed under the License is distributed on an "AS IS" BASIS,
    - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    - * See the License for the specific language governing permissions and
    - * limitations under the License.
    - */
    -
    -package org.apache.hadoop.hive.ql.log;
    -
    -import java.lang.management.ManagementFactory;
    -import java.lang.management.RuntimeMXBean;
    -
    -import org.apache.log4j.DailyRollingFileAppender;
    -
    -public class PidDailyRollingFileAppender extends DailyRollingFileAppender {
    -
    - @Override
    - public void setFile(String file) {
    - RuntimeMXBean rt = ManagementFactory.getRuntimeMXBean();
    - super.setFile(file + '.' + rt.getName());
    - }
    -}

    http://git-wip-us.apache.org/repos/asf/hive/blob/c93d6c77/ql/src/java/org/apache/hadoop/hive/ql/log/PidFilePatternConverter.java
    ----------------------------------------------------------------------
    diff --git a/ql/src/java/org/apache/hadoop/hive/ql/log/PidFilePatternConverter.java b/ql/src/java/org/apache/hadoop/hive/ql/log/PidFilePatternConverter.java
    new file mode 100644
    index 0000000..4db10bb
    --- /dev/null
    +++ b/ql/src/java/org/apache/hadoop/hive/ql/log/PidFilePatternConverter.java
    @@ -0,0 +1,62 @@
    +/**
    + * Licensed to the Apache Software Foundation (ASF) under one
    + * or more contributor license agreements. See the NOTICE file
    + * distributed with this work for additional information
    + * regarding copyright ownership. The ASF licenses this file
    + * to you under the Apache License, Version 2.0 (the
    + * "License"); you may not use this file except in compliance
    + * with the License. You may obtain a copy of the License at
    + * <p/>
    + * http://www.apache.org/licenses/LICENSE-2.0
    + * <p/>
    + * Unless required by applicable law or agreed to in writing, software
    + * distributed under the License is distributed on an "AS IS" BASIS,
    + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    + * See the License for the specific language governing permissions and
    + * limitations under the License.
    + */
    +package org.apache.hadoop.hive.ql.log;
    +
    +import java.lang.management.ManagementFactory;
    +
    +import org.apache.logging.log4j.core.config.plugins.Plugin;
    +import org.apache.logging.log4j.core.config.plugins.PluginFactory;
    +import org.apache.logging.log4j.core.pattern.AbstractPatternConverter;
    +import org.apache.logging.log4j.core.pattern.ArrayPatternConverter;
    +import org.apache.logging.log4j.core.pattern.ConverterKeys;
    +
    +/**
    + * FilePattern converter that converts %pid pattern to <process-id>@<hostname> information
    + * obtained at runtime.
    + *
    + * Example usage:
    + * <RollingFile name="Rolling-default" fileName="test.log" filePattern="test.log.%pid.gz">
    + *
    + * Will generate output file with name containing <process-id>@<hostname> like below
    + * test.log.95232@localhost.gz
    + */
    +@Plugin(name = "PidFilePatternConverter", category = "FileConverter")
    +@ConverterKeys({ "pid" })
    +public class PidFilePatternConverter extends AbstractPatternConverter implements
    + ArrayPatternConverter {
    +
    + /**
    + * Private constructor.
    + */
    + private PidFilePatternConverter() {
    + super("pid", "pid");
    + }
    +
    + @PluginFactory
    + public static PidFilePatternConverter newInstance() {
    + return new PidFilePatternConverter();
    + }
    +
    + public void format(StringBuilder toAppendTo, Object... objects) {
    + toAppendTo.append(ManagementFactory.getRuntimeMXBean().getName());
    + }
    +
    + public void format(Object obj, StringBuilder toAppendTo) {
    + toAppendTo.append(ManagementFactory.getRuntimeMXBean().getName());
    + }
    +}

    http://git-wip-us.apache.org/repos/asf/hive/blob/c93d6c77/ql/src/main/resources/hive-exec-log4j.properties
    ----------------------------------------------------------------------
    diff --git a/ql/src/main/resources/hive-exec-log4j.properties b/ql/src/main/resources/hive-exec-log4j.properties
    deleted file mode 100644
    index 9eaa6b6..0000000
    --- a/ql/src/main/resources/hive-exec-log4j.properties
    +++ /dev/null
    @@ -1,77 +0,0 @@
    -# Licensed to the Apache Software Foundation (ASF) under one
    -# or more contributor license agreements. See the NOTICE file
    -# distributed with this work for additional information
    -# regarding copyright ownership. The ASF licenses this file
    -# to you under the Apache License, Version 2.0 (the
    -# "License"); you may not use this file except in compliance
    -# with the License. You may obtain a copy of the License at
    -#
    -# http://www.apache.org/licenses/LICENSE-2.0
    -#
    -# Unless required by applicable law or agreed to in writing, software
    -# distributed under the License is distributed on an "AS IS" BASIS,
    -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    -# See the License for the specific language governing permissions and
    -# limitations under the License.
    -
    -# Define some default values that can be overridden by system properties
    -hive.log.threshold=ALL
    -hive.root.logger=INFO,FA
    -hive.log.dir=${java.io.tmpdir}/${user.name}
    -hive.query.id=hadoop
    -hive.log.file=${hive.query.id}.log
    -
    -# Define the root logger to the system property "hadoop.root.logger".
    -log4j.rootLogger=${hive.root.logger}, EventCounter
    -
    -# Logging Threshold
    -log4j.threshhold=${hive.log.threshold}
    -
    -#
    -# File Appender
    -#
    -
    -log4j.appender.FA=org.apache.log4j.FileAppender
    -log4j.appender.FA.File=${hive.log.dir}/${hive.log.file}
    -log4j.appender.FA.layout=org.apache.log4j.PatternLayout
    -
    -# Pattern format: Date LogLevel LoggerName LogMessage
    -#log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n
    -# Debugging Pattern format
    -log4j.appender.FA.layout.ConversionPattern=%d{ISO8601} %-5p [%t]: %c{2} (%F:%M(%L)) - %m%n
    -
    -
    -#
    -# console
    -# Add "console" to rootlogger above if you want to use this
    -#
    -
    -log4j.appender.console=org.apache.log4j.ConsoleAppender
    -log4j.appender.console.target=System.err
    -log4j.appender.console.layout=org.apache.log4j.PatternLayout
    -log4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} [%t] %p %c{2}: %m%n
    -
    -#custom logging levels
    -#log4j.logger.xxx=DEBUG
    -
    -#
    -# Event Counter Appender
    -# Sends counts of logging messages at different severity levels to Hadoop Metrics.
    -#
    -log4j.appender.EventCounter=org.apache.hadoop.hive.shims.HiveEventCounter
    -
    -
    -log4j.category.DataNucleus=ERROR,FA
    -log4j.category.Datastore=ERROR,FA
    -log4j.category.Datastore.Schema=ERROR,FA
    -log4j.category.JPOX.Datastore=ERROR,FA
    -log4j.category.JPOX.Plugin=ERROR,FA
    -log4j.category.JPOX.MetaData=ERROR,FA
    -log4j.category.JPOX.Query=ERROR,FA
    -log4j.category.JPOX.General=ERROR,FA
    -log4j.category.JPOX.Enhancer=ERROR,FA
    -
    -
    -# Silence useless ZK logs
    -log4j.logger.org.apache.zookeeper.server.NIOServerCnxn=WARN,FA
    -log4j.logger.org.apache.zookeeper.ClientCnxnSocketNIO=WARN,FA

    http://git-wip-us.apache.org/repos/asf/hive/blob/c93d6c77/ql/src/main/resources/hive-exec-log4j2.xml
    ----------------------------------------------------------------------
    diff --git a/ql/src/main/resources/hive-exec-log4j2.xml b/ql/src/main/resources/hive-exec-log4j2.xml
    new file mode 100644
    index 0000000..c93437c
    --- /dev/null
    +++ b/ql/src/main/resources/hive-exec-log4j2.xml
    @@ -0,0 +1,110 @@
    +<?xml version="1.0" encoding="UTF-8"?>
    +<!--
    + Licensed to the Apache Software Foundation (ASF) under one or more
    + contributor license agreements. See the NOTICE file distributed with
    + this work for additional information regarding copyright ownership.
    + The ASF licenses this file to You under the Apache License, Version 2.0
    + (the "License"); you may not use this file except in compliance with
    + the License. You may obtain a copy of the License at
    +
    + http://www.apache.org/licenses/LICENSE-2.0
    +
    + Unless required by applicable law or agreed to in writing, software
    + distributed under the License is distributed on an "AS IS" BASIS,
    + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    + See the License for the specific language governing permissions and
    + limitations under the License.
    +-->
    +
    +<Configuration status="info" strict="true" name="HiveExecLog4j2"
    + packages="org.apache.hadoop.hive.ql.log">
    +
    + <Properties>
    + <Property name="hive.log.threshold">ALL</Property>
    + <Property name="hive.log.level">INFO</Property>
    + <Property name="hive.root.logger">FA</Property>
    + <Property name="hive.log.dir">${sys:java.io.tmpdir}/${sys:user.name}</Property>
    + <Property name="hive.query.id">hadoop</Property>
    + <Property name="hive.log.file">${sys:hive.query.id}.log</Property>
    + </Properties>
    +
    + <Appenders>
    + <Console name="console" target="SYSTEM_ERR">
    + <PatternLayout pattern="%d{yy/MM/dd HH:mm:ss} [%t]: %p %c{2}: %m%n"/>
    + </Console>
    +
    + <!-- Regular File Appender -->
    + <!-- NOTE: if enabling multiple file appender make sure to use different file names -->
    + <File name="FA" fileName="${sys:hive.log.dir}/${sys:hive.log.file}">
    + <PatternLayout pattern="%d{ISO8601} %-5p [%t]: %c{2} (%F:%M(%L)) - %m%n" />
    + </File>
    +
    + <!-- Daily Rolling File Appender -->
    + <!-- NOTE: if enabling multiple file appender make sure to use different file names -->
    + <!-- Use %pid in the filePattern to append <process-id>@<host-name> to the filename if you want separate log files for different CLI session -->
    + <!-- <RollingFile name="DRFA" fileName="${sys:hive.log.dir}/${sys:hive.log.file}"
    + filePattern="${sys:hive.log.dir}/${sys:hive.log.file}.%d{yyyy-MM-dd}">
    + <PatternLayout pattern="%d{ISO8601} %-5p [%t]: %c{2} (%F:%M(%L)) - %m%n" />
    + <Policies>
    + <TimeBasedTriggeringPolicy interval="1" modulate="true"/>
    + </Policies>
    + <DefaultRolloverStrategy max="30"/>
    + </RollingFile> -->
    +
    + <!-- Size based Rolling File Appender -->
    + <!-- NOTE: if enabling multiple file appender make sure to use different file names -->
    + <!-- <RollingFile name="RFA" fileName="${sys:hive.log.dir}/${sys:hive.log.file}"
    + filePattern="${sys:hive.log.dir}/${sys:hive.log.file}.%i">
    + <PatternLayout pattern="%d{ISO8601} %-5p [%t]: %c{2} (%F:%M(%L)) - %m%n" />
    + <Policies>
    + <SizeBasedTriggeringPolicy size="256 MB" />
    + </Policies>
    + <DefaultRolloverStrategy max="10"/>
    + </RollingFile> -->
    +
    + <!-- hiveeventcounter appender is loaded from configuration packages attribute.sends counts of logging messages@different severity levels to hadoop metrics. -->
    + <HiveEventCounter name="EventCounter"/>
    + </Appenders>
    +
    + <Loggers>
    + <Root level="${sys:hive.log.threshold}">
    + <AppenderRef ref="${sys:hive.root.logger}" level="${sys:hive.log.level}"/>
    + <AppenderRef ref="EventCounter" />
    + </Root>
    +
    + <Logger name="org.apache.zookeeper.server.NIOServerCnxn" level="WARN">
    + <AppenderRef ref="${sys:hive.root.logger}"/>
    + </Logger>
    + <Logger name="org.apache.zookeeper.ClientCnxnSocketNIO" level="WARN">
    + <AppenderRef ref="${sys:hive.root.logger}"/>
    + </Logger>
    + <Logger name="DataNucleus" level="ERROR">
    + <AppenderRef ref="${sys:hive.root.logger}"/>
    + </Logger>
    + <Logger name="Datastore" level="ERROR">
    + <AppenderRef ref="${sys:hive.root.logger}"/>
    + </Logger>
    + <Logger name="Datastore.Schema" level="ERROR">
    + <AppenderRef ref="${sys:hive.root.logger}"/>
    + </Logger>
    + <Logger name="JPOX.Datastore" level="ERROR">
    + <AppenderRef ref="${sys:hive.root.logger}"/>
    + </Logger>
    + <Logger name="JPOX.Plugin" level="ERROR">
    + <AppenderRef ref="${sys:hive.root.logger}"/>
    + </Logger>
    + <Logger name="JPOX.Metadata" level="ERROR">
    + <AppenderRef ref="${sys:hive.root.logger}"/>
    + </Logger>
    + <Logger name="JPOX.Query" level="ERROR">
    + <AppenderRef ref="${sys:hive.root.logger}"/>
    + </Logger>
    + <Logger name="JPOX.General" level="ERROR">
    + <AppenderRef ref="${sys:hive.root.logger}"/>
    + </Logger>
    + <Logger name="JPOX.Enhancer" level="ERROR">
    + <AppenderRef ref="${sys:hive.root.logger}"/>
    + </Logger>
    + </Loggers>
    +
    +</Configuration>

    http://git-wip-us.apache.org/repos/asf/hive/blob/c93d6c77/ql/src/test/org/apache/hadoop/hive/ql/log/TestLog4j2Appenders.java
    ----------------------------------------------------------------------
    diff --git a/ql/src/test/org/apache/hadoop/hive/ql/log/TestLog4j2Appenders.java b/ql/src/test/org/apache/hadoop/hive/ql/log/TestLog4j2Appenders.java
    new file mode 100644
    index 0000000..bdd837e
    --- /dev/null
    +++ b/ql/src/test/org/apache/hadoop/hive/ql/log/TestLog4j2Appenders.java
    @@ -0,0 +1,95 @@
    +/**
    + * Licensed to the Apache Software Foundation (ASF) under one
    + * or more contributor license agreements. See the NOTICE file
    + * distributed with this work for additional information
    + * regarding copyright ownership. The ASF licenses this file
    + * to you under the Apache License, Version 2.0 (the
    + * "License"); you may not use this file except in compliance
    + * with the License. You may obtain a copy of the License at
    + * <p/>
    + * http://www.apache.org/licenses/LICENSE-2.0
    + * <p/>
    + * Unless required by applicable law or agreed to in writing, software
    + * distributed under the License is distributed on an "AS IS" BASIS,
    + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    + * See the License for the specific language governing permissions and
    + * limitations under the License.
    + */
    +package org.apache.hadoop.hive.ql.log;
    +
    +import static org.junit.Assert.assertEquals;
    +
    +import org.apache.hadoop.hive.ql.metadata.StringAppender;
    +import org.apache.logging.log4j.Level;
    +import org.apache.logging.log4j.LogManager;
    +import org.apache.logging.log4j.Logger;
    +import org.apache.logging.log4j.core.LoggerContext;
    +import org.apache.logging.log4j.core.config.Configuration;
    +import org.apache.logging.log4j.core.config.LoggerConfig;
    +import org.junit.Before;
    +import org.junit.Test;
    +
    +/**
    + *
    + */
    +public class TestLog4j2Appenders {
    +
    + @Before
    + public void setup() {
    + // programmatically set root logger level to INFO. By default if log4j2-test.xml is not
    + // available root logger will use ERROR log level
    + LoggerContext ctx = (LoggerContext) LogManager.getContext(false);
    + Configuration config = ctx.getConfiguration();
    + LoggerConfig loggerConfig = config.getLoggerConfig(LogManager.ROOT_LOGGER_NAME);
    + loggerConfig.setLevel(Level.INFO);
    + ctx.updateLoggers();
    + }
    +
    + @Test
    + public void testStringAppender() throws Exception {
    + // Get the RootLogger which, if you don't have log4j2-test.xml defined, will only log ERRORs
    + Logger logger = LogManager.getRootLogger();
    + // Create a String Appender to capture log output
    + StringAppender appender = StringAppender.createStringAppender("%m");
    + appender.addToLogger(logger.getName(), Level.INFO);
    + appender.start();
    +
    + // Log to the string appender
    + logger.info("Hello!");
    + logger.info(" World");
    +
    + assertEquals("Hello! World", appender.getOutput());
    + appender.removeFromLogger(LogManager.getRootLogger().getName());
    + }
    +
    + @Test
    + public void testHiveEventCounterAppender() throws Exception {
    + Logger logger = LogManager.getRootLogger();
    + HiveEventCounter appender = HiveEventCounter.createInstance("EventCounter", true, null, null);
    + appender.addToLogger(logger.getName(), Level.INFO);
    + appender.start();
    +
    + logger.info("Test");
    + logger.info("Test");
    + logger.info("Test");
    + logger.info("Test");
    +
    + logger.error("Test");
    + logger.error("Test");
    + logger.error("Test");
    +
    + logger.warn("Test");
    + logger.warn("Test");
    +
    + logger.fatal("Test");
    +
    + // HiveEventCounter will be loaded from hive-log4j2-test.xml before tests are run. The 2 log
    + // info msgs from previous test case will also be counted along with 4 log info msgs in this
    + // test and hence we assert for 6 here
    + assertEquals(6, appender.getInfo());
    + assertEquals(3, appender.getError());
    + assertEquals(2, appender.getWarn());
    + assertEquals(1, appender.getFatal());
    + appender.removeFromLogger(LogManager.getRootLogger().getName());
    + }
    +}

    http://git-wip-us.apache.org/repos/asf/hive/blob/c93d6c77/ql/src/test/org/apache/hadoop/hive/ql/metadata/StringAppender.java
    ----------------------------------------------------------------------
    diff --git a/ql/src/test/org/apache/hadoop/hive/ql/metadata/StringAppender.java b/ql/src/test/org/apache/hadoop/hive/ql/metadata/StringAppender.java
    new file mode 100644
    index 0000000..17b64d6
    --- /dev/null
    +++ b/ql/src/test/org/apache/hadoop/hive/ql/metadata/StringAppender.java
    @@ -0,0 +1,128 @@
    +/**
    + * Licensed to the Apache Software Foundation (ASF) under one
    + * or more contributor license agreements. See the NOTICE file
    + * distributed with this work for additional information
    + * regarding copyright ownership. The ASF licenses this file
    + * to you under the Apache License, Version 2.0 (the
    + * "License"); you may not use this file except in compliance
    + * with the License. You may obtain a copy of the License at
    + * <p/>
    + * http://www.apache.org/licenses/LICENSE-2.0
    + * <p/>
    + * Unless required by applicable law or agreed to in writing, software
    + * distributed under the License is distributed on an "AS IS" BASIS,
    + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    + * See the License for the specific language governing permissions and
    + * limitations under the License.
    + */
    +package org.apache.hadoop.hive.ql.metadata;
    +
    +import java.io.ByteArrayOutputStream;
    +import java.io.OutputStream;
    +import java.io.OutputStreamWriter;
    +import java.io.Serializable;
    +
    +import org.apache.logging.log4j.Level;
    +import org.apache.logging.log4j.LogManager;
    +import org.apache.logging.log4j.core.Filter;
    +import org.apache.logging.log4j.core.Layout;
    +import org.apache.logging.log4j.core.LoggerContext;
    +import org.apache.logging.log4j.core.appender.AbstractOutputStreamAppender;
    +import org.apache.logging.log4j.core.appender.OutputStreamManager;
    +import org.apache.logging.log4j.core.config.Configuration;
    +import org.apache.logging.log4j.core.config.LoggerConfig;
    +import org.apache.logging.log4j.core.config.plugins.Plugin;
    +import org.apache.logging.log4j.core.config.plugins.PluginAttribute;
    +import org.apache.logging.log4j.core.config.plugins.PluginFactory;
    +import org.apache.logging.log4j.core.layout.PatternLayout;
    +
    +import com.google.common.annotations.VisibleForTesting;
    +
    +/**
    + * Log4j2 appender that writers to in-memory string object.
    + */
    +@Plugin(name = "StringAppender", category = "Core", elementType = "appender", printObject = true)
    +public class StringAppender
    + extends AbstractOutputStreamAppender<StringAppender.StringOutputStreamManager> {
    +
    + private static final String APPENDER_NAME = "StringAppender";
    + private static LoggerContext context = (LoggerContext) LogManager.getContext(false);
    + private static Configuration configuration = context.getConfiguration();
    + private StringOutputStreamManager manager;
    +
    + /**
    + * Instantiate a WriterAppender and set the output destination to a
    + * new {@link OutputStreamWriter} initialized with <code>os</code>
    + * as its {@link OutputStream}.
    + *
    + * @param name The name of the Appender.
    + * @param layout The layout to format the message.
    + * @param filter
    + * @param ignoreExceptions
    + * @param immediateFlush
    + * @param manager The OutputStreamManager.
    + */
    + protected StringAppender(String name,
    + Layout<? extends Serializable> layout, Filter filter,
    + boolean ignoreExceptions, boolean immediateFlush,
    + StringOutputStreamManager manager) {
    + super(name, layout, filter, ignoreExceptions, immediateFlush, manager);
    + this.manager = manager;
    + }
    +
    + @PluginFactory
    + public static StringAppender createStringAppender(
    + @PluginAttribute("name") String nullablePatternString) {
    + PatternLayout layout;
    + if (nullablePatternString == null) {
    + layout = PatternLayout.createDefaultLayout();
    + } else {
    + layout = PatternLayout.createLayout(nullablePatternString, configuration,
    + null, null, true, false, null, null);
    + }
    +
    + return new StringAppender(APPENDER_NAME, layout, null, false, true,
    + new StringOutputStreamManager(new ByteArrayOutputStream(), "StringStream", layout));
    + }
    +
    + @VisibleForTesting
    + public void addToLogger(String loggerName, Level level) {
    + LoggerConfig loggerConfig = configuration.getLoggerConfig(loggerName);
    + loggerConfig.addAppender(this, level, null);
    + context.updateLoggers();
    + }
    +
    + @VisibleForTesting
    + public void removeFromLogger(String loggerName) {
    + LoggerConfig loggerConfig = configuration.getLoggerConfig(loggerName);
    + loggerConfig.removeAppender(APPENDER_NAME);
    + context.updateLoggers();
    + }
    +
    + public String getOutput() {
    + manager.flush();
    + return new String(manager.getStream().toByteArray());
    + }
    +
    + public void reset() {
    + manager.reset();
    + }
    +
    + protected static class StringOutputStreamManager extends OutputStreamManager {
    + ByteArrayOutputStream stream;
    +
    + protected StringOutputStreamManager(ByteArrayOutputStream os, String streamName,
    + Layout<?> layout) {
    + super(os, streamName, layout);
    + stream = os;
    + }
    +
    + public ByteArrayOutputStream getStream() {
    + return stream;
    + }
    +
    + public void reset() {
    + stream.reset();
    + }
    + }
    +}

    http://git-wip-us.apache.org/repos/asf/hive/blob/c93d6c77/ql/src/test/org/apache/hadoop/hive/ql/metadata/TestHive.java
    ----------------------------------------------------------------------
    diff --git a/ql/src/test/org/apache/hadoop/hive/ql/metadata/TestHive.java b/ql/src/test/org/apache/hadoop/hive/ql/metadata/TestHive.java
    index 99fbd5d..1e2feaa 100755
    --- a/ql/src/test/org/apache/hadoop/hive/ql/metadata/TestHive.java
    +++ b/ql/src/test/org/apache/hadoop/hive/ql/metadata/TestHive.java
    @@ -20,7 +20,6 @@ package org.apache.hadoop.hive.ql.metadata;

      import static org.apache.hadoop.hive.metastore.MetaStoreUtils.DEFAULT_DATABASE_NAME;

    -import java.io.StringWriter;
      import java.util.ArrayList;
      import java.util.Arrays;
      import java.util.HashMap;
    @@ -29,8 +28,6 @@ import java.util.List;
      import java.util.Map;
      import java.util.regex.Pattern;

    -import junit.framework.TestCase;
    -
      import org.apache.hadoop.fs.FileStatus;
      import org.apache.hadoop.fs.FileSystem;
      import org.apache.hadoop.fs.Path;
    @@ -56,15 +53,19 @@ import org.apache.hadoop.mapred.SequenceFileInputFormat;
      import org.apache.hadoop.mapred.SequenceFileOutputFormat;
      import org.apache.hadoop.mapred.TextInputFormat;
      import org.apache.hadoop.util.StringUtils;
    -import org.apache.log4j.Level;
    -import org.apache.log4j.Logger;
    -import org.apache.log4j.PatternLayout;
    -import org.apache.log4j.WriterAppender;
    +import org.apache.logging.log4j.Level;
    +import org.apache.logging.log4j.LogManager;
    +import org.apache.logging.log4j.Logger;
    +import org.apache.logging.log4j.core.LoggerContext;
    +import org.apache.logging.log4j.core.config.Configuration;
    +import org.apache.logging.log4j.core.config.LoggerConfig;
      import org.apache.thrift.protocol.TBinaryProtocol;
      import org.junit.Assert;

      import com.google.common.collect.ImmutableMap;

    +import junit.framework.TestCase;
    +
      /**
       * TestHive.
       *
    @@ -248,36 +249,39 @@ public class TestHive extends TestCase {
         * @throws Throwable
         */
        public void testMetaStoreApiTiming() throws Throwable {
    - // set log level to DEBUG, as this is logged at debug level
    - Logger logger = Logger.getLogger("hive.ql.metadata.Hive");
    - Level origLevel = logger.getLevel();
    - logger.setLevel(Level.DEBUG);
    -
    - // create an appender to capture the logs in a string
    - StringWriter writer = new StringWriter();
    - WriterAppender appender = new WriterAppender(new PatternLayout(), writer);
    + // Get the RootLogger which, if you don't have log4j2-test.xml defined, will only log ERRORs
    + Logger logger = LogManager.getLogger("hive.ql.metadata.Hive");
    + Level oldLevel = logger.getLevel();
    + LoggerContext ctx = (LoggerContext) LogManager.getContext(false);
    + Configuration config = ctx.getConfiguration();
    + LoggerConfig loggerConfig = config.getLoggerConfig(logger.getName());
    + loggerConfig.setLevel(Level.DEBUG);
    + ctx.updateLoggers();
    +
    + // Create a String Appender to capture log output
    + StringAppender appender = StringAppender.createStringAppender("%m");
    + appender.addToLogger(logger.getName(), Level.DEBUG);
    + appender.start();

          try {
    - logger.addAppender(appender);
    -
            hm.clearMetaCallTiming();
            hm.getAllDatabases();
            hm.dumpAndClearMetaCallTiming("test");
    - String logStr = writer.toString();
    + String logStr = appender.getOutput();
            String expectedString = "getAllDatabases_()=";
            Assert.assertTrue(logStr + " should contain <" + expectedString,
                logStr.contains(expectedString));

            // reset the log buffer, verify new dump without any api call does not contain func
    - writer.getBuffer().setLength(0);
    + appender.reset();
            hm.dumpAndClearMetaCallTiming("test");
    - logStr = writer.toString();
    + logStr = appender.getOutput();
            Assert.assertFalse(logStr + " should not contain <" + expectedString,
                logStr.contains(expectedString));
    -
          } finally {
    - logger.setLevel(origLevel);
    - logger.removeAppender(appender);
    + loggerConfig.setLevel(oldLevel);
    + ctx.updateLoggers();
    + appender.removeFromLogger(logger.getName());
          }
        }


    http://git-wip-us.apache.org/repos/asf/hive/blob/c93d6c77/service/src/java/org/apache/hive/service/cli/CLIServiceUtils.java
    ----------------------------------------------------------------------
    diff --git a/service/src/java/org/apache/hive/service/cli/CLIServiceUtils.java b/service/src/java/org/apache/hive/service/cli/CLIServiceUtils.java
    index 9d64b10..876ade8 100644
    --- a/service/src/java/org/apache/hive/service/cli/CLIServiceUtils.java
    +++ b/service/src/java/org/apache/hive/service/cli/CLIServiceUtils.java
    @@ -18,9 +18,6 @@

      package org.apache.hive.service.cli;

    -import org.apache.log4j.Layout;
    -import org.apache.log4j.PatternLayout;
    -
      /**
       * CLIServiceUtils.
       *
    @@ -29,10 +26,6 @@ public class CLIServiceUtils {


        private static final char SEARCH_STRING_ESCAPE = '\\';
    - public static final Layout verboseLayout = new PatternLayout(
    - "%d{yy/MM/dd HH:mm:ss} %p %c{2}: %m%n");
    - public static final Layout nonVerboseLayout = new PatternLayout(
    - "%-5p : %m%n");

        /**
         * Convert a SQL search pattern into an equivalent Java Regex.

    http://git-wip-us.apache.org/repos/asf/hive/blob/c93d6c77/service/src/java/org/apache/hive/service/cli/operation/LogDivertAppender.java
    ----------------------------------------------------------------------
    diff --git a/service/src/java/org/apache/hive/service/cli/operation/LogDivertAppender.java b/service/src/java/org/apache/hive/service/cli/operation/LogDivertAppender.java
    index 70340bd..fb3921f 100644
    --- a/service/src/java/org/apache/hive/service/cli/operation/LogDivertAppender.java
    +++ b/service/src/java/org/apache/hive/service/cli/operation/LogDivertAppender.java
    @@ -6,44 +6,94 @@
       * to you under the Apache License, Version 2.0 (the
       * "License"); you may not use this file except in compliance
       * with the License. You may obtain a copy of the License at
    - *
    - * http://www.apache.org/licenses/LICENSE-2.0
    - *
    + * <p/>
    + * http://www.apache.org/licenses/LICENSE-2.0
    + * <p/>
       * Unless required by applicable law or agreed to in writing, software
       * distributed under the License is distributed on an "AS IS" BASIS,
       * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
       * See the License for the specific language governing permissions and
       * limitations under the License.
       */
    -
      package org.apache.hive.service.cli.operation;
    -import java.io.CharArrayWriter;
    -import java.util.Enumeration;
    +
    +import java.io.ByteArrayOutputStream;
    +import java.io.OutputStream;
    +import java.io.OutputStreamWriter;
    +import java.io.Serializable;
      import java.util.regex.Pattern;

      import org.apache.hadoop.hive.ql.exec.Task;
      import org.apache.hadoop.hive.ql.log.PerfLogger;
      import org.apache.hadoop.hive.ql.session.OperationLog;
    -import org.apache.hadoop.hive.ql.session.OperationLog.LoggingLevel;
    -import org.apache.hive.service.cli.CLIServiceUtils;
    -import org.apache.log4j.Appender;
    -import org.apache.log4j.ConsoleAppender;
    -import org.apache.log4j.Layout;
    -import org.apache.log4j.Logger;
    -import org.apache.log4j.WriterAppender;
    -import org.apache.log4j.spi.Filter;
    -import org.apache.log4j.spi.LoggingEvent;
    +import org.apache.logging.log4j.LogManager;
    +import org.apache.logging.log4j.Logger;
    +import org.apache.logging.log4j.core.Appender;
    +import org.apache.logging.log4j.core.Filter;
    +import org.apache.logging.log4j.core.Layout;
    +import org.apache.logging.log4j.core.LogEvent;
    +import org.apache.logging.log4j.core.LoggerContext;
    +import org.apache.logging.log4j.core.appender.AbstractOutputStreamAppender;
    +import org.apache.logging.log4j.core.appender.ConsoleAppender;
    +import org.apache.logging.log4j.core.appender.OutputStreamManager;
    +import org.apache.logging.log4j.core.config.Configuration;
    +import org.apache.logging.log4j.core.filter.AbstractFilter;
    +import org.apache.logging.log4j.core.layout.PatternLayout;

      import com.google.common.base.Joiner;

      /**
    - * An Appender to divert logs from individual threads to the LogObject they belong to.
    + * Divert appender to redirect operation logs to separate files.
       */
    -public class LogDivertAppender extends WriterAppender {
    - private static final Logger LOG = Logger.getLogger(LogDivertAppender.class.getName());
    +public class LogDivertAppender
    + extends AbstractOutputStreamAppender<LogDivertAppender.StringOutputStreamManager> {
    + private static final Logger LOG = LogManager.getLogger(LogDivertAppender.class.getName());
    + private static LoggerContext context = (LoggerContext) LogManager.getContext(false);
    + private static Configuration configuration = context.getConfiguration();
    + public static final Layout<? extends Serializable> verboseLayout = PatternLayout.createLayout(
    + "%d{yy/MM/dd HH:mm:ss} %p %c{2}: %m%n", configuration, null, null, true, false, null, null);
    + public static final Layout<? extends Serializable> nonVerboseLayout = PatternLayout.createLayout(
    + "%-5p : %m%n", configuration, null, null, true, false, null, null);
    +
        private final OperationManager operationManager;
    + private StringOutputStreamManager manager;
        private boolean isVerbose;
    - private Layout verboseLayout;
    + private final Layout<? extends Serializable> layout;
    +
    + /**
    + * Instantiate a WriterAppender and set the output destination to a
    + * new {@link OutputStreamWriter} initialized with <code>os</code>
    + * as its {@link OutputStream}.
    + *
    + * @param name The name of the Appender.
    + * @param filter Filter
    + * @param manager The OutputStreamManager.
    + * @param operationManager Operation manager
    + */
    + protected LogDivertAppender(String name, Filter filter,
    + StringOutputStreamManager manager, OperationManager operationManager,
    + OperationLog.LoggingLevel loggingMode) {
    + super(name, null, filter, false, true, manager);
    + this.operationManager = operationManager;
    + this.manager = manager;
    + this.isVerbose = (loggingMode == OperationLog.LoggingLevel.VERBOSE);
    + this.layout = getDefaultLayout();
    + }
    +
    + public Layout<? extends Serializable> getDefaultLayout() {
    + // There should be a ConsoleAppender. Copy its Layout.
    + Logger root = LogManager.getRootLogger();
    + Layout layout = null;
    +
    + for (Appender ap : ((org.apache.logging.log4j.core.Logger) root).getAppenders().values()) {
    + if (ap.getClass().equals(ConsoleAppender.class)) {
    + layout = ap.getLayout();
    + break;
    + }
    + }
    +
    + return layout;
    + }

        /**
         * A log filter that filters messages coming from the logger with the given names.
    @@ -52,31 +102,31 @@ public class LogDivertAppender extends WriterAppender {
         * they don't generate more logs for themselves when they process logs.
         * White list filter is used for less verbose log collection
         */
    - private static class NameFilter extends Filter {
    + private static class NameFilter extends AbstractFilter {
          private Pattern namePattern;
    - private LoggingLevel loggingMode;
    + private OperationLog.LoggingLevel loggingMode;
          private OperationManager operationManager;

          /* Patterns that are excluded in verbose logging level.
           * Filter out messages coming from log processing classes, or we'll run an infinite loop.
           */
          private static final Pattern verboseExcludeNamePattern = Pattern.compile(Joiner.on("|").
    - join(new String[] {LOG.getName(), OperationLog.class.getName(),
    - OperationManager.class.getName()}));
    + join(new String[]{LOG.getName(), OperationLog.class.getName(),
    + OperationManager.class.getName()}));

          /* Patterns that are included in execution logging level.
           * In execution mode, show only select logger messages.
           */
          private static final Pattern executionIncludeNamePattern = Pattern.compile(Joiner.on("|").
    - join(new String[] {"org.apache.hadoop.mapreduce.JobSubmitter",
    - "org.apache.hadoop.mapreduce.Job", "SessionState", Task.class.getName(),
    - "org.apache.hadoop.hive.ql.exec.spark.status.SparkJobMonitor"}));
    + join(new String[]{"org.apache.hadoop.mapreduce.JobSubmitter",
    + "org.apache.hadoop.mapreduce.Job", "SessionState", Task.class.getName(),
    + "org.apache.hadoop.hive.ql.exec.spark.status.SparkJobMonitor"}));

          /* Patterns that are included in performance logging level.
           * In performance mode, show execution and performance logger messages.
           */
          private static final Pattern performanceIncludeNamePattern = Pattern.compile(
    - executionIncludeNamePattern.pattern() + "|" + PerfLogger.class.getName());
    + executionIncludeNamePattern.pattern() + "|" + PerfLogger.class.getName());

          private void setCurrentNamePattern(OperationLog.LoggingLevel mode) {
            if (mode == OperationLog.LoggingLevel.VERBOSE) {
    @@ -88,26 +138,25 @@ public class LogDivertAppender extends WriterAppender {
            }
          }

    - public NameFilter(
    - OperationLog.LoggingLevel loggingMode, OperationManager op) {
    + public NameFilter(OperationLog.LoggingLevel loggingMode, OperationManager op) {
            this.operationManager = op;
            this.loggingMode = loggingMode;
            setCurrentNamePattern(loggingMode);
          }

          @Override
    - public int decide(LoggingEvent ev) {
    + public Result filter(LogEvent event) {
            OperationLog log = operationManager.getOperationLogByThread();
            boolean excludeMatches = (loggingMode == OperationLog.LoggingLevel.VERBOSE);

            if (log == null) {
    - return Filter.DENY;
    + return Result.DENY;
            }

            OperationLog.LoggingLevel currentLoggingMode = log.getOpLoggingLevel();
            // If logging is disabled, deny everything.
            if (currentLoggingMode == OperationLog.LoggingLevel.NONE) {
    - return Filter.DENY;
    + return Result.DENY;
            }
            // Look at the current session's setting
            // and set the pattern and excludeMatches accordingly.
    @@ -116,88 +165,58 @@ public class LogDivertAppender extends WriterAppender {
              setCurrentNamePattern(loggingMode);
            }

    - boolean isMatch = namePattern.matcher(ev.getLoggerName()).matches();
    + boolean isMatch = namePattern.matcher(event.getLoggerName()).matches();

            if (excludeMatches == isMatch) {
              // Deny if this is black-list filter (excludeMatches = true) and it
    - // matched
    - // or if this is whitelist filter and it didn't match
    - return Filter.DENY;
    + // matched or if this is whitelist filter and it didn't match
    + return Result.DENY;
            }
    - return Filter.NEUTRAL;
    + return Result.NEUTRAL;
          }
        }

    - /** This is where the log message will go to */
    - private final CharArrayWriter writer = new CharArrayWriter();
    -
    - private void setLayout (boolean isVerbose, Layout lo) {
    - if (isVerbose) {
    - if (lo == null) {
    - lo = CLIServiceUtils.verboseLayout;
    - LOG.info("Cannot find a Layout from a ConsoleAppender. Using default Layout pattern.");
    - }
    - } else {
    - lo = CLIServiceUtils.nonVerboseLayout;
    - }
    - setLayout(lo);
    + public static LogDivertAppender createInstance(OperationManager operationManager,
    + OperationLog.LoggingLevel loggingMode) {
    + return new LogDivertAppender("LogDivertAppender", new NameFilter(loggingMode, operationManager),
    + new StringOutputStreamManager(new ByteArrayOutputStream(), "StringStream", null),
    + operationManager, loggingMode);
        }

    - private void initLayout(boolean isVerbose) {
    - // There should be a ConsoleAppender. Copy its Layout.
    - Logger root = Logger.getRootLogger();
    - Layout layout = null;
    -
    - Enumeration<?> appenders = root.getAllAppenders();
    - while (appenders.hasMoreElements()) {
    - Appender ap = (Appender) appenders.nextElement();
    - if (ap.getClass().equals(ConsoleAppender.class)) {
    - layout = ap.getLayout();
    - break;
    - }
    - }
    - setLayout(isVerbose, layout);
    + public String getOutput() {
    + return new String(manager.getStream().toByteArray());
        }

    - public LogDivertAppender(OperationManager operationManager,
    - OperationLog.LoggingLevel loggingMode) {
    - isVerbose = (loggingMode == OperationLog.LoggingLevel.VERBOSE);
    - initLayout(isVerbose);
    - setWriter(writer);
    - setName("LogDivertAppender");
    - this.operationManager = operationManager;
    - this.verboseLayout = isVerbose ? layout : CLIServiceUtils.verboseLayout;
    - addFilter(new NameFilter(loggingMode, operationManager));
    + @Override
    + public void start() {
    + super.start();
        }

        @Override
    - public void doAppend(LoggingEvent event) {
    - OperationLog log = operationManager.getOperationLogByThread();
    + public Layout<? extends Serializable> getLayout() {

    - // Set current layout depending on the verbose/non-verbose mode.
    + // If there is a logging level change from verbose->non-verbose or vice-versa since
    + // the last subAppend call, change the layout to preserve consistency.
    + OperationLog log = operationManager.getOperationLogByThread();
          if (log != null) {
    - boolean isCurrModeVerbose = (log.getOpLoggingLevel() == OperationLog.LoggingLevel.VERBOSE);
    + isVerbose = (log.getOpLoggingLevel() == OperationLog.LoggingLevel.VERBOSE);
    + }

    - // If there is a logging level change from verbose->non-verbose or vice-versa since
    - // the last subAppend call, change the layout to preserve consistency.
    - if (isCurrModeVerbose != isVerbose) {
    - isVerbose = isCurrModeVerbose;
    - setLayout(isVerbose, verboseLayout);
    - }
    + // layout is immutable in log4j2, so we cheat here and return a different layout when
    + // verbosity changes
    + if (isVerbose) {
    + return verboseLayout;
    + } else {
    + return layout == null ? nonVerboseLayout : layout;
          }
    - super.doAppend(event);
        }

    - /**
    - * Overrides WriterAppender.subAppend(), which does the real logging. No need
    - * to worry about concurrency since log4j calls this synchronously.
    - */
        @Override
    - protected void subAppend(LoggingEvent event) {
    - super.subAppend(event);
    - // That should've gone into our writer. Notify the LogContext.
    - String logOutput = writer.toString();
    - writer.reset();
    + public void append(LogEvent event) {
    + super.append(event);
    +
    + String logOutput = getOutput();
    + manager.reset();

          OperationLog log = operationManager.getOperationLogByThread();
          if (log == null) {
    @@ -206,4 +225,22 @@ public class LogDivertAppender extends WriterAppender {
          }
          log.writeOperationLog(logOutput);
        }
    +
    + protected static class StringOutputStreamManager extends OutputStreamManager {
    + ByteArrayOutputStream stream;
    +
    + protected StringOutputStreamManager(ByteArrayOutputStream os, String streamName,
    + Layout<?> layout) {
    + super(os, streamName, layout);
    + stream = os;
    + }
    +
    + public ByteArrayOutputStream getStream() {
    + return stream;
    + }
    +
    + public void reset() {
    + stream.reset();
    + }
    + }
      }

    http://git-wip-us.apache.org/repos/asf/hive/blob/c93d6c77/service/src/java/org/apache/hive/service/cli/operation/OperationManager.java
    ----------------------------------------------------------------------
    diff --git a/service/src/java/org/apache/hive/service/cli/operation/OperationManager.java b/service/src/java/org/apache/hive/service/cli/operation/OperationManager.java
    index 9b0a519..304a525 100644
    --- a/service/src/java/org/apache/hive/service/cli/operation/OperationManager.java
    +++ b/service/src/java/org/apache/hive/service/cli/operation/OperationManager.java
    @@ -41,8 +41,11 @@ import org.apache.hive.service.cli.RowSet;
      import org.apache.hive.service.cli.RowSetFactory;
      import org.apache.hive.service.cli.TableSchema;
      import org.apache.hive.service.cli.session.HiveSession;
    -import org.apache.log4j.Appender;
    -import org.apache.log4j.Logger;
    +import org.apache.logging.log4j.LogManager;
    +import org.apache.logging.log4j.core.Appender;
    +import org.apache.logging.log4j.core.LoggerContext;
    +import org.apache.logging.log4j.core.config.Configuration;
    +import org.apache.logging.log4j.core.config.LoggerConfig;

      /**
       * OperationManager.
    @@ -50,7 +53,6 @@ import org.apache.log4j.Logger;
       */
      public class OperationManager extends AbstractService {
        private final Log LOG = LogFactory.getLog(OperationManager.class.getName());
    -
        private final Map<OperationHandle, Operation> handleToOperation =
            new HashMap<OperationHandle, Operation>();

    @@ -83,8 +85,13 @@ public class OperationManager extends AbstractService {

        private void initOperationLogCapture(String loggingMode) {
          // Register another Appender (with the same layout) that talks to us.
    - Appender ap = new LogDivertAppender(this, OperationLog.getLoggingLevel(loggingMode));
    - Logger.getRootLogger().addAppender(ap);
    + Appender ap = LogDivertAppender.createInstance(this, OperationLog.getLoggingLevel(loggingMode));
    + LoggerContext context = (LoggerContext) LogManager.getContext(false);
    + Configuration configuration = context.getConfiguration();
    + LoggerConfig loggerConfig = configuration.getLoggerConfig(LogManager.getLogger().getName());
    + loggerConfig.addAppender(ap, null, null);
    + context.updateLoggers();
    + ap.start();
        }

        public ExecuteStatementOperation newExecuteStatementOperation(HiveSession parentSession,

    http://git-wip-us.apache.org/repos/asf/hive/blob/c93d6c77/shims/common/pom.xml
    ----------------------------------------------------------------------
    diff --git a/shims/common/pom.xml b/shims/common/pom.xml
    index 9e9a3b7..dfdec2b 100644
    --- a/shims/common/pom.xml
    +++ b/shims/common/pom.xml
    @@ -41,14 +41,19 @@
            <version>${commons-logging.version}</version>
          </dependency>
          <dependency>
    - <groupId>log4j</groupId>
    - <artifactId>log4j</artifactId>
    - <version>${log4j.version}</version>
    + <groupId>org.apache.logging.log4j</groupId>
    + <artifactId>log4j-1.2-api</artifactId>
    + <version>${log4j2.version}</version>
          </dependency>
          <dependency>
    - <groupId>log4j</groupId>
    - <artifactId>apache-log4j-extras</artifactId>
    - <version>${log4j-extras.version}</version>
    + <groupId>org.apache.logging.log4j</groupId>
    + <artifactId>log4j-slf4j-impl</artifactId>
    + <version>${log4j2.version}</version>
    + </dependency>
    + <dependency>
    + <groupId>org.apache.logging.log4j</groupId>
    + <artifactId>log4j-jcl</artifactId>
    + <version>${log4j2.version}</version>
          </dependency>
          <dependency>
            <groupId>com.google.guava</groupId>

    http://git-wip-us.apache.org/repos/asf/hive/blob/c93d6c77/shims/common/src/main/java/org/apache/hadoop/hive/shims/HiveEventCounter.java
    ----------------------------------------------------------------------
    diff --git a/shims/common/src/main/java/org/apache/hadoop/hive/shims/HiveEventCounter.java b/shims/common/src/main/java/org/apache/hadoop/hive/shims/HiveEventCounter.java
    deleted file mode 100644
    index 224b135..0000000
    --- a/shims/common/src/main/java/org/apache/hadoop/hive/shims/HiveEventCounter.java
    +++ /dev/null
    @@ -1,102 +0,0 @@
    -/**
    - * Licensed to the Apache Software Foundation (ASF) under one
    - * or more contributor license agreements. See the NOTICE file
    - * distributed with this work for additional information
    - * regarding copyright ownership. The ASF licenses this file
    - * to you under the Apache License, Version 2.0 (the
    - * "License"); you may not use this file except in compliance
    - * with the License. You may obtain a copy of the License at
    - *
    - * http://www.apache.org/licenses/LICENSE-2.0
    - *
    - * Unless required by applicable law or agreed to in writing, software
    - * distributed under the License is distributed on an "AS IS" BASIS,
    - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    - * See the License for the specific language governing permissions and
    - * limitations under the License.
    - */
    -
    -package org.apache.hadoop.hive.shims;
    -
    -import org.apache.log4j.Appender;
    -import org.apache.log4j.AppenderSkeleton;
    -import org.apache.log4j.Layout;
    -import org.apache.log4j.spi.ErrorHandler;
    -import org.apache.log4j.spi.Filter;
    -import org.apache.log4j.spi.LoggingEvent;
    -import org.apache.log4j.spi.OptionHandler;
    -
    -public class HiveEventCounter implements Appender, OptionHandler {
    -
    - AppenderSkeleton hadoopEventCounter;
    -
    - public HiveEventCounter() {
    - hadoopEventCounter = ShimLoader.getEventCounter();
    - }
    -
    - @Override
    - public void close() {
    - hadoopEventCounter.close();
    - }
    -
    - @Override
    - public boolean requiresLayout() {
    - return hadoopEventCounter.requiresLayout();
    - }
    -
    - @Override
    - public void addFilter(Filter filter) {
    - hadoopEventCounter.addFilter(filter);
    - }
    -
    - @Override
    - public void clearFilters() {
    - hadoopEventCounter.clearFilters();
    - }
    -
    - @Override
    - public void doAppend(LoggingEvent event) {
    - hadoopEventCounter.doAppend(event);
    - }
    -
    - @Override
    - public ErrorHandler getErrorHandler() {
    - return hadoopEventCounter.getErrorHandler();
    - }
    -
    - @Override
    - public Filter getFilter() {
    - return hadoopEventCounter.getFilter();
    - }
    -
    - @Override
    - public Layout getLayout() {
    - return hadoopEventCounter.getLayout();
    - }
    -
    - @Override
    - public String getName() {
    - return hadoopEventCounter.getName();
    - }
    -
    - @Override
    - public void setErrorHandler(ErrorHandler handler) {
    - hadoopEventCounter.setErrorHandler(handler);
    - }
    -
    - @Override
    - public void setLayout(Layout layout) {
    - hadoopEventCounter.setLayout(layout);
    - }
    -
    - @Override
    - public void setName(String name) {
    - hadoopEventCounter.setName(name);
    - }
    -
    - @Override
    - public void activateOptions() {
    - hadoopEventCounter.activateOptions();
    - }
    -
    -}

    http://git-wip-us.apache.org/repos/asf/hive/blob/c93d6c77/spark-client/src/test/resources/log4j.properties
    ----------------------------------------------------------------------
    diff --git a/spark-client/src/test/resources/log4j.properties b/spark-client/src/test/resources/log4j.properties
    deleted file mode 100644
    index 93a60cc..0000000
    --- a/spark-client/src/test/resources/log4j.properties
    +++ /dev/null
    @@ -1,23 +0,0 @@
    -#
    -# Licensed to the Apache Software Foundation (ASF) under one or more
    -# contributor license agreements. See the NOTICE file distributed with
    -# this work for additional information regarding copyright ownership.
    -# The ASF licenses this file to You under the Apache License, Version 2.0
    -# (the "License"); you may not use this file except in compliance with
    -# the License. You may obtain a copy of the License at
    -#
    -# http://www.apache.org/licenses/LICENSE-2.0
    -#
    -# Unless required by applicable law or agreed to in writing, software
    -# distributed under the License is distributed on an "AS IS" BASIS,
    -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    -# See the License for the specific language governing permissions and
    -# limitations under the License.
    -#
    -
    -# Set everything to be logged to the file target/unit-tests.log
    -log4j.rootCategory=DEBUG, console
    -log4j.appender.console=org.apache.log4j.ConsoleAppender
    -log4j.appender.console.target=System.err
    -log4j.appender.console.layout=org.apache.log4j.PatternLayout
    -log4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{1}: %m%n

    http://git-wip-us.apache.org/repos/asf/hive/blob/c93d6c77/spark-client/src/test/resources/log4j2.xml
    ----------------------------------------------------------------------
    diff --git a/spark-client/src/test/resources/log4j2.xml b/spark-client/src/test/resources/log4j2.xml
    new file mode 100644
    index 0000000..a435069
    --- /dev/null
    +++ b/spark-client/src/test/resources/log4j2.xml
    @@ -0,0 +1,39 @@
    +<?xml version="1.0" encoding="UTF-8"?>
    +<!--
    + Licensed to the Apache Software Foundation (ASF) under one or more
    + contributor license agreements. See the NOTICE file distributed with
    + this work for additional information regarding copyright ownership.
    + The ASF licenses this file to You under the Apache License, Version 2.0
    + (the "License"); you may not use this file except in compliance with
    + the License. You may obtain a copy of the License at
    +
    + http://www.apache.org/licenses/LICENSE-2.0
    +
    + Unless required by applicable law or agreed to in writing, software
    + distributed under the License is distributed on an "AS IS" BASIS,
    + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    + See the License for the specific language governing permissions and
    + limitations under the License.
    +-->
    +
    +<Configuration status="info" strict="true" name="SparkClientLog4j2"
    + packages="org.apache.hadoop.hive.ql.log">
    +
    + <Properties>
    + <Property name="spark.log.level">DEBUG</Property>
    + <Property name="spark.root.logger">console</Property>
    + </Properties>
    +
    + <Appenders>
    + <Console name="console" target="SYSTEM_ERR">
    + <PatternLayout pattern="%d{yy/MM/dd HH:mm:ss} %p %c{1}: %m%n"/>
    + </Console>
    + </Appenders>
    +
    + <Loggers>
    + <Root level="DEBUG">
    + <AppenderRef ref="${sys:spark.root.logger}" level="${sys:spark.log.level}"/>
    + </Root>
    + </Loggers>
    +
    +</Configuration>

    http://git-wip-us.apache.org/repos/asf/hive/blob/c93d6c77/storage-api/pom.xml
    ----------------------------------------------------------------------
    diff --git a/storage-api/pom.xml b/storage-api/pom.xml
    index 71b51b8..71b79f1 100644
    --- a/storage-api/pom.xml
    +++ b/storage-api/pom.xml
    @@ -32,13 +32,6 @@
        </properties>

        <dependencies>
    - <!-- dependencies are always listed in sorted order by groupId, artifectId -->
    - <!-- inter-project -->
    - <dependency>
    - <groupId>log4j</groupId>
    - <artifactId>log4j</artifactId>
    - <version>${log4j.version}</version>
    - </dependency>
          <!-- test inter-project -->
          <dependency>
            <groupId>junit</groupId>

    http://git-wip-us.apache.org/repos/asf/hive/blob/c93d6c77/testutils/ptest2/pom.xml
    ----------------------------------------------------------------------
    diff --git a/testutils/ptest2/pom.xml b/testutils/ptest2/pom.xml
    index 211678e..2cf7f45 100644
    --- a/testutils/ptest2/pom.xml
    +++ b/testutils/ptest2/pom.xml
    @@ -64,6 +64,26 @@ limitations under the License.
            <version>15.0</version>
          </dependency>
          <dependency>
    + <groupId>org.apache.logging.log4j</groupId>
    + <artifactId>log4j-1.2-api</artifactId>
    + <version>${log4j2.version}</version>
    + </dependency>
    + <dependency>
    + <groupId>org.apache.logging.log4j</groupId>
    + <artifactId>log4j-web</artifactId>
    + <version>${log4j2.version}</version>
    + </dependency>
    + <dependency>
    + <groupId>org.apache.logging.log4j</groupId>
    + <artifactId>log4j-slf4j-impl</artifactId>
    + <version>${log4j2.version}</version>
    + </dependency>
    + <dependency>
    + <groupId>org.apache.logging.log4j</groupId>
    + <artifactId>log4j-jcl</artifactId>
    + <version>${log4j2.version}</version>
    + </dependency>
    + <dependency>
            <groupId>log4j</groupId>
            <artifactId>log4j</artifactId>
            <version>1.2.17</version>

    http://git-wip-us.apache.org/repos/asf/hive/blob/c93d6c77/testutils/ptest2/src/main/resources/log4j.properties
    ----------------------------------------------------------------------
    diff --git a/testutils/ptest2/src/main/resources/log4j.properties b/testutils/ptest2/src/main/resources/log4j.properties
    deleted file mode 100644
    index edb9696..0000000
    --- a/testutils/ptest2/src/main/resources/log4j.properties
    +++ /dev/null
    @@ -1,37 +0,0 @@
    -# Licensed to the Apache Software Foundation (ASF) under one or more
    -# contributor license agreements. See the NOTICE file distributed with
    -# this work for additional information regarding copyright ownership.
    -# The ASF licenses this file to You under the Apache License, Version 2.0
    -# (the "License"); you may not use this file except in compliance with
    -# the License. You may obtain a copy of the License at
    -#
    -# http://www.apache.org/licenses/LICENSE-2.0
    -#
    -# Unless required by applicable law or agreed to in writing, software
    -# distributed under the License is distributed on an "AS IS" BASIS,
    -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    -# See the License for the specific language governing permissions and
    -# limitations under the License.
    -
    -hive.ptest.logdir=target
    -
    -log4j.rootLogger=DEBUG,FILE
    -log4j.threshhold=ALL
    -
    -log4j.appender.FILE=org.apache.log4j.RollingFileAppender
    -log4j.appender.FILE.File=${hive.ptest.logdir}/ptest.log
    -log4j.appender.FILE.MaxFileSize=50MB
    -log4j.appender.FILE.MaxBackupIndex=1
    -log4j.appender.FILE.layout=org.apache.log4j.PatternLayout
    -log4j.appender.FILE.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n
    -
    -log4j.logger.org.apache.http=INFO
    -log4j.logger.org.springframework=INFO
    -log4j.logger.org.jclouds=INFO
    -log4j.logger.jclouds=INFO
    -log4j.logger.org.apache.hive=DEBUG
    -log4j.logger.org.apache.http=TRACE
    -
    -# Silence useless ZK logs
    -log4j.logger.org.apache.zookeeper.server.NIOServerCnxn=WARN
    -log4j.logger.org.apache.zookeeper.ClientCnxnSocketNIO=WARN
  • Sershe at Aug 25, 2015 at 9:28 pm
    http://git-wip-us.apache.org/repos/asf/hive/blob/c93d6c77/testutils/ptest2/src/main/resources/log4j2.xml
    ----------------------------------------------------------------------
    diff --git a/testutils/ptest2/src/main/resources/log4j2.xml b/testutils/ptest2/src/main/resources/log4j2.xml
    new file mode 100644
    index 0000000..992462e
    --- /dev/null
    +++ b/testutils/ptest2/src/main/resources/log4j2.xml
    @@ -0,0 +1,79 @@
    +<?xml version="1.0" encoding="UTF-8"?>
    +<!--
    + Licensed to the Apache Software Foundation (ASF) under one or more
    + contributor license agreements. See the NOTICE file distributed with
    + this work for additional information regarding copyright ownership.
    + The ASF licenses this file to You under the Apache License, Version 2.0
    + (the "License"); you may not use this file except in compliance with
    + the License. You may obtain a copy of the License at
    +
    + http://www.apache.org/licenses/LICENSE-2.0
    +
    + Unless required by applicable law or agreed to in writing, software
    + distributed under the License is distributed on an "AS IS" BASIS,
    + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    + See the License for the specific language governing permissions and
    + limitations under the License.
    +-->
    +
    +<Configuration status="info" strict="true" name="PTest2Log4j2"
    + packages="org.apache.hadoop.hive.ql.log">
    +
    + <Properties>
    + <Property name="hive.ptest.log.threshold">ALL</Property>
    + <Property name="hive.ptest.log.level">DEBUG</Property>
    + <Property name="hive.ptest.root.logger">FILE</Property>
    + <Property name="hive.ptest.log.dir">target</Property>
    + <Property name="hive.ptest.log.file">ptest.log</Property>
    + </Properties>
    +
    + <Appenders>
    + <Console name="console" target="SYSTEM_ERR">
    + <PatternLayout pattern="%d{yy/MM/dd HH:mm:ss} %p %c{2}: %m%n"/>
    + </Console>
    +
    + <!-- Size based Rolling File Appender -->
    + <RollingFile name="FILE" fileName="${sys:hive.ptest.log.dir}/${sys:hive.ptest.log.file}"
    + filePattern="${sys:hive.ptest.log.dir}/${sys:hive.ptest.log.file}.%i">
    + <PatternLayout pattern="%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n" />
    + <Policies>
    + <SizeBasedTriggeringPolicy size="50 MB" />
    + </Policies>
    + <DefaultRolloverStrategy max="1"/>
    + </RollingFile>
    +
    + </Appenders>
    +
    + <Loggers>
    + <Root level="${sys:hive.ptest.log.threshold}">
    + <AppenderRef ref="${sys:hive.ptest.root.logger}" level="${sys:hive.ptest.log.level}"/>
    + </Root>
    +
    + <Logger name="org.apache.http" level="INFO">
    + <AppenderRef ref="${sys:hive.ptest.root.logger}"/>
    + </Logger>
    + <Logger name="org.springframework" level="INFO">
    + <AppenderRef ref="${sys:hive.ptest.root.logger}"/>
    + </Logger>
    + <Logger name="org.jclouds" level="INFO">
    + <AppenderRef ref="${sys:hive.ptest.root.logger}"/>
    + </Logger>
    + <Logger name="jclouds" level="INFO">
    + <AppenderRef ref="${sys:hive.ptest.root.logger}"/>
    + </Logger>
    + <Logger name="org.apache.hive" level="DEBUG">
    + <AppenderRef ref="${sys:hive.ptest.root.logger}"/>
    + </Logger>
    + <Logger name="org.apache.http" level="TRACE">
    + <AppenderRef ref="${sys:hive.ptest.root.logger}"/>
    + </Logger>
    +
    + <!-- Silence useless ZK logs -->
    + <Logger name="org.apache.zookeeper.server.NIOServerCnxn" level="WARN">
    + <AppenderRef ref="${sys:hive.ptest.root.logger}"/>
    + </Logger>
    + <Logger name="org.apache.zookeeper.ClientCnxnSocketNIO" level="WARN">
    + <AppenderRef ref="${sys:hive.ptest.root.logger}"/>
    + </Logger>
    +
    +</Configuration>
  • Sershe at Aug 25, 2015 at 9:28 pm
    HIVE-11546: Projected columns read size should be scaled to split size for ORC Splits (Prasanth Jayachandran reviewed by Sergey Shelukhin)


    Project: http://git-wip-us.apache.org/repos/asf/hive/repo
    Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/e8b2c605
    Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/e8b2c605
    Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/e8b2c605

    Branch: refs/heads/llap
    Commit: e8b2c605a05d06c1bfcd4c6bc611bc7f83306b38
    Parents: cf0481f
    Author: Prasanth Jayachandran <j.prasanth.j@gmail.com>
    Authored: Fri Aug 14 14:21:51 2015 -0700
    Committer: Prasanth Jayachandran <j.prasanth.j@gmail.com>
    Committed: Fri Aug 14 14:21:51 2015 -0700

    ----------------------------------------------------------------------
      .../hadoop/hive/ql/io/orc/OrcInputFormat.java | 30 ++++++-
      .../apache/hadoop/hive/ql/io/orc/Reader.java | 7 ++
      .../hadoop/hive/ql/io/orc/ReaderImpl.java | 5 +-
      .../hive/ql/io/orc/TestInputOutputFormat.java | 95 ++++++++++++++++++--
      4 files changed, 125 insertions(+), 12 deletions(-)
    ----------------------------------------------------------------------


    http://git-wip-us.apache.org/repos/asf/hive/blob/e8b2c605/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java
    ----------------------------------------------------------------------
    diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java
    index fe2eccd..6ed7872 100644
    --- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java
    +++ b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java
    @@ -717,6 +717,7 @@ public class OrcInputFormat implements InputFormat<NullWritable, OrcStruct>,
          private ReaderImpl.FileMetaInfo fileMetaInfo;
          private Metadata metadata;
          private List<OrcProto.Type> types;
    + private boolean[] includedCols;
          private final boolean isOriginal;
          private final List<DeltaMetaData> deltas;
          private final boolean hasBase;
    @@ -830,8 +831,14 @@ public class OrcInputFormat implements InputFormat<NullWritable, OrcStruct>,
              hosts = new String[hostList.size()];
              hostList.toArray(hosts);
            }
    +
    + // scale the raw data size to split level based on ratio of split wrt to file length
    + final long fileLen = file.getLen();
    + final double splitRatio = (double) length / (double) fileLen;
    + final long scaledProjSize = projColsUncompressedSize > 0 ?
    + (long) (splitRatio * projColsUncompressedSize) : fileLen;
            return new OrcSplit(file.getPath(), offset, length, hosts, fileMetaInfo,
    - isOriginal, hasBase, deltas, projColsUncompressedSize);
    + isOriginal, hasBase, deltas, scaledProjSize);
          }

          /**
    @@ -845,11 +852,12 @@ public class OrcInputFormat implements InputFormat<NullWritable, OrcStruct>,

            // figure out which stripes we need to read
            boolean[] includeStripe = null;
    +
            // we can't eliminate stripes if there are deltas because the
            // deltas may change the rows making them match the predicate.
            if (deltas.isEmpty()) {
              Reader.Options options = new Reader.Options();
    - options.include(genIncludedColumns(types, context.conf, isOriginal));
    + options.include(includedCols);
              setSearchArgument(options, types, context.conf, isOriginal);
              // only do split pruning if HIVE-8732 has been fixed in the writer
              if (options.getSearchArgument() != null &&
    @@ -930,8 +938,6 @@ public class OrcInputFormat implements InputFormat<NullWritable, OrcStruct>,
          private void populateAndCacheStripeDetails() throws IOException {
            Reader orcReader = OrcFile.createReader(file.getPath(),
                OrcFile.readerOptions(context.conf).filesystem(fs));
    - List<String> projCols = ColumnProjectionUtils.getReadColumnNames(context.conf);
    - // TODO: produce projColsUncompressedSize from projCols
            if (fileInfo != null) {
              stripes = fileInfo.stripeInfos;
              fileMetaInfo = fileInfo.fileMetaInfo;
    @@ -959,6 +965,22 @@ public class OrcInputFormat implements InputFormat<NullWritable, OrcStruct>,
                        metadata, types, fileMetaInfo, writerVersion));
              }
            }
    + includedCols = genIncludedColumns(types, context.conf, isOriginal);
    + projColsUncompressedSize = computeProjectionSize(orcReader, includedCols, isOriginal);
    + }
    +
    + private long computeProjectionSize(final Reader orcReader, final boolean[] includedCols,
    + final boolean isOriginal) {
    + final int rootIdx = getRootColumn(isOriginal);
    + List<Integer> internalColIds = Lists.newArrayList();
    + if (includedCols != null) {
    + for (int i = 0; i < includedCols.length; i++) {
    + if (includedCols[i]) {
    + internalColIds.add(rootIdx + i);
    + }
    + }
    + }
    + return orcReader.getRawDataSizeFromColIndices(internalColIds);
          }

          private boolean isStripeSatisfyPredicate(StripeStatistics stripeStatistics,

    http://git-wip-us.apache.org/repos/asf/hive/blob/e8b2c605/ql/src/java/org/apache/hadoop/hive/ql/io/orc/Reader.java
    ----------------------------------------------------------------------
    diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/Reader.java b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/Reader.java
    index 6f4f013..7bddefc 100644
    --- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/Reader.java
    +++ b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/Reader.java
    @@ -52,6 +52,13 @@ public interface Reader {
        long getRawDataSizeOfColumns(List<String> colNames);

        /**
    + * Get the deserialized data size of the specified columns ids
    + * @param colIds - internal column id (check orcfiledump for column ids)
    + * @return raw data size of columns
    + */
    + long getRawDataSizeFromColIndices(List<Integer> colIds);
    +
    + /**
         * Get the user metadata keys.
         * @return the set of metadata keys
         */

    http://git-wip-us.apache.org/repos/asf/hive/blob/e8b2c605/ql/src/java/org/apache/hadoop/hive/ql/io/orc/ReaderImpl.java
    ----------------------------------------------------------------------
    diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/ReaderImpl.java b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/ReaderImpl.java
    index 3c0de3c..a6448b6 100644
    --- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/ReaderImpl.java
    +++ b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/ReaderImpl.java
    @@ -575,7 +575,8 @@ public class ReaderImpl implements Reader {
          return deserializedSize;
        }

    - private long getRawDataSizeFromColIndices(List<Integer> colIndices) {
    + @Override
    + public long getRawDataSizeFromColIndices(List<Integer> colIndices) {
          long result = 0;
          for (int colIdx : colIndices) {
            result += getRawDataSizeOfColumn(colIdx);
    @@ -620,7 +621,7 @@ public class ReaderImpl implements Reader {
          case BYTE:
            return numVals * JavaDataModel.get().primitive1();
          default:
    - LOG.debug("Unknown primitive category.");
    + LOG.debug("Unknown primitive category: " + type.getKind());
            break;
          }


    http://git-wip-us.apache.org/repos/asf/hive/blob/e8b2c605/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestInputOutputFormat.java
    ----------------------------------------------------------------------
    diff --git a/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestInputOutputFormat.java b/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestInputOutputFormat.java
    index 6cb8529..0c12c89 100644
    --- a/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestInputOutputFormat.java
    +++ b/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestInputOutputFormat.java
    @@ -17,9 +17,6 @@
       */
      package org.apache.hadoop.hive.ql.io.orc;

    -import com.esotericsoftware.kryo.Kryo;
    -import com.esotericsoftware.kryo.io.Output;
    -
      import static org.junit.Assert.assertArrayEquals;
      import static org.junit.Assert.assertEquals;
      import static org.junit.Assert.assertTrue;
    @@ -108,6 +105,9 @@ import org.junit.Rule;
      import org.junit.Test;
      import org.junit.rules.TestName;

    +import com.esotericsoftware.kryo.Kryo;
    +import com.esotericsoftware.kryo.io.Output;
    +
      public class TestInputOutputFormat {

        public static String toKryo(SearchArgument sarg) {
    @@ -902,14 +902,25 @@ public class TestInputOutputFormat {
          }
          fill(buffer, offset);
          footer.addTypes(OrcProto.Type.newBuilder()
    - .setKind(OrcProto.Type.Kind.STRUCT)
    - .addFieldNames("col1")
    - .addSubtypes(1));
    + .setKind(OrcProto.Type.Kind.STRUCT)
    + .addFieldNames("col1")
    + .addSubtypes(1));
          footer.addTypes(OrcProto.Type.newBuilder()
              .setKind(OrcProto.Type.Kind.STRING));
          footer.setNumberOfRows(1000 * stripeLengths.length)
                .setHeaderLength(headerLen)
                .setContentLength(offset - headerLen);
    + footer.addStatistics(OrcProto.ColumnStatistics.newBuilder()
    + .setNumberOfValues(1000 * stripeLengths.length).build());
    + footer.addStatistics(OrcProto.ColumnStatistics.newBuilder()
    + .setNumberOfValues(1000 * stripeLengths.length)
    + .setStringStatistics(
    + OrcProto.StringStatistics.newBuilder()
    + .setMaximum("zzz")
    + .setMinimum("aaa")
    + .setSum(1000 * 3 * stripeLengths.length)
    + .build()
    + ).build());
          footer.build().writeTo(buffer);
          int footerEnd = buffer.getLength();
          OrcProto.PostScript ps =
    @@ -1013,6 +1024,78 @@ public class TestInputOutputFormat {
        }

        @Test
    + public void testProjectedColumnSize() throws Exception {
    + long[] stripeSizes =
    + new long[]{200, 200, 200, 200, 100};
    + MockFileSystem fs = new MockFileSystem(conf,
    + new MockFile("mock:/a/file", 500,
    + createMockOrcFile(stripeSizes),
    + new MockBlock("host1-1", "host1-2", "host1-3"),
    + new MockBlock("host2-1", "host0", "host2-3"),
    + new MockBlock("host0", "host3-2", "host3-3"),
    + new MockBlock("host4-1", "host4-2", "host4-3"),
    + new MockBlock("host5-1", "host5-2", "host5-3")));
    + conf.setInt(OrcInputFormat.MAX_SPLIT_SIZE, 300);
    + conf.setInt(OrcInputFormat.MIN_SPLIT_SIZE, 200);
    + conf.setBoolean(ColumnProjectionUtils.READ_ALL_COLUMNS, false);
    + conf.set(ColumnProjectionUtils.READ_COLUMN_IDS_CONF_STR, "0");
    + OrcInputFormat.Context context = new OrcInputFormat.Context(conf);
    + OrcInputFormat.SplitGenerator splitter =
    + new OrcInputFormat.SplitGenerator(new OrcInputFormat.SplitInfo(context, fs,
    + fs.getFileStatus(new Path("/a/file")), null, true,
    + new ArrayList<AcidInputFormat.DeltaMetaData>(), true, null, null));
    + List<OrcSplit> results = splitter.call();
    + OrcSplit result = results.get(0);
    + assertEquals(3, results.size());
    + assertEquals(3, result.getStart());
    + assertEquals(400, result.getLength());
    + assertEquals(167468, result.getProjectedColumnsUncompressedSize());
    + result = results.get(1);
    + assertEquals(403, result.getStart());
    + assertEquals(400, result.getLength());
    + assertEquals(167468, result.getProjectedColumnsUncompressedSize());
    + result = results.get(2);
    + assertEquals(803, result.getStart());
    + assertEquals(100, result.getLength());
    + assertEquals(41867, result.getProjectedColumnsUncompressedSize());
    +
    + // test min = 0, max = 0 generates each stripe
    + conf.setInt(OrcInputFormat.MIN_SPLIT_SIZE, 0);
    + conf.setInt(OrcInputFormat.MAX_SPLIT_SIZE, 0);
    + context = new OrcInputFormat.Context(conf);
    + splitter = new OrcInputFormat.SplitGenerator(new OrcInputFormat.SplitInfo(context, fs,
    + fs.getFileStatus(new Path("/a/file")), null, true,
    + new ArrayList<AcidInputFormat.DeltaMetaData>(),
    + true, null, null));
    + results = splitter.call();
    + assertEquals(5, results.size());
    + for (int i = 0; i < stripeSizes.length; ++i) {
    + assertEquals("checking stripe " + i + " size",
    + stripeSizes[i], results.get(i).getLength());
    + if (i == stripeSizes.length - 1) {
    + assertEquals(41867, results.get(i).getProjectedColumnsUncompressedSize());
    + } else {
    + assertEquals(83734, results.get(i).getProjectedColumnsUncompressedSize());
    + }
    + }
    +
    + // single split
    + conf.setInt(OrcInputFormat.MIN_SPLIT_SIZE, 100000);
    + conf.setInt(OrcInputFormat.MAX_SPLIT_SIZE, 1000);
    + context = new OrcInputFormat.Context(conf);
    + splitter = new OrcInputFormat.SplitGenerator(new OrcInputFormat.SplitInfo(context, fs,
    + fs.getFileStatus(new Path("/a/file")), null, true,
    + new ArrayList<AcidInputFormat.DeltaMetaData>(),
    + true, null, null));
    + results = splitter.call();
    + assertEquals(1, results.size());
    + result = results.get(0);
    + assertEquals(3, result.getStart());
    + assertEquals(900, result.getLength());
    + assertEquals(376804, result.getProjectedColumnsUncompressedSize());
    + }
    +
    + @Test
        @SuppressWarnings("unchecked,deprecation")
        public void testInOutFormat() throws Exception {
          Properties properties = new Properties();
  • Sershe at Aug 25, 2015 at 9:28 pm
    HIVE-11563: Perflogger loglines are repeated (Prasanth Jayachandran reviewed by Sergey Shelukhin)


    Project: http://git-wip-us.apache.org/repos/asf/hive/repo
    Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/0fab86c9
    Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/0fab86c9
    Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/0fab86c9

    Branch: refs/heads/llap
    Commit: 0fab86c9d7730275cb9962c24ce9d2f92f0c7150
    Parents: d307abb
    Author: Prasanth Jayachandran <j.prasanth.j@gmail.com>
    Authored: Fri Aug 14 14:02:51 2015 -0700
    Committer: Prasanth Jayachandran <j.prasanth.j@gmail.com>
    Committed: Fri Aug 14 14:02:51 2015 -0700

    ----------------------------------------------------------------------
      data/conf/hive-log4j2.xml | 2 +-
      1 file changed, 1 insertion(+), 1 deletion(-)
    ----------------------------------------------------------------------


    http://git-wip-us.apache.org/repos/asf/hive/blob/0fab86c9/data/conf/hive-log4j2.xml
    ----------------------------------------------------------------------
    diff --git a/data/conf/hive-log4j2.xml b/data/conf/hive-log4j2.xml
    index a40be2c..ff8e4d3 100644
    --- a/data/conf/hive-log4j2.xml
    +++ b/data/conf/hive-log4j2.xml
    @@ -95,7 +95,7 @@
          <Logger name="org.apache.zookeeper.ClientCnxnSocketNIO" level="WARN">
            <AppenderRef ref="${sys:hive.root.logger}"/>
          </Logger>
    - <Logger name="org.apache.hadoop.hive.ql.log.PerfLogger" level="${sys:hive.ql.log.PerfLogger.level}">
    + <Logger name="org.apache.hadoop.hive.ql.log.PerfLogger" level="${sys:hive.ql.log.PerfLogger.level}" additivity="false">
            <AppenderRef ref="${sys:hive.root.logger}"/>
          </Logger>
          <Logger name="org.apache.hadoop.hive.ql.exec.Operator" level="INFO">
  • Sershe at Aug 25, 2015 at 9:28 pm
    HIVE-11549: Hide Hive configuration from spark driver launching process (reviewed by Chao)


    Project: http://git-wip-us.apache.org/repos/asf/hive/repo
    Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/fe1efe52
    Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/fe1efe52
    Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/fe1efe52

    Branch: refs/heads/llap
    Commit: fe1efe520b807f24085d87fea8fc1c5ee0c9e44c
    Parents: 2ccd061
    Author: Xuefu Zhang <xzhang@cloudera.com>
    Authored: Fri Aug 14 15:30:52 2015 -0700
    Committer: Xuefu Zhang <xzhang@cloudera.com>
    Committed: Fri Aug 14 15:30:52 2015 -0700

    ----------------------------------------------------------------------
      .../main/java/org/apache/hive/spark/client/SparkClientImpl.java | 4 ++++
      1 file changed, 4 insertions(+)
    ----------------------------------------------------------------------


    http://git-wip-us.apache.org/repos/asf/hive/blob/fe1efe52/spark-client/src/main/java/org/apache/hive/spark/client/SparkClientImpl.java
    ----------------------------------------------------------------------
    diff --git a/spark-client/src/main/java/org/apache/hive/spark/client/SparkClientImpl.java b/spark-client/src/main/java/org/apache/hive/spark/client/SparkClientImpl.java
    index e1e64a7..2546a46 100644
    --- a/spark-client/src/main/java/org/apache/hive/spark/client/SparkClientImpl.java
    +++ b/spark-client/src/main/java/org/apache/hive/spark/client/SparkClientImpl.java
    @@ -423,6 +423,10 @@ class SparkClientImpl implements SparkClient {
            LOG.info("Running client driver with argv: {}", cmd);
            ProcessBuilder pb = new ProcessBuilder("sh", "-c", cmd);

    + // Prevent hive configurations from being visible in Spark.
    + pb.environment().remove("HIVE_HOME");
    + pb.environment().remove("HIVE_CONF_DIR");
    +
            if (isTesting != null) {
              pb.environment().put("SPARK_TESTING", isTesting);
            }
  • Sershe at Aug 25, 2015 at 9:28 pm
    HIVE-11304: Migrate to Log4j2 from Log4j 1.x (Prasanth Jayachandran reviewed by Thejas Nair, Sergey Shelukhin)


    Project: http://git-wip-us.apache.org/repos/asf/hive/repo
    Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/c93d6c77
    Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/c93d6c77
    Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/c93d6c77

    Branch: refs/heads/llap
    Commit: c93d6c77e31e2eb9b40f5167ab3491d44eae351a
    Parents: a4849cb
    Author: Prasanth Jayachandran <j.prasanth.j@gmail.com>
    Authored: Fri Aug 14 10:17:20 2015 -0700
    Committer: Prasanth Jayachandran <j.prasanth.j@gmail.com>
    Committed: Fri Aug 14 10:17:20 2015 -0700

    ----------------------------------------------------------------------
      accumulo-handler/pom.xml | 4 -
      .../src/main/resources/beeline-log4j.properties | 24 --
      beeline/src/main/resources/beeline-log4j2.xml | 40 ++++
      bin/ext/beeline.sh | 2 +-
      bin/hive | 3 +
      common/pom.xml | 22 +-
      .../org/apache/hadoop/hive/common/LogUtils.java | 18 +-
      .../org/apache/hadoop/hive/conf/HiveConf.java | 8 +-
      common/src/main/resources/hive-log4j.properties | 88 --------
      common/src/main/resources/hive-log4j2.xml | 111 +++++++++
      .../hadoop/hive/conf/TestHiveLogging.java | 8 +-
      .../resources/hive-exec-log4j-test.properties | 59 -----
      .../test/resources/hive-exec-log4j2-test.xml | 86 +++++++
      .../test/resources/hive-log4j-test.properties | 71 ------
      common/src/test/resources/hive-log4j2-test.xml | 95 ++++++++
      data/conf/hive-log4j-old.properties | 82 -------
      data/conf/hive-log4j.properties | 97 --------
      data/conf/hive-log4j2.xml | 148 ++++++++++++
      data/conf/spark/log4j.properties | 24 --
      data/conf/spark/log4j2.xml | 74 ++++++
      docs/xdocs/language_manual/cli.xml | 2 +-
      hcatalog/bin/hcat_server.sh | 2 +-
      hcatalog/bin/templeton.cmd | 4 +-
      hcatalog/scripts/hcat_server_start.sh | 2 +-
      .../content/xdocs/configuration.xml | 2 +-
      .../src/documentation/content/xdocs/install.xml | 2 +-
      .../deployers/config/hive/hive-log4j.properties | 88 --------
      .../deployers/config/hive/hive-log4j2.xml | 111 +++++++++
      .../templeton/deployers/start_hive_services.sh | 2 +-
      .../webhcat/svr/src/main/bin/webhcat_server.sh | 4 +-
      .../src/main/config/webhcat-log4j.properties | 45 ----
      .../svr/src/main/config/webhcat-log4j2.xml | 75 +++++++
      .../main/java/org/apache/hive/hplsql/Exec.java | 2 +
      .../operation/TestOperationLoggingLayout.java | 136 +++++++++++
      itests/pom.xml | 2 +-
      .../hadoop/hive/metastore/HiveMetaStore.java | 2 +-
      .../metastore/txn/TestCompactionTxnHandler.java | 40 +++-
      .../hive/metastore/txn/TestTxnHandler.java | 66 ++++--
      packaging/src/main/assembly/bin.xml | 14 +-
      pom.xml | 37 ++-
      ql/pom.xml | 17 +-
      .../hadoop/hive/ql/exec/mr/ExecDriver.java | 29 ++-
      .../hive/ql/exec/mr/HadoopJobExecHelper.java | 20 +-
      .../ql/io/rcfile/stats/PartialScanTask.java | 20 +-
      .../hadoop/hive/ql/log/HiveEventCounter.java | 135 +++++++++++
      .../apache/hadoop/hive/ql/log/NullAppender.java | 63 ++++++
      .../ql/log/PidDailyRollingFileAppender.java | 33 ---
      .../hive/ql/log/PidFilePatternConverter.java | 62 ++++++
      .../main/resources/hive-exec-log4j.properties | 77 -------
      ql/src/main/resources/hive-exec-log4j2.xml | 110 +++++++++
      .../hadoop/hive/ql/log/TestLog4j2Appenders.java | 95 ++++++++
      .../hadoop/hive/ql/metadata/StringAppender.java | 128 +++++++++++
      .../hadoop/hive/ql/metadata/TestHive.java | 50 +++--
      .../hive/service/cli/CLIServiceUtils.java | 7 -
      .../cli/operation/LogDivertAppender.java | 223 +++++++++++--------
      .../service/cli/operation/OperationManager.java | 17 +-
      shims/common/pom.xml | 17 +-
      .../hadoop/hive/shims/HiveEventCounter.java | 102 ---------
      .../src/test/resources/log4j.properties | 23 --
      spark-client/src/test/resources/log4j2.xml | 39 ++++
      storage-api/pom.xml | 7 -
      testutils/ptest2/pom.xml | 20 ++
      .../ptest2/src/main/resources/log4j.properties | 37 ---
      testutils/ptest2/src/main/resources/log4j2.xml | 79 +++++++
      64 files changed, 1989 insertions(+), 1123 deletions(-)
    ----------------------------------------------------------------------


    http://git-wip-us.apache.org/repos/asf/hive/blob/c93d6c77/accumulo-handler/pom.xml
    ----------------------------------------------------------------------
    diff --git a/accumulo-handler/pom.xml b/accumulo-handler/pom.xml
    index ee40f72..4e3a087 100644
    --- a/accumulo-handler/pom.xml
    +++ b/accumulo-handler/pom.xml
    @@ -91,10 +91,6 @@
            <artifactId>slf4j-api</artifactId>
          </dependency>
          <dependency>
    - <groupId>org.slf4j</groupId>
    - <artifactId>slf4j-log4j12</artifactId>
    - </dependency>
    - <dependency>
            <groupId>junit</groupId>
            <artifactId>junit</artifactId>
            <scope>test</scope>

    http://git-wip-us.apache.org/repos/asf/hive/blob/c93d6c77/beeline/src/main/resources/beeline-log4j.properties
    ----------------------------------------------------------------------
    diff --git a/beeline/src/main/resources/beeline-log4j.properties b/beeline/src/main/resources/beeline-log4j.properties
    deleted file mode 100644
    index fe47d94..0000000
    --- a/beeline/src/main/resources/beeline-log4j.properties
    +++ /dev/null
    @@ -1,24 +0,0 @@
    -# Licensed to the Apache Software Foundation (ASF) under one
    -# or more contributor license agreements. See the NOTICE file
    -# distributed with this work for additional information
    -# regarding copyright ownership. The ASF licenses this file
    -# to you under the Apache License, Version 2.0 (the
    -# "License"); you may not use this file except in compliance
    -# with the License. You may obtain a copy of the License at
    -#
    -# http://www.apache.org/licenses/LICENSE-2.0
    -#
    -# Unless required by applicable law or agreed to in writing, software
    -# distributed under the License is distributed on an "AS IS" BASIS,
    -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    -# See the License for the specific language governing permissions and
    -# limitations under the License.
    -
    -log4j.rootLogger=WARN, console
    -
    -######## console appender ########
    -log4j.appender.console=org.apache.log4j.ConsoleAppender
    -log4j.appender.console.target=System.err
    -log4j.appender.console.layout=org.apache.log4j.PatternLayout
    -log4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} [%t]: %p %c{2}: %m%n
    -log4j.appender.console.encoding=UTF-8

    http://git-wip-us.apache.org/repos/asf/hive/blob/c93d6c77/beeline/src/main/resources/beeline-log4j2.xml
    ----------------------------------------------------------------------
    diff --git a/beeline/src/main/resources/beeline-log4j2.xml b/beeline/src/main/resources/beeline-log4j2.xml
    new file mode 100644
    index 0000000..5f09741
    --- /dev/null
    +++ b/beeline/src/main/resources/beeline-log4j2.xml
    @@ -0,0 +1,40 @@
    +<?xml version="1.0" encoding="UTF-8"?>
    +<!--
    + Licensed to the Apache Software Foundation (ASF) under one or more
    + contributor license agreements. See the NOTICE file distributed with
    + this work for additional information regarding copyright ownership.
    + The ASF licenses this file to You under the Apache License, Version 2.0
    + (the "License"); you may not use this file except in compliance with
    + the License. You may obtain a copy of the License at
    +
    + http://www.apache.org/licenses/LICENSE-2.0
    +
    + Unless required by applicable law or agreed to in writing, software
    + distributed under the License is distributed on an "AS IS" BASIS,
    + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    + See the License for the specific language governing permissions and
    + limitations under the License.
    +-->
    +
    +<Configuration status="info" strict="true" name="BeelineLog4j2"
    + packages="org.apache.hadoop.hive.ql.log">
    +
    + <Properties>
    + <Property name="hive.log.threshold">ALL</Property>
    + <Property name="hive.log.level">WARN</Property>
    + <Property name="hive.root.logger">console</Property>
    + </Properties>
    +
    + <Appenders>
    + <Console name="console" target="SYSTEM_ERR">
    + <PatternLayout pattern="%d{yy/MM/dd HH:mm:ss} [%t]: %p %c{2}: %m%n"/>
    + </Console>
    + </Appenders>
    +
    + <Loggers>
    + <Root level="${sys:hive.log.threshold}">
    + <AppenderRef ref="${sys:hive.root.logger}" level="${sys:hive.log.level}"/>
    + </Root>
    + </Loggers>
    +
    +</Configuration>

    http://git-wip-us.apache.org/repos/asf/hive/blob/c93d6c77/bin/ext/beeline.sh
    ----------------------------------------------------------------------
    diff --git a/bin/ext/beeline.sh b/bin/ext/beeline.sh
    index ab3dc1a..9de8f6c 100644
    --- a/bin/ext/beeline.sh
    +++ b/bin/ext/beeline.sh
    @@ -31,7 +31,7 @@ beeline () {
          hadoopClasspath="${HADOOP_CLASSPATH}:"
        fi
        export HADOOP_CLASSPATH="${hadoopClasspath}${HIVE_CONF_DIR}:${beelineJarPath}:${superCsvJarPath}:${jlineJarPath}:${jdbcStandaloneJarPath}"
    - export HADOOP_CLIENT_OPTS="$HADOOP_CLIENT_OPTS -Dlog4j.configuration=beeline-log4j.properties "
    + export HADOOP_CLIENT_OPTS="$HADOOP_CLIENT_OPTS -Dlog4j.configurationFile=beeline-log4j2.xml "

        exec $HADOOP jar ${beelineJarPath} $CLASS $HIVE_OPTS "$@"
      }

    http://git-wip-us.apache.org/repos/asf/hive/blob/c93d6c77/bin/hive
    ----------------------------------------------------------------------
    diff --git a/bin/hive b/bin/hive
    index 5dc93fb..ad7139e 100755
    --- a/bin/hive
    +++ b/bin/hive
    @@ -193,6 +193,9 @@ if [ "$HADOOP_HOME" == "" ]; then
        exit 4;
      fi

    +# to avoid errors from log4j2 automatic configuration loading
    +export HADOOP_CLIENT_OPTS="$HADOOP_CLIENT_OPTS -Dlog4j.configurationFile=hive-log4j2.xml "
    +
      HADOOP=$HADOOP_HOME/bin/hadoop
      if [ ! -f ${HADOOP} ]; then
        echo "Cannot find hadoop installation: \$HADOOP_HOME or \$HADOOP_PREFIX must be set or hadoop must be in the path";

    http://git-wip-us.apache.org/repos/asf/hive/blob/c93d6c77/common/pom.xml
    ----------------------------------------------------------------------
    diff --git a/common/pom.xml b/common/pom.xml
    index a7997e2..dba814d 100644
    --- a/common/pom.xml
    +++ b/common/pom.xml
    @@ -66,14 +66,24 @@
            <version>${joda.version}</version>
          </dependency>
          <dependency>
    - <groupId>log4j</groupId>
    - <artifactId>log4j</artifactId>
    - <version>${log4j.version}</version>
    + <groupId>org.apache.logging.log4j</groupId>
    + <artifactId>log4j-1.2-api</artifactId>
    + <version>${log4j2.version}</version>
          </dependency>
          <dependency>
    - <groupId>log4j</groupId>
    - <artifactId>apache-log4j-extras</artifactId>
    - <version>${log4j-extras.version}</version>
    + <groupId>org.apache.logging.log4j</groupId>
    + <artifactId>log4j-web</artifactId>
    + <version>${log4j2.version}</version>
    + </dependency>
    + <dependency>
    + <groupId>org.apache.logging.log4j</groupId>
    + <artifactId>log4j-slf4j-impl</artifactId>
    + <version>${log4j2.version}</version>
    + </dependency>
    + <dependency>
    + <groupId>org.apache.logging.log4j</groupId>
    + <artifactId>log4j-jcl</artifactId>
    + <version>${log4j2.version}</version>
          </dependency>
          <dependency>
            <groupId>org.apache.commons</groupId>

    http://git-wip-us.apache.org/repos/asf/hive/blob/c93d6c77/common/src/java/org/apache/hadoop/hive/common/LogUtils.java
    ----------------------------------------------------------------------
    diff --git a/common/src/java/org/apache/hadoop/hive/common/LogUtils.java b/common/src/java/org/apache/hadoop/hive/common/LogUtils.java
    index 9118675..3ca5c0f 100644
    --- a/common/src/java/org/apache/hadoop/hive/common/LogUtils.java
    +++ b/common/src/java/org/apache/hadoop/hive/common/LogUtils.java
    @@ -18,26 +18,22 @@

      package org.apache.hadoop.hive.common;

    -import java.net.URL;
      import java.io.File;
    -import java.io.IOException;
    -import java.io.FileNotFoundException;
    +import java.net.URL;

    -import org.apache.commons.lang.StringUtils;
      import org.apache.commons.logging.Log;
      import org.apache.commons.logging.LogFactory;
    -import org.apache.log4j.LogManager;
    -import org.apache.log4j.PropertyConfigurator;
      import org.apache.hadoop.hive.conf.HiveConf;
      import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
    +import org.apache.logging.log4j.core.config.Configurator;

      /**
       * Utilities common to logging operations.
       */
      public class LogUtils {

    - private static final String HIVE_L4J = "hive-log4j.properties";
    - private static final String HIVE_EXEC_L4J = "hive-exec-log4j.properties";
    + private static final String HIVE_L4J = "hive-log4j2.xml";
    + private static final String HIVE_EXEC_L4J = "hive-exec-log4j2.xml";
        private static final Log l4j = LogFactory.getLog(LogUtils.class);

        @SuppressWarnings("serial")
    @@ -95,8 +91,7 @@ public class LogUtils {
                }
                System.setProperty(HiveConf.ConfVars.HIVEQUERYID.toString(), queryId);
              }
    - LogManager.resetConfiguration();
    - PropertyConfigurator.configure(log4jFileName);
    + Configurator.initialize(null, log4jFileName);
              logConfigLocation(conf);
              return ("Logging initialized using configuration in " + log4jConfigFile);
            }
    @@ -123,8 +118,7 @@ public class LogUtils {
              break;
          }
          if (hive_l4j != null) {
    - LogManager.resetConfiguration();
    - PropertyConfigurator.configure(hive_l4j);
    + Configurator.initialize(null, hive_l4j.toString());
            logConfigLocation(conf);
            return (logMessage + "\n" + "Logging initialized using configuration in " + hive_l4j);
          } else {

    http://git-wip-us.apache.org/repos/asf/hive/blob/c93d6c77/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
    ----------------------------------------------------------------------
    diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
    index 11b9f78..730f5be 100644
    --- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
    +++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
    @@ -1696,13 +1696,13 @@ public class HiveConf extends Configuration {
          // logging configuration
          HIVE_LOG4J_FILE("hive.log4j.file", "",
              "Hive log4j configuration file.\n" +
    - "If the property is not set, then logging will be initialized using hive-log4j.properties found on the classpath.\n" +
    - "If the property is set, the value must be a valid URI (java.net.URI, e.g. \"file:///tmp/my-logging.properties\"), \n" +
    + "If the property is not set, then logging will be initialized using hive-log4j2.xml found on the classpath.\n" +
    + "If the property is set, the value must be a valid URI (java.net.URI, e.g. \"file:///tmp/my-logging.xml\"), \n" +
              "which you can then extract a URL from and pass to PropertyConfigurator.configure(URL)."),
          HIVE_EXEC_LOG4J_FILE("hive.exec.log4j.file", "",
              "Hive log4j configuration file for execution mode(sub command).\n" +
    - "If the property is not set, then logging will be initialized using hive-exec-log4j.properties found on the classpath.\n" +
    - "If the property is set, the value must be a valid URI (java.net.URI, e.g. \"file:///tmp/my-logging.properties\"), \n" +
    + "If the property is not set, then logging will be initialized using hive-exec-log4j2.xml found on the classpath.\n" +
    + "If the property is set, the value must be a valid URI (java.net.URI, e.g. \"file:///tmp/my-logging.xml\"), \n" +
              "which you can then extract a URL from and pass to PropertyConfigurator.configure(URL)."),

          HIVE_LOG_EXPLAIN_OUTPUT("hive.log.explain.output", false,

    http://git-wip-us.apache.org/repos/asf/hive/blob/c93d6c77/common/src/main/resources/hive-log4j.properties
    ----------------------------------------------------------------------
    diff --git a/common/src/main/resources/hive-log4j.properties b/common/src/main/resources/hive-log4j.properties
    deleted file mode 100644
    index 14fa725..0000000
    --- a/common/src/main/resources/hive-log4j.properties
    +++ /dev/null
    @@ -1,88 +0,0 @@
    -# Licensed to the Apache Software Foundation (ASF) under one
    -# or more contributor license agreements. See the NOTICE file
    -# distributed with this work for additional information
    -# regarding copyright ownership. The ASF licenses this file
    -# to you under the Apache License, Version 2.0 (the
    -# "License"); you may not use this file except in compliance
    -# with the License. You may obtain a copy of the License at
    -#
    -# http://www.apache.org/licenses/LICENSE-2.0
    -#
    -# Unless required by applicable law or agreed to in writing, software
    -# distributed under the License is distributed on an "AS IS" BASIS,
    -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    -# See the License for the specific language governing permissions and
    -# limitations under the License.
    -
    -# Define some default values that can be overridden by system properties
    -hive.log.threshold=ALL
    -hive.root.logger=INFO,DRFA
    -hive.log.dir=${java.io.tmpdir}/${user.name}
    -hive.log.file=hive.log
    -
    -# Define the root logger to the system property "hadoop.root.logger".
    -log4j.rootLogger=${hive.root.logger}, EventCounter
    -
    -# Logging Threshold
    -log4j.threshold=${hive.log.threshold}
    -
    -#
    -# Daily Rolling File Appender
    -#
    -# Use the PidDailyerRollingFileAppend class instead if you want to use separate log files
    -# for different CLI session.
    -#
    -# log4j.appender.DRFA=org.apache.hadoop.hive.ql.log.PidDailyRollingFileAppender
    -
    -log4j.appender.DRFA=org.apache.log4j.DailyRollingFileAppender
    -
    -log4j.appender.DRFA.File=${hive.log.dir}/${hive.log.file}
    -
    -# Rollver at midnight
    -log4j.appender.DRFA.DatePattern=.yyyy-MM-dd
    -
    -# 30-day backup
    -#log4j.appender.DRFA.MaxBackupIndex=30
    -log4j.appender.DRFA.layout=org.apache.log4j.PatternLayout
    -
    -# Pattern format: Date LogLevel LoggerName LogMessage
    -#log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n
    -# Debugging Pattern format
    -log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %-5p [%t]: %c{2} (%F:%M(%L)) - %m%n
    -
    -
    -#
    -# console
    -# Add "console" to rootlogger above if you want to use this
    -#
    -
    -log4j.appender.console=org.apache.log4j.ConsoleAppender
    -log4j.appender.console.target=System.err
    -log4j.appender.console.layout=org.apache.log4j.PatternLayout
    -log4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} [%t]: %p %c{2}: %m%n
    -log4j.appender.console.encoding=UTF-8
    -
    -#custom logging levels
    -#log4j.logger.xxx=DEBUG
    -
    -#
    -# Event Counter Appender
    -# Sends counts of logging messages at different severity levels to Hadoop Metrics.
    -#
    -log4j.appender.EventCounter=org.apache.hadoop.hive.shims.HiveEventCounter
    -
    -
    -log4j.category.DataNucleus=ERROR,DRFA
    -log4j.category.Datastore=ERROR,DRFA
    -log4j.category.Datastore.Schema=ERROR,DRFA
    -log4j.category.JPOX.Datastore=ERROR,DRFA
    -log4j.category.JPOX.Plugin=ERROR,DRFA
    -log4j.category.JPOX.MetaData=ERROR,DRFA
    -log4j.category.JPOX.Query=ERROR,DRFA
    -log4j.category.JPOX.General=ERROR,DRFA
    -log4j.category.JPOX.Enhancer=ERROR,DRFA
    -
    -
    -# Silence useless ZK logs
    -log4j.logger.org.apache.zookeeper.server.NIOServerCnxn=WARN,DRFA
    -log4j.logger.org.apache.zookeeper.ClientCnxnSocketNIO=WARN,DRFA

    http://git-wip-us.apache.org/repos/asf/hive/blob/c93d6c77/common/src/main/resources/hive-log4j2.xml
    ----------------------------------------------------------------------
    diff --git a/common/src/main/resources/hive-log4j2.xml b/common/src/main/resources/hive-log4j2.xml
    new file mode 100644
    index 0000000..31b8fcc
    --- /dev/null
    +++ b/common/src/main/resources/hive-log4j2.xml
    @@ -0,0 +1,111 @@
    +<?xml version="1.0" encoding="UTF-8"?>
    +<!--
    + Licensed to the Apache Software Foundation (ASF) under one or more
    + contributor license agreements. See the NOTICE file distributed with
    + this work for additional information regarding copyright ownership.
    + The ASF licenses this file to You under the Apache License, Version 2.0
    + (the "License"); you may not use this file except in compliance with
    + the License. You may obtain a copy of the License at
    +
    + http://www.apache.org/licenses/LICENSE-2.0
    +
    + Unless required by applicable law or agreed to in writing, software
    + distributed under the License is distributed on an "AS IS" BASIS,
    + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    + See the License for the specific language governing permissions and
    + limitations under the License.
    +-->
    +
    +<Configuration status="info" strict="true" name="HiveLog4j2"
    + packages="org.apache.hadoop.hive.ql.log">
    +
    + <Properties>
    + <Property name="hive.log.threshold">ALL</Property>
    + <Property name="hive.log.level">INFO</Property>
    + <Property name="hive.root.logger">DRFA</Property>
    + <Property name="hive.log.dir">${sys:java.io.tmpdir}/${sys:user.name}</Property>
    + <Property name="hive.log.file">hive.log</Property>
    + </Properties>
    +
    + <Appenders>
    + <Console name="console" target="SYSTEM_ERR">
    + <PatternLayout pattern="%d{yy/MM/dd HH:mm:ss} [%t]: %p %c{2}: %m%n"/>
    + </Console>
    +
    + <!-- Regular File Appender -->
    + <!-- NOTE: if enabling multiple file appender make sure to use different file names -->
    + <!-- <File name="FA" fileName="${sys:hive.log.dir}/${sys:hive.log.file}">
    + <PatternLayout pattern="%d{ISO8601} %-5p [%t]: %c{2} (%F:%M(%L)) - %m%n" />
    + </File> -->
    +
    + <!-- Daily Rolling File Appender -->
    + <!-- NOTE: if enabling multiple file appender make sure to use different file names -->
    + <!-- Use %pid in the filePattern to append <process-id>@<host-name> to the filename if you want separate log files for different CLI session -->
    + <RollingFile name="DRFA" fileName="${sys:hive.log.dir}/${sys:hive.log.file}"
    + filePattern="${sys:hive.log.dir}/${sys:hive.log.file}.%d{yyyy-MM-dd}">
    + <PatternLayout pattern="%d{ISO8601} %-5p [%t]: %c{2} (%F:%M(%L)) - %m%n" />
    + <Policies>
    + <!-- rollover@mignight (interval = 1 means daily) -->
    + <TimeBasedTriggeringPolicy interval="1" modulate="true"/>
    + </Policies>
    + <!-- 30-day backup -->
    + <!-- <DefaultRolloverStrategy max="30"/> -->
    + </RollingFile>
    +
    + <!-- Size based Rolling File Appender -->
    + <!-- NOTE: if enabling multiple file appender make sure to use different file names -->
    + <!-- <RollingFile name="RFA" fileName="${sys:hive.log.dir}/${sys:hive.log.file}"
    + filePattern="${sys:hive.log.dir}/${sys:hive.log.file}.%i">
    + <PatternLayout pattern="%d{ISO8601} %-5p [%t]: %c{2} (%F:%M(%L)) - %m%n" />
    + <Policies>
    + <SizeBasedTriggeringPolicy size="256 MB" />
    + </Policies>
    + <DefaultRolloverStrategy max="10"/>
    + </RollingFile> -->
    +
    + <!-- hiveeventcounter appender is loaded from configuration packages attribute.sends counts of logging messages@different severity levels to hadoop metrics. -->
    + <HiveEventCounter name="EventCounter"/>
    + </Appenders>
    +
    + <Loggers>
    + <Root level="${sys:hive.log.threshold}">
    + <AppenderRef ref="${sys:hive.root.logger}" level="${sys:hive.log.level}"/>
    + <AppenderRef ref="EventCounter" />
    + </Root>
    +
    + <Logger name="org.apache.zookeeper.server.NIOServerCnxn" level="WARN">
    + <AppenderRef ref="${sys:hive.root.logger}"/>
    + </Logger>
    + <Logger name="org.apache.zookeeper.ClientCnxnSocketNIO" level="WARN">
    + <AppenderRef ref="${sys:hive.root.logger}"/>
    + </Logger>
    + <Logger name="DataNucleus" level="ERROR">
    + <AppenderRef ref="${sys:hive.root.logger}"/>
    + </Logger>
    + <Logger name="Datastore" level="ERROR">
    + <AppenderRef ref="${sys:hive.root.logger}"/>
    + </Logger>
    + <Logger name="Datastore.Schema" level="ERROR">
    + <AppenderRef ref="${sys:hive.root.logger}"/>
    + </Logger>
    + <Logger name="JPOX.Datastore" level="ERROR">
    + <AppenderRef ref="${sys:hive.root.logger}"/>
    + </Logger>
    + <Logger name="JPOX.Plugin" level="ERROR">
    + <AppenderRef ref="${sys:hive.root.logger}"/>
    + </Logger>
    + <Logger name="JPOX.Metadata" level="ERROR">
    + <AppenderRef ref="${sys:hive.root.logger}"/>
    + </Logger>
    + <Logger name="JPOX.Query" level="ERROR">
    + <AppenderRef ref="${sys:hive.root.logger}"/>
    + </Logger>
    + <Logger name="JPOX.General" level="ERROR">
    + <AppenderRef ref="${sys:hive.root.logger}"/>
    + </Logger>
    + <Logger name="JPOX.Enhancer" level="ERROR">
    + <AppenderRef ref="${sys:hive.root.logger}"/>
    + </Logger>
    + </Loggers>
    +
    +</Configuration>

    http://git-wip-us.apache.org/repos/asf/hive/blob/c93d6c77/common/src/test/org/apache/hadoop/hive/conf/TestHiveLogging.java
    ----------------------------------------------------------------------
    diff --git a/common/src/test/org/apache/hadoop/hive/conf/TestHiveLogging.java b/common/src/test/org/apache/hadoop/hive/conf/TestHiveLogging.java
    index d5cedb1..92269e7 100644
    --- a/common/src/test/org/apache/hadoop/hive/conf/TestHiveLogging.java
    +++ b/common/src/test/org/apache/hadoop/hive/conf/TestHiveLogging.java
    @@ -21,12 +21,12 @@ import java.io.BufferedReader;
      import java.io.File;
      import java.io.InputStreamReader;

    -import junit.framework.TestCase;
    -
      import org.apache.hadoop.hive.common.LogUtils;
      import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
      import org.apache.hive.common.util.HiveTestUtils;

    +import junit.framework.TestCase;
    +
      /**
       * TestHiveLogging
       *
    @@ -104,9 +104,9 @@ public class TestHiveLogging extends TestCase {
          // customized log4j config log file to be: /${test.tmp.dir}/TestHiveLogging/hiveLog4jTest.log
          File customLogPath = new File(new File(System.getProperty("test.tmp.dir")),
              System.getProperty("user.name") + "-TestHiveLogging/");
    - String customLogName = "hiveLog4jTest.log";
    + String customLogName = "hiveLog4j2Test.log";
          File customLogFile = new File(customLogPath, customLogName);
          RunTest(customLogFile,
    - "hive-log4j-test.properties", "hive-exec-log4j-test.properties");
    + "hive-log4j2-test.xml", "hive-exec-log4j2-test.xml");
        }
      }

    http://git-wip-us.apache.org/repos/asf/hive/blob/c93d6c77/common/src/test/resources/hive-exec-log4j-test.properties
    ----------------------------------------------------------------------
    diff --git a/common/src/test/resources/hive-exec-log4j-test.properties b/common/src/test/resources/hive-exec-log4j-test.properties
    deleted file mode 100644
    index 1e53f26..0000000
    --- a/common/src/test/resources/hive-exec-log4j-test.properties
    +++ /dev/null
    @@ -1,59 +0,0 @@
    -# Define some default values that can be overridden by system properties
    -hive.root.logger=INFO,FA
    -hive.log.dir=/${test.tmp.dir}/${user.name}-TestHiveLogging
    -hive.log.file=hiveExecLog4jTest.log
    -
    -# Define the root logger to the system property "hadoop.root.logger".
    -log4j.rootLogger=${hive.root.logger}, EventCounter
    -
    -# Logging Threshold
    -log4j.threshhold=WARN
    -
    -#
    -# File Appender
    -#
    -
    -log4j.appender.FA=org.apache.log4j.FileAppender
    -log4j.appender.FA.File=${hive.log.dir}/${hive.log.file}
    -log4j.appender.FA.layout=org.apache.log4j.PatternLayout
    -
    -# Pattern format: Date LogLevel LoggerName LogMessage
    -#log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n
    -# Debugging Pattern format
    -log4j.appender.FA.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n
    -
    -
    -#
    -# console
    -# Add "console" to rootlogger above if you want to use this
    -#
    -
    -log4j.appender.console=org.apache.log4j.ConsoleAppender
    -log4j.appender.console.target=System.err
    -log4j.appender.console.layout=org.apache.log4j.PatternLayout
    -log4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{2}: %m%n
    -
    -#custom logging levels
    -#log4j.logger.xxx=DEBUG
    -
    -#
    -# Event Counter Appender
    -# Sends counts of logging messages at different severity levels to Hadoop Metrics.
    -#
    -log4j.appender.EventCounter=org.apache.hadoop.metrics.jvm.EventCounter
    -
    -
    -log4j.category.DataNucleus=ERROR,FA
    -log4j.category.Datastore=ERROR,FA
    -log4j.category.Datastore.Schema=ERROR,FA
    -log4j.category.JPOX.Datastore=ERROR,FA
    -log4j.category.JPOX.Plugin=ERROR,FA
    -log4j.category.JPOX.MetaData=ERROR,FA
    -log4j.category.JPOX.Query=ERROR,FA
    -log4j.category.JPOX.General=ERROR,FA
    -log4j.category.JPOX.Enhancer=ERROR,FA
    -
    -
    -# Silence useless ZK logs
    -log4j.logger.org.apache.zookeeper.server.NIOServerCnxn=WARN,FA
    -log4j.logger.org.apache.zookeeper.ClientCnxnSocketNIO=WARN,FA

    http://git-wip-us.apache.org/repos/asf/hive/blob/c93d6c77/common/src/test/resources/hive-exec-log4j2-test.xml
    ----------------------------------------------------------------------
    diff --git a/common/src/test/resources/hive-exec-log4j2-test.xml b/common/src/test/resources/hive-exec-log4j2-test.xml
    new file mode 100644
    index 0000000..b5f2cb4
    --- /dev/null
    +++ b/common/src/test/resources/hive-exec-log4j2-test.xml
    @@ -0,0 +1,86 @@
    +<?xml version="1.0" encoding="UTF-8"?>
    +<!--
    + Licensed to the Apache Software Foundation (ASF) under one or more
    + contributor license agreements. See the NOTICE file distributed with
    + this work for additional information regarding copyright ownership.
    + The ASF licenses this file to You under the Apache License, Version 2.0
    + (the "License"); you may not use this file except in compliance with
    + the License. You may obtain a copy of the License at
    +
    + http://www.apache.org/licenses/LICENSE-2.0
    +
    + Unless required by applicable law or agreed to in writing, software
    + distributed under the License is distributed on an "AS IS" BASIS,
    + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    + See the License for the specific language governing permissions and
    + limitations under the License.
    +-->
    +
    +<Configuration status="info" strict="true" name="HiveExecLog4j2Test"
    + packages="org.apache.hadoop.hive.ql.log">
    +
    + <Properties>
    + <Property name="hive.log.threshold">ALL</Property>
    + <Property name="hive.log.level">INFO</Property>
    + <Property name="hive.root.logger">FA</Property>
    + <Property name="hive.log.dir">${sys:test.tmp.dir}/${sys:user.name}-TestHiveLogging</Property>
    + <Property name="hive.log.file">hiveExecLog4j2Test.log</Property>
    + </Properties>
    +
    + <Appenders>
    + <Console name="console" target="SYSTEM_ERR">
    + <PatternLayout pattern="%d{yy/MM/dd HH:mm:ss} [%t]: %p %c{2}: %m%n"/>
    + </Console>
    +
    + <File name="FA" fileName="${sys:hive.log.dir}/${sys:hive.log.file}">
    + <PatternLayout pattern="%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n" />
    + </File>
    +
    + <!-- hiveeventcounter appender is loaded from configuration packages attribute.sends counts of logging messages@different severity levels to hadoop metrics. -->
    + <HiveEventCounter name="EventCounter"/>
    + </Appenders>
    +
    + <Loggers>
    + <Root level="${sys:hive.log.threshold}">
    + <AppenderRef ref="${sys:hive.root.logger}" level="${sys:hive.log.level}"/>
    + <AppenderRef ref="EventCounter" />
    + </Root>
    +
    + <!-- Silence useless ZK logs -->
    + <Logger name="org.apache.zookeeper.server.NIOServerCnxn" level="WARN">
    + <AppenderRef ref="${sys:hive.root.logger}"/>
    + </Logger>
    + <Logger name="org.apache.zookeeper.ClientCnxnSocketNIO" level="WARN">
    + <AppenderRef ref="${sys:hive.root.logger}"/>
    + </Logger>
    +
    + <Logger name="DataNucleus" level="ERROR">
    + <AppenderRef ref="${sys:hive.root.logger}"/>
    + </Logger>
    + <Logger name="Datastore" level="ERROR">
    + <AppenderRef ref="${sys:hive.root.logger}"/>
    + </Logger>
    + <Logger name="Datastore.Schema" level="ERROR">
    + <AppenderRef ref="${sys:hive.root.logger}"/>
    + </Logger>
    + <Logger name="JPOX.Datastore" level="ERROR">
    + <AppenderRef ref="${sys:hive.root.logger}"/>
    + </Logger>
    + <Logger name="JPOX.Plugin" level="ERROR">
    + <AppenderRef ref="${sys:hive.root.logger}"/>
    + </Logger>
    + <Logger name="JPOX.Metadata" level="ERROR">
    + <AppenderRef ref="${sys:hive.root.logger}"/>
    + </Logger>
    + <Logger name="JPOX.Query" level="ERROR">
    + <AppenderRef ref="${sys:hive.root.logger}"/>
    + </Logger>
    + <Logger name="JPOX.General" level="ERROR">
    + <AppenderRef ref="${sys:hive.root.logger}"/>
    + </Logger>
    + <Logger name="JPOX.Enhancer" level="ERROR">
    + <AppenderRef ref="${sys:hive.root.logger}"/>
    + </Logger>
    + </Loggers>
    +
    +</Configuration>

    http://git-wip-us.apache.org/repos/asf/hive/blob/c93d6c77/common/src/test/resources/hive-log4j-test.properties
    ----------------------------------------------------------------------
    diff --git a/common/src/test/resources/hive-log4j-test.properties b/common/src/test/resources/hive-log4j-test.properties
    deleted file mode 100644
    index 0348325..0000000
    --- a/common/src/test/resources/hive-log4j-test.properties
    +++ /dev/null
    @@ -1,71 +0,0 @@
    -# Define some default values that can be overridden by system properties
    -hive.root.logger=WARN,DRFA
    -hive.log.dir=${test.tmp.dir}/${user.name}-TestHiveLogging
    -hive.log.file=hiveLog4jTest.log
    -
    -# Define the root logger to the system property "hadoop.root.logger".
    -log4j.rootLogger=${hive.root.logger}, EventCounter
    -
    -# Logging Threshold
    -log4j.threshhold=WARN
    -
    -#
    -# Daily Rolling File Appender
    -#
    -# Use the PidDailyerRollingFileAppend class instead if you want to use separate log files
    -# for different CLI session.
    -#
    -# log4j.appender.DRFA=org.apache.hadoop.hive.ql.log.PidDailyRollingFileAppender
    -
    -log4j.appender.DRFA=org.apache.log4j.DailyRollingFileAppender
    -
    -log4j.appender.DRFA.File=${hive.log.dir}/${hive.log.file}
    -
    -# Rollver at midnight
    -log4j.appender.DRFA.DatePattern=.yyyy-MM-dd
    -
    -# 30-day backup
    -#log4j.appender.DRFA.MaxBackupIndex=30
    -log4j.appender.DRFA.layout=org.apache.log4j.PatternLayout
    -
    -# Pattern format: Date LogLevel LoggerName LogMessage
    -#log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n
    -# Debugging Pattern format
    -log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n
    -
    -
    -#
    -# console
    -# Add "console" to rootlogger above if you want to use this
    -#
    -
    -log4j.appender.console=org.apache.log4j.ConsoleAppender
    -log4j.appender.console.target=System.err
    -log4j.appender.console.layout=org.apache.log4j.PatternLayout
    -log4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{2}: %m%n
    -log4j.appender.console.encoding=UTF-8
    -
    -#custom logging levels
    -#log4j.logger.xxx=DEBUG
    -
    -#
    -# Event Counter Appender
    -# Sends counts of logging messages at different severity levels to Hadoop Metrics.
    -#
    -log4j.appender.EventCounter=org.apache.hadoop.metrics.jvm.EventCounter
    -
    -
    -log4j.category.DataNucleus=ERROR,DRFA
    -log4j.category.Datastore=ERROR,DRFA
    -log4j.category.Datastore.Schema=ERROR,DRFA
    -log4j.category.JPOX.Datastore=ERROR,DRFA
    -log4j.category.JPOX.Plugin=ERROR,DRFA
    -log4j.category.JPOX.MetaData=ERROR,DRFA
    -log4j.category.JPOX.Query=ERROR,DRFA
    -log4j.category.JPOX.General=ERROR,DRFA
    -log4j.category.JPOX.Enhancer=ERROR,DRFA
    -
    -
    -# Silence useless ZK logs
    -log4j.logger.org.apache.zookeeper.server.NIOServerCnxn=WARN,DRFA
    -log4j.logger.org.apache.zookeeper.ClientCnxnSocketNIO=WARN,DRFA

    http://git-wip-us.apache.org/repos/asf/hive/blob/c93d6c77/common/src/test/resources/hive-log4j2-test.xml
    ----------------------------------------------------------------------
    diff --git a/common/src/test/resources/hive-log4j2-test.xml b/common/src/test/resources/hive-log4j2-test.xml
    new file mode 100644
    index 0000000..63b46c8
    --- /dev/null
    +++ b/common/src/test/resources/hive-log4j2-test.xml
    @@ -0,0 +1,95 @@
    +<?xml version="1.0" encoding="UTF-8"?>
    +<!--
    + Licensed to the Apache Software Foundation (ASF) under one or more
    + contributor license agreements. See the NOTICE file distributed with
    + this work for additional information regarding copyright ownership.
    + The ASF licenses this file to You under the Apache License, Version 2.0
    + (the "License"); you may not use this file except in compliance with
    + the License. You may obtain a copy of the License at
    +
    + http://www.apache.org/licenses/LICENSE-2.0
    +
    + Unless required by applicable law or agreed to in writing, software
    + distributed under the License is distributed on an "AS IS" BASIS,
    + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    + See the License for the specific language governing permissions and
    + limitations under the License.
    +-->
    +
    +<Configuration status="info" strict="true" name="HiveLog4j2Test"
    + packages="org.apache.hadoop.hive.ql.log">
    +
    + <Properties>
    + <Property name="hive.log.threshold">ALL</Property>
    + <Property name="hive.log.level">WARN</Property>
    + <Property name="hive.root.logger">DRFA</Property>
    + <Property name="hive.log.dir">${sys:test.tmp.dir}/${sys:user.name}-TestHiveLogging</Property>
    + <Property name="hive.log.file">hiveLog4j2Test.log</Property>
    + </Properties>
    +
    + <Appenders>
    + <Console name="console" target="SYSTEM_ERR">
    + <PatternLayout pattern="%d{yy/MM/dd HH:mm:ss} [%t]: %p %c{2}: %m%n"/>
    + </Console>
    +
    + <!-- Daily Rolling File Appender -->
    + <!-- Use %pid in the filePattern to append <process-id>@<host-name> to the filename if you want separate log files for different CLI session -->
    + <RollingFile name="DRFA" fileName="${sys:hive.log.dir}/${sys:hive.log.file}"
    + filePattern="${sys:hive.log.dir}/${sys:hive.log.file}.%d{yyyy-MM-dd}">
    + <PatternLayout pattern="%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n" />
    + <Policies>
    + <!-- rollover@mignight (interval = 1 means daily) -->
    + <TimeBasedTriggeringPolicy interval="1" modulate="true"/>
    + </Policies>
    + <!-- 30-day backup -->
    + <!-- <DefaultRolloverStrategy max="30"/> -->
    + </RollingFile>
    +
    + <!-- hiveeventcounter appender is loaded from configuration packages attribute.sends counts of logging messages@different severity levels to hadoop metrics. -->
    + <HiveEventCounter name="EventCounter"/>
    + </Appenders>
    +
    + <Loggers>
    + <Root level="${sys:hive.log.threshold}">
    + <AppenderRef ref="${sys:hive.root.logger}" level="${sys:hive.log.level}"/>
    + <AppenderRef ref="EventCounter" />
    + </Root>
    +
    + <!-- Silence useless ZK logs -->
    + <Logger name="org.apache.zookeeper.server.NIOServerCnxn" level="WARN">
    + <AppenderRef ref="${sys:hive.root.logger}"/>
    + </Logger>
    + <Logger name="org.apache.zookeeper.ClientCnxnSocketNIO" level="WARN">
    + <AppenderRef ref="${sys:hive.root.logger}"/>
    + </Logger>
    +
    + <Logger name="DataNucleus" level="ERROR">
    + <AppenderRef ref="${sys:hive.root.logger}"/>
    + </Logger>
    + <Logger name="Datastore" level="ERROR">
    + <AppenderRef ref="${sys:hive.root.logger}"/>
    + </Logger>
    + <Logger name="Datastore.Schema" level="ERROR">
    + <AppenderRef ref="${sys:hive.root.logger}"/>
    + </Logger>
    + <Logger name="JPOX.Datastore" level="ERROR">
    + <AppenderRef ref="${sys:hive.root.logger}"/>
    + </Logger>
    + <Logger name="JPOX.Plugin" level="ERROR">
    + <AppenderRef ref="${sys:hive.root.logger}"/>
    + </Logger>
    + <Logger name="JPOX.Metadata" level="ERROR">
    + <AppenderRef ref="${sys:hive.root.logger}"/>
    + </Logger>
    + <Logger name="JPOX.Query" level="ERROR">
    + <AppenderRef ref="${sys:hive.root.logger}"/>
    + </Logger>
    + <Logger name="JPOX.General" level="ERROR">
    + <AppenderRef ref="${sys:hive.root.logger}"/>
    + </Logger>
    + <Logger name="JPOX.Enhancer" level="ERROR">
    + <AppenderRef ref="${sys:hive.root.logger}"/>
    + </Logger>
    + </Loggers>
    +
    +</Configuration>

    http://git-wip-us.apache.org/repos/asf/hive/blob/c93d6c77/data/conf/hive-log4j-old.properties
    ----------------------------------------------------------------------
    diff --git a/data/conf/hive-log4j-old.properties b/data/conf/hive-log4j-old.properties
    deleted file mode 100644
    index f274b8c..0000000
    --- a/data/conf/hive-log4j-old.properties
    +++ /dev/null
    @@ -1,82 +0,0 @@
    -# Licensed to the Apache Software Foundation (ASF) under one
    -# or more contributor license agreements. See the NOTICE file
    -# distributed with this work for additional information
    -# regarding copyright ownership. The ASF licenses this file
    -# to you under the Apache License, Version 2.0 (the
    -# "License"); you may not use this file except in compliance
    -# with the License. You may obtain a copy of the License at
    -#
    -# http://www.apache.org/licenses/LICENSE-2.0
    -#
    -# Unless required by applicable law or agreed to in writing, software
    -# distributed under the License is distributed on an "AS IS" BASIS,
    -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    -# See the License for the specific language governing permissions and
    -# limitations under the License.
    -
    -# Define some default values that can be overridden by system properties
    -hive.root.logger=DEBUG,DRFA
    -hive.log.dir=${build.dir.hive}/ql/tmp/
    -hive.log.file=hive.log
    -
    -# Define the root logger to the system property "hadoop.root.logger".
    -log4j.rootLogger=${hive.root.logger}, EventCounter
    -
    -# Logging Threshold
    -log4j.threshhold=WARN
    -
    -#
    -# Daily Rolling File Appender
    -#
    -
    -log4j.appender.DRFA=org.apache.log4j.DailyRollingFileAppender
    -log4j.appender.DRFA.File=${hive.log.dir}/${hive.log.file}
    -
    -# Rollver at midnight
    -log4j.appender.DRFA.DatePattern=.yyyy-MM-dd
    -
    -# 30-day backup
    -#log4j.appender.DRFA.MaxBackupIndex=30
    -log4j.appender.DRFA.layout=org.apache.log4j.PatternLayout
    -
    -# Pattern format: Date LogLevel LoggerName LogMessage
    -#log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n
    -# Debugging Pattern format
    -log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n
    -
    -
    -#
    -# console
    -# Add "console" to rootlogger above if you want to use this
    -#
    -
    -log4j.appender.console=org.apache.log4j.ConsoleAppender
    -log4j.appender.console.target=System.err
    -log4j.appender.console.layout=org.apache.log4j.PatternLayout
    -log4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{2}: %m%n
    -
    -#custom logging levels
    -#log4j.logger.xxx=DEBUG
    -
    -#
    -# Event Counter Appender
    -# Sends counts of logging messages at different severity levels to Hadoop Metrics.
    -#
    -log4j.appender.EventCounter=org.apache.hadoop.hive.shims.HiveEventCounter
    -
    -
    -log4j.category.DataNucleus=ERROR,DRFA
    -log4j.category.Datastore=ERROR,DRFA
    -log4j.category.Datastore.Schema=ERROR,DRFA
    -log4j.category.JPOX.Datastore=ERROR,DRFA
    -log4j.category.JPOX.Plugin=ERROR,DRFA
    -log4j.category.JPOX.MetaData=ERROR,DRFA
    -log4j.category.JPOX.Query=ERROR,DRFA
    -log4j.category.JPOX.General=ERROR,DRFA
    -log4j.category.JPOX.Enhancer=ERROR,DRFA
    -log4j.logger.org.apache.hadoop.conf.Configuration=ERROR,DRFA
    -
    -
    -# Silence useless ZK logs
    -log4j.logger.org.apache.zookeeper.server.NIOServerCnxn=WARN,DRFA
    -log4j.logger.org.apache.zookeeper.ClientCnxnSocketNIO=WARN,DRFA

    http://git-wip-us.apache.org/repos/asf/hive/blob/c93d6c77/data/conf/hive-log4j.properties
    ----------------------------------------------------------------------
    diff --git a/data/conf/hive-log4j.properties b/data/conf/hive-log4j.properties
    deleted file mode 100644
    index 023e3c2..0000000
    --- a/data/conf/hive-log4j.properties
    +++ /dev/null
    @@ -1,97 +0,0 @@
    -# Licensed to the Apache Software Foundation (ASF) under one
    -# or more contributor license agreements. See the NOTICE file
    -# distributed with this work for additional information
    -# regarding copyright ownership. The ASF licenses this file
    -# to you under the Apache License, Version 2.0 (the
    -# "License"); you may not use this file except in compliance
    -# with the License. You may obtain a copy of the License at
    -#
    -# http://www.apache.org/licenses/LICENSE-2.0
    -#
    -# Unless required by applicable law or agreed to in writing, software
    -# distributed under the License is distributed on an "AS IS" BASIS,
    -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    -# See the License for the specific language governing permissions and
    -# limitations under the License.
    -
    -# Define some default values that can be overridden by system properties
    -hive.root.logger=DEBUG,DRFA
    -hive.log.dir=${test.tmp.dir}/log/
    -hive.log.file=hive.log
    -
    -# Define the root logger to the system property "hadoop.root.logger".
    -log4j.rootLogger=${hive.root.logger}, EventCounter
    -
    -# Logging Threshold
    -log4j.threshhold=WARN
    -
    -#
    -# Daily Rolling File Appender
    -#
    -
    -log4j.appender.DRFA=org.apache.log4j.DailyRollingFileAppender
    -log4j.appender.DRFA.File=${hive.log.dir}/${hive.log.file}
    -
    -# Rollver at midnight
    -log4j.appender.DRFA.DatePattern=.yyyy-MM-dd
    -
    -# 30-day backup
    -#log4j.appender.DRFA.MaxBackupIndex=30
    -log4j.appender.DRFA.layout=org.apache.log4j.PatternLayout
    -
    -# Pattern format: Date LogLevel LoggerName LogMessage
    -#log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n
    -# Debugging Pattern format
    -log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %-5p [%t]: %c{2} (%F:%M(%L)) - %m%n
    -
    -# Warning: If you enable the following appender it will fill up your disk if you don't have a cleanup job!
    -# This uses the updated rolling file appender from log4j-extras that supports a reliable time-based rolling policy.
    -# See http://logging.apache.org/log4j/companions/extras/apidocs/org/apache/log4j/rolling/TimeBasedRollingPolicy.html
    -# Add "DAILY" to hive.root.logger above if you want to use this.
    -log4j.appender.DAILY=org.apache.log4j.rolling.RollingFileAppender
    -log4j.appender.DAILY.rollingPolicy=org.apache.log4j.rolling.TimeBasedRollingPolicy
    -log4j.appender.DAILY.rollingPolicy.ActiveFileName=${hive.log.dir}/${hive.log.file}
    -log4j.appender.DAILY.rollingPolicy.FileNamePattern=${hive.log.dir}/${hive.log.file}.%d{yyyy-MM-dd}
    -log4j.appender.DAILY.layout=org.apache.log4j.PatternLayout
    -log4j.appender.DAILY.layout.ConversionPattern=%d{dd MMM yyyy HH:mm:ss,SSS} %-5p [%t] (%C.%M:%L) %x - %m%n
    -
    -#
    -# console
    -# Add "console" to rootlogger above if you want to use this
    -#
    -
    -log4j.appender.console=org.apache.log4j.ConsoleAppender
    -log4j.appender.console.target=System.err
    -log4j.appender.console.layout=org.apache.log4j.PatternLayout
    -log4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{2}: %m%n
    -
    -#custom logging levels
    -#log4j.logger.xxx=DEBUG
    -
    -#
    -# Event Counter Appender
    -# Sends counts of logging messages at different severity levels to Hadoop Metrics.
    -#
    -log4j.appender.EventCounter=org.apache.hadoop.hive.shims.HiveEventCounter
    -
    -
    -log4j.category.DataNucleus=ERROR,DRFA
    -log4j.category.Datastore=ERROR,DRFA
    -log4j.category.Datastore.Schema=ERROR,DRFA
    -log4j.category.JPOX.Datastore=ERROR,DRFA
    -log4j.category.JPOX.Plugin=ERROR,DRFA
    -log4j.category.JPOX.MetaData=ERROR,DRFA
    -log4j.category.JPOX.Query=ERROR,DRFA
    -log4j.category.JPOX.General=ERROR,DRFA
    -log4j.category.JPOX.Enhancer=ERROR,DRFA
    -log4j.logger.org.apache.hadoop.conf.Configuration=ERROR,DRFA
    -log4j.logger.org.apache.zookeeper=INFO,DRFA
    -log4j.logger.org.apache.zookeeper.server.ServerCnxn=WARN,DRFA
    -log4j.logger.org.apache.zookeeper.server.NIOServerCnxn=WARN,DRFA
    -log4j.logger.org.apache.zookeeper.ClientCnxn=WARN,DRFA
    -log4j.logger.org.apache.zookeeper.ClientCnxnSocket=WARN,DRFA
    -log4j.logger.org.apache.zookeeper.ClientCnxnSocketNIO=WARN,DRFA
    -log4j.logger.org.apache.hadoop.hive.ql.log.PerfLogger=${hive.ql.log.PerfLogger.level}
    -log4j.logger.org.apache.hadoop.hive.ql.exec.Operator=INFO,DRFA
    -log4j.logger.org.apache.hadoop.hive.serde2.lazy=INFO,DRFA
    -log4j.logger.org.apache.hadoop.hive.metastore.ObjectStore=INFO,DRFA

    http://git-wip-us.apache.org/repos/asf/hive/blob/c93d6c77/data/conf/hive-log4j2.xml
    ----------------------------------------------------------------------
    diff --git a/data/conf/hive-log4j2.xml b/data/conf/hive-log4j2.xml
    new file mode 100644
    index 0000000..c9adfa2
    --- /dev/null
    +++ b/data/conf/hive-log4j2.xml
    @@ -0,0 +1,148 @@
    +<?xml version="1.0" encoding="UTF-8"?>
    +<!--
    + Licensed to the Apache Software Foundation (ASF) under one or more
    + contributor license agreements. See the NOTICE file distributed with
    + this work for additional information regarding copyright ownership.
    + The ASF licenses this file to You under the Apache License, Version 2.0
    + (the "License"); you may not use this file except in compliance with
    + the License. You may obtain a copy of the License at
    +
    + http://www.apache.org/licenses/LICENSE-2.0
    +
    + Unless required by applicable law or agreed to in writing, software
    + distributed under the License is distributed on an "AS IS" BASIS,
    + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    + See the License for the specific language governing permissions and
    + limitations under the License.
    +-->
    +
    +<Configuration status="info" strict="true" name="HiveLog4j2"
    + packages="org.apache.hadoop.hive.ql.log">
    +
    + <Properties>
    + <Property name="hive.log.threshold">ALL</Property>
    + <Property name="hive.log.level">DEBUG</Property>
    + <Property name="hive.root.logger">DRFA</Property>
    + <Property name="hive.log.dir">${sys:test.tmp.dir}/log</Property>
    + <Property name="hive.log.file">hive.log</Property>
    + <Property name="hive.ql.log.PerfLogger.level">INFO</Property>
    + </Properties>
    +
    + <Appenders>
    + <Console name="console" target="SYSTEM_ERR">
    + <PatternLayout pattern="%d{yy/MM/dd HH:mm:ss} %p %c{2}: %m%n"/>
    + </Console>
    +
    + <!-- Regular File Appender -->
    + <!-- NOTE: if enabling multiple file appender make sure to use different file names -->
    + <!-- <File name="FA" fileName="${sys:hive.log.dir}/${sys:hive.log.file}">
    + <PatternLayout pattern="%d{ISO8601} %-5p [%t]: %c{2} (%F:%M(%L)) - %m%n" />
    + </File> -->
    +
    + <!-- Daily Rolling File Appender -->
    + <!-- NOTE: if enabling multiple file appender make sure to use different file names -->
    + <!-- Use %pid in the filePattern to append <process-id>@<host-name> to the filename if you want separate log files for different CLI session -->
    + <RollingFile name="DRFA" fileName="${sys:hive.log.dir}/${sys:hive.log.file}"
    + filePattern="${sys:hive.log.dir}/${sys:hive.log.file}.%d{yyyy-MM-dd}">
    + <PatternLayout pattern="%d{ISO8601} %-5p [%t]: %c{2} (%F:%M(%L)) - %m%n" />
    + <Policies>
    + <!-- rollover@mignight (interval = 1 means daily) -->
    + <TimeBasedTriggeringPolicy interval="1" modulate="true"/>
    + </Policies>
    + <!-- 30-day backup -->
    + <!-- <DefaultRolloverStrategy max="30"/> -->
    + </RollingFile>
    +
    + <!-- Size based Rolling File Appender -->
    + <!-- NOTE: if enabling multiple file appender make sure to use different file names -->
    + <!-- <RollingFile name="RFA" fileName="${sys:hive.log.dir}/${sys:hive.log.file}"
    + filePattern="${sys:hive.log.dir}/${sys:hive.log.file}.%i">
    + <PatternLayout pattern="%d{ISO8601} %-5p [%t]: %c{2} (%F:%M(%L)) - %m%n" />
    + <Policies>
    + <SizeBasedTriggeringPolicy size="256 MB" />
    + </Policies>
    + <DefaultRolloverStrategy max="10"/>
    + </RollingFile> -->
    +
    + <!-- hiveeventcounter appender is loaded from configuration packages attribute.sends counts of logging messages@different severity levels to hadoop metrics. -->
    + <HiveEventCounter name="EventCounter"/>
    + </Appenders>
    +
    + <Loggers>
    + <Root level="${sys:hive.log.threshold}">
    + <AppenderRef ref="${sys:hive.root.logger}" level="{sys:hive.log.level}"/>
    + <AppenderRef ref="EventCounter" />
    + </Root>
    +
    + <Logger name="org.apache.hadoop.conf.Configuration" level="ERROR">
    + <AppenderRef ref="${sys:hive.root.logger}"/>
    + </Logger>
    + <Logger name="org.apache.zookeeper" level="INFO">
    + <AppenderRef ref="${sys:hive.root.logger}"/>
    + </Logger>
    + <Logger name="org.apache.zookeeper.server.ServerCnxn" level="WARN">
    + <AppenderRef ref="${sys:hive.root.logger}"/>
    + </Logger>
    + <Logger name="org.apache.zookeeper.server.NIOServerCnxn" level="WARN">
    + <AppenderRef ref="${sys:hive.root.logger}"/>
    + </Logger>
    + <Logger name="org.apache.zookeeper.ClientCnxn" level="WARN">
    + <AppenderRef ref="${sys:hive.root.logger}"/>
    + </Logger>
    + <Logger name="org.apache.zookeeper.ClientCnxnSocket" level="WARN">
    + <AppenderRef ref="${sys:hive.root.logger}"/>
    + </Logger>
    + <Logger name="org.apache.zookeeper.ClientCnxnSocketNIO" level="WARN">
    + <AppenderRef ref="${sys:hive.root.logger}"/>
    + </Logger>
    + <Logger name="org.apache.hadoop.hive.ql.log.PerfLogger" level="${sys:hive.ql.log.PerfLogger.level}">
    + <AppenderRef ref="${sys:hive.root.logger}"/>
    + </Logger>
    + <Logger name="org.apache.hadoop.hive.ql.exec.Operator" level="INFO">
    + <AppenderRef ref="${sys:hive.root.logger}"/>
    + </Logger>
    + <Logger name="org.apache.hadoop.hive.serde2.lazy" level="INFO">
    + <AppenderRef ref="${sys:hive.root.logger}"/>
    + </Logger>
    + <Logger name="org.apache.hadoop.hive.metastore.ObjectStore" level="INFO">
    + <AppenderRef ref="${sys:hive.root.logger}"/>
    + </Logger>
    + <Logger name="DataNucleus" level="ERROR">
    + <AppenderRef ref="${sys:hive.root.logger}"/>
    + </Logger>
    + <Logger name="Datastore" level="ERROR">
    + <AppenderRef ref="${sys:hive.root.logger}"/>
    + </Logger>
    + <Logger name="Datastore.Schema" level="ERROR">
    + <AppenderRef ref="${sys:hive.root.logger}"/>
    + </Logger>
    + <Logger name="JPOX.Datastore" level="ERROR">
    + <AppenderRef ref="${sys:hive.root.logger}"/>
    + </Logger>
    + <Logger name="JPOX.Plugin" level="ERROR">
    + <AppenderRef ref="${sys:hive.root.logger}"/>
    + </Logger>
    + <Logger name="JPOX.Metadata" level="ERROR">
    + <AppenderRef ref="${sys:hive.root.logger}"/>
    + </Logger>
    + <Logger name="JPOX.Query" level="ERROR">
    + <AppenderRef ref="${sys:hive.root.logger}"/>
    + </Logger>
    + <Logger name="JPOX.General" level="ERROR">
    + <AppenderRef ref="${sys:hive.root.logger}"/>
    + </Logger>
    + <Logger name="JPOX.Enhancer" level="ERROR">
    + <AppenderRef ref="${sys:hive.root.logger}"/>
    + </Logger>
    + <Logger name="JPOX.Enhancer" level="ERROR">
    + <AppenderRef ref="${sys:hive.root.logger}"/>
    + </Logger>
    + <Logger name="JPOX.Enhancer" level="ERROR">
    + <AppenderRef ref="${sys:hive.root.logger}"/>
    + </Logger>
    + <Logger name="JPOX.Enhancer" level="ERROR">
    + <AppenderRef ref="${sys:hive.root.logger}"/>
    + </Logger>
    + </Loggers>
    +
    +</Configuration>

    http://git-wip-us.apache.org/repos/asf/hive/blob/c93d6c77/data/conf/spark/log4j.properties
    ----------------------------------------------------------------------
    diff --git a/data/conf/spark/log4j.properties b/data/conf/spark/log4j.properties
    deleted file mode 100644
    index 8838c0e..0000000
    --- a/data/conf/spark/log4j.properties
    +++ /dev/null
    @@ -1,24 +0,0 @@
    -log4j.rootCategory=DEBUG, DRFA
    -
    -hive.spark.log.dir=target/tmp/log
    -# Settings to quiet third party logs that are too verbose
    -log4j.logger.org.eclipse.jetty=WARN
    -log4j.logger.org.eclipse.jetty.util.component.AbstractLifeCycle=ERROR
    -log4j.logger.org.apache.spark.repl.SparkIMain$exprTyper=INFO
    -log4j.logger.org.apache.spark.repl.SparkILoop$SparkILoopInterpreter=INFO
    -
    -log4j.appender.DRFA=org.apache.log4j.DailyRollingFileAppender
    -
    -log4j.appender.DRFA.File=${hive.spark.log.dir}/spark.log
    -
    -# Rollver at midnight
    -log4j.appender.DRFA.DatePattern=.yyyy-MM-dd
    -
    -# 30-day backup
    -#log4j.appender.DRFA.MaxBackupIndex=30
    -log4j.appender.DRFA.layout=org.apache.log4j.PatternLayout
    -
    -# Pattern format: Date LogLevel LoggerName LogMessage
    -#log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n
    -# Debugging Pattern format
    -log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %-5p [%t]: %c{2} (%F:%M(%L)) - %m%n

    http://git-wip-us.apache.org/repos/asf/hive/blob/c93d6c77/data/conf/spark/log4j2.xml
    ----------------------------------------------------------------------
    diff --git a/data/conf/spark/log4j2.xml b/data/conf/spark/log4j2.xml
    new file mode 100644
    index 0000000..395a2bf
    --- /dev/null
    +++ b/data/conf/spark/log4j2.xml
    @@ -0,0 +1,74 @@
    +<?xml version="1.0" encoding="UTF-8"?>
    +<!--
    + Licensed to the Apache Software Foundation (ASF) under one or more
    + contributor license agreements. See the NOTICE file distributed with
    + this work for additional information regarding copyright ownership.
    + The ASF licenses this file to You under the Apache License, Version 2.0
    + (the "License"); you may not use this file except in compliance with
    + the License. You may obtain a copy of the License at
    +
    + http://www.apache.org/licenses/LICENSE-2.0
    +
    + Unless required by applicable law or agreed to in writing, software
    + distributed under the License is distributed on an "AS IS" BASIS,
    + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    + See the License for the specific language governing permissions and
    + limitations under the License.
    +-->
    +
    +<Configuration status="info" strict="true" name="SparkLog4j2"
    + packages="org.apache.hadoop.hive.ql.log">
    +
    + <Properties>
    + <Property name="spark.log.level">DEBUG</Property>
    + <Property name="spark.root.logger">DRFA</Property>
    + <Property name="spark.log.dir">target/tmp/log</Property>
    + <Property name="spark.log.file">spark.log</Property>
    + </Properties>
    +
    + <Appenders>
    + <Console name="console" target="SYSTEM_ERR">
    + <PatternLayout pattern="%d{yy/MM/dd HH:mm:ss} [%t]: %p %c{2}: %m%n"/>
    + </Console>
    +
    + <!-- Regular File Appender -->
    + <!-- NOTE: if enabling multiple file appender make sure to use different file names -->
    + <!-- <File name="FA" fileName="${sys:spark.log.dir}/${sys:spark.log.file}">
    + <PatternLayout pattern="%d{ISO8601} %-5p [%t]: %c{2} (%F:%M(%L)) - %m%n" />
    + </File> -->
    +
    + <!-- Daily Rolling File Appender -->
    + <!-- Use %pid in the filePattern to append <process-id>@<host-name> to the filename if you want separate log files for different CLI session -->
    + <RollingFile name="DRFA" fileName="${sys:spark.log.dir}/${sys:spark.log.file}"
    + filePattern="${sys:spark.log.dir}/${sys:spark.log.file}.%d{yyyy-MM-dd}">
    + <PatternLayout pattern="%d{ISO8601} %-5p [%t]: %c{2} (%F:%M(%L)) - %m%n" />
    + <Policies>
    + <!-- rollover@mignight (interval = 1 means daily) -->
    + <TimeBasedTriggeringPolicy interval="1" modulate="true"/>
    + </Policies>
    + <!-- 30-day backup -->
    + <!-- <DefaultRolloverStrategy max="30"/> -->
    + </RollingFile>
    +
    + </Appenders>
    +
    + <Loggers>
    + <Root level="DEBUG">
    + <AppenderRef ref="${sys:spark.root.logger}" level="${sys:spark.log.level}"/>
    + </Root>
    +
    + <Logger name="org.apache.spark.repl.SparkIMain$exprTyper" level="INFO">
    + <AppenderRef ref="${sys:spark.root.logger}"/>
    + </Logger>
    + <Logger name="org.apache.spark.repl.SparkILoop$SparkILoopInterpreter" level="INFO">
    + <AppenderRef ref="${sys:spark.root.logger}"/>
    + </Logger>
    + <Logger name="org.eclipse.jetty" level="WARN">
    + <AppenderRef ref="${sys:spark.root.logger}"/>
    + </Logger>
    + <Logger name="org.eclipse.jetty.util.component.AbstractLifeCycle" level="ERROR">
    + <AppenderRef ref="${sys:spark.root.logger}"/>
    + </Logger>
    + </Loggers>
    +
    +</Configuration>

    http://git-wip-us.apache.org/repos/asf/hive/blob/c93d6c77/docs/xdocs/language_manual/cli.xml
    ----------------------------------------------------------------------
    diff --git a/docs/xdocs/language_manual/cli.xml b/docs/xdocs/language_manual/cli.xml
    index a293382..eb91e44 100644
    --- a/docs/xdocs/language_manual/cli.xml
    +++ b/docs/xdocs/language_manual/cli.xml
    @@ -163,7 +163,7 @@ Sample Usage:

      <section name="Logging" href="logging">
      <p>
    -Hive uses log4j for logging. These logs are not emitted to the standard output by default but are instead captured to a log file specified by Hive's log4j properties file. By default Hive will use <i>hive-log4j.default</i> in the <i>conf/</i> directory of the hive installation which writes out logs to <i>/tmp/$USER/hive.log</i> and uses the <i>WARN</i> level.
    +Hive uses log4j for logging. These logs are not emitted to the standard output by default but are instead captured to a log file specified by Hive's log4j properties file. By default Hive will use <i>hive-log4j2.xml</i> in the <i>conf/</i> directory of the hive installation which writes out logs to <i>/tmp/$USER/hive.log</i> and uses the <i>WARN</i> level.
      </p>
      <p>
      It is often desirable to emit the logs to the standard output and/or change the logging level for debugging purposes. These can be done from the command line as follows: </p>

    http://git-wip-us.apache.org/repos/asf/hive/blob/c93d6c77/hcatalog/bin/hcat_server.sh
    ----------------------------------------------------------------------
    diff --git a/hcatalog/bin/hcat_server.sh b/hcatalog/bin/hcat_server.sh
    index 6b09d3e..d1aecb8 100644
    --- a/hcatalog/bin/hcat_server.sh
    +++ b/hcatalog/bin/hcat_server.sh
    @@ -84,7 +84,7 @@ function start_hcat() {
        export AUX_CLASSPATH=${AUX_CLASSPATH}

        export HADOOP_HOME=$HADOOP_HOME
    - #export HADOOP_OPTS="-Dlog4j.configuration=file://${HCAT_PREFIX}/conf/log4j.properties"
    + #export HADOOP_OPTS="-Dlog4j.configurationFile=file://${HCAT_PREFIX}/conf/log4j2.xml"
        export HADOOP_OPTS="${HADOOP_OPTS} -server -XX:+UseConcMarkSweepGC -XX:ErrorFile=${HCAT_LOG_DIR}/hcat_err_pid%p.log -Xloggc:${HCAT_LOG_DIR}/hcat_gc.log-`date +'%Y%m%d%H%M'` -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintGCDateStamps"
        export HADOOP_HEAPSIZE=${HADOOP_HEAPSIZE:-2048} # 8G is better if you have it
        export METASTORE_PORT=${METASTORE_PORT:-9083}

    http://git-wip-us.apache.org/repos/asf/hive/blob/c93d6c77/hcatalog/bin/templeton.cmd
    ----------------------------------------------------------------------
    diff --git a/hcatalog/bin/templeton.cmd b/hcatalog/bin/templeton.cmd
    index e9a735d..759f654 100644
    --- a/hcatalog/bin/templeton.cmd
    +++ b/hcatalog/bin/templeton.cmd
    @@ -59,9 +59,9 @@ setlocal enabledelayedexpansion

        if not defined TEMPLETON_LOG4J (
          @rem must be prefixed with file: otherwise config is not picked up
    - set TEMPLETON_LOG4J=file:%WEBHCAT_CONF_DIR%\webhcat-log4j.properties
    + set TEMPLETON_LOG4J=file:%WEBHCAT_CONF_DIR%\webhcat-log4j2.xml
        )
    - set TEMPLETON_OPTS=-Dtempleton.log.dir=%TEMPLETON_LOG_DIR% -Dlog4j.configuration=%TEMPLETON_LOG4J% %HADOOP_OPTS%
    + set TEMPLETON_OPTS=-Dtempleton.log.dir=%TEMPLETON_LOG_DIR% -Dlog4j.configurationFile=%TEMPLETON_LOG4J% %HADOOP_OPTS%
        set arguments=%JAVA_HEAP_MAX% %TEMPLETON_OPTS% -classpath %CLASSPATH% org.apache.hive.hcatalog.templeton.Main

        if defined service_entry (

    http://git-wip-us.apache.org/repos/asf/hive/blob/c93d6c77/hcatalog/scripts/hcat_server_start.sh
    ----------------------------------------------------------------------
    diff --git a/hcatalog/scripts/hcat_server_start.sh b/hcatalog/scripts/hcat_server_start.sh
    index 1670b70..872d1b5 100755
    --- a/hcatalog/scripts/hcat_server_start.sh
    +++ b/hcatalog/scripts/hcat_server_start.sh
    @@ -70,7 +70,7 @@ export AUX_CLASSPATH=${AUX_CLASSPATH}


      export HADOOP_HOME=$HADOOP_HOME
    -#export HADOOP_OPTS="-Dlog4j.configuration=file://${ROOT}/conf/log4j.properties"
    +#export HADOOP_OPTS="-Dlog4j.configurationFile=file://${ROOT}/conf/log4j2.xml"
      export HADOOP_OPTS="${HADOOP_OPTS} -server -XX:+UseConcMarkSweepGC -XX:ErrorFile=${ROOT}/var/log/hcat_err_pid%p.log -Xloggc:${ROOT}/var/log/hcat_gc.log-`date +'%Y%m%d%H%M'` -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintGCDateStamps"
      export HADOOP_HEAPSIZE=2048 # 8G is better if you have it


    http://git-wip-us.apache.org/repos/asf/hive/blob/c93d6c77/hcatalog/src/docs/src/documentation/content/xdocs/configuration.xml
    ----------------------------------------------------------------------
    diff --git a/hcatalog/src/docs/src/documentation/content/xdocs/configuration.xml b/hcatalog/src/docs/src/documentation/content/xdocs/configuration.xml
    index 9757b9c..6385e40 100644
    --- a/hcatalog/src/docs/src/documentation/content/xdocs/configuration.xml
    +++ b/hcatalog/src/docs/src/documentation/content/xdocs/configuration.xml
    @@ -66,7 +66,7 @@ ${env.PIG_HOME}/bin/pig
           uncertainty.</p>

        <p><strong>Note:</strong> The location of the log files created by Templeton and some other properties
    - of the logging system are set in the webhcat-log4j.properties file.</p>
    + of the logging system are set in the webhcat-log4j2.xml file.</p>

        <section>
        <title>Variables</title>

    http://git-wip-us.apache.org/repos/asf/hive/blob/c93d6c77/hcatalog/src/docs/src/documentation/content/xdocs/install.xml
    ----------------------------------------------------------------------
    diff --git a/hcatalog/src/docs/src/documentation/content/xdocs/install.xml b/hcatalog/src/docs/src/documentation/content/xdocs/install.xml
    index 16da248..e2953a9 100644
    --- a/hcatalog/src/docs/src/documentation/content/xdocs/install.xml
    +++ b/hcatalog/src/docs/src/documentation/content/xdocs/install.xml
    @@ -241,7 +241,7 @@

          <p>Server activity logs are located in
          <em>root</em><code>/var/log/hcat_server</code>. Logging configuration is located at
    - <em>root</em><code>/conf/log4j.properties</code>. Server logging uses
    + <em>root</em><code>/conf/log4j2.xml</code>. Server logging uses
          <code>DailyRollingFileAppender</code> by default. It will generate a new
          file per day and does not expire old log files automatically.</p>


    http://git-wip-us.apache.org/repos/asf/hive/blob/c93d6c77/hcatalog/src/test/e2e/templeton/deployers/config/hive/hive-log4j.properties
    ----------------------------------------------------------------------
    diff --git a/hcatalog/src/test/e2e/templeton/deployers/config/hive/hive-log4j.properties b/hcatalog/src/test/e2e/templeton/deployers/config/hive/hive-log4j.properties
    deleted file mode 100644
    index 82684b3..0000000
    --- a/hcatalog/src/test/e2e/templeton/deployers/config/hive/hive-log4j.properties
    +++ /dev/null
    @@ -1,88 +0,0 @@
    -# Licensed to the Apache Software Foundation (ASF) under one
    -# or more contributor license agreements. See the NOTICE file
    -# distributed with this work for additional information
    -# regarding copyright ownership. The ASF licenses this file
    -# to you under the Apache License, Version 2.0 (the
    -# "License"); you may not use this file except in compliance
    -# with the License. You may obtain a copy of the License at
    -#
    -# http://www.apache.org/licenses/LICENSE-2.0
    -#
    -# Unless required by applicable law or agreed to in writing, software
    -# distributed under the License is distributed on an "AS IS" BASIS,
    -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    -# See the License for the specific language governing permissions and
    -# limitations under the License.
    -
    -# Define some default values that can be overridden by system properties
    -hive.log.threshold=ALL
    -hive.root.logger=DEBUG,DRFA
    -hive.log.dir=/tmp/ekoifman
    -hive.log.file=hive.log
    -
    -# Define the root logger to the system property "hadoop.root.logger".
    -log4j.rootLogger=${hive.root.logger}, EventCounter
    -
    -# Logging Threshold
    -log4j.threshold=${hive.log.threshold}
    -
    -#
    -# Daily Rolling File Appender
    -#
    -# Use the PidDailyerRollingFileAppend class instead if you want to use separate log files
    -# for different CLI session.
    -#
    -# log4j.appender.DRFA=org.apache.hadoop.hive.ql.log.PidDailyRollingFileAppender
    -
    -log4j.appender.DRFA=org.apache.log4j.DailyRollingFileAppender
    -
    -log4j.appender.DRFA.File=${hive.log.dir}/${hive.log.file}
    -
    -# Rollver at midnight
    -log4j.appender.DRFA.DatePattern=.yyyy-MM-dd
    -
    -# 30-day backup
    -#log4j.appender.DRFA.MaxBackupIndex=30
    -log4j.appender.DRFA.layout=org.apache.log4j.PatternLayout
    -
    -# Pattern format: Date LogLevel LoggerName LogMessage
    -#log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n
    -# Debugging Pattern format
    -log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %-5p [%t]: %c{2} (%F:%M(%L)) - %m%n
    -
    -
    -#
    -# console
    -# Add "console" to rootlogger above if you want to use this
    -#
    -
    -log4j.appender.console=org.apache.log4j.ConsoleAppender
    -log4j.appender.console.target=System.err
    -log4j.appender.console.layout=org.apache.log4j.PatternLayout
    -log4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} [%t]: %p %c{2}: %m%n
    -log4j.appender.console.encoding=UTF-8
    -
    -#custom logging levels
    -#log4j.logger.xxx=DEBUG
    -
    -#
    -# Event Counter Appender
    -# Sends counts of logging messages at different severity levels to Hadoop Metrics.
    -#
    -log4j.appender.EventCounter=org.apache.hadoop.hive.shims.HiveEventCounter
    -
    -
    -log4j.category.DataNucleus=ERROR,DRFA
    -log4j.category.Datastore=ERROR,DRFA
    -log4j.category.Datastore.Schema=ERROR,DRFA
    -log4j.category.JPOX.Datastore=ERROR,DRFA
    -log4j.category.JPOX.Plugin=ERROR,DRFA
    -log4j.category.JPOX.MetaData=ERROR,DRFA
    -log4j.category.JPOX.Query=ERROR,DRFA
    -log4j.category.JPOX.General=ERROR,DRFA
    -log4j.category.JPOX.Enhancer=ERROR,DRFA
    -
    -
    -# Silence useless ZK logs
    -log4j.logger.org.apache.zookeeper.server.NIOServerCnxn=WARN,DRFA
    -log4j.logger.org.apache.zookeeper.ClientCnxnSocketNIO=WARN,DRFA

    http://git-wip-us.apache.org/repos/asf/hive/blob/c93d6c77/hcatalog/src/test/e2e/templeton/deployers/config/hive/hive-log4j2.xml
    ----------------------------------------------------------------------
    diff --git a/hcatalog/src/test/e2e/templeton/deployers/config/hive/hive-log4j2.xml b/hcatalog/src/test/e2e/templeton/deployers/config/hive/hive-log4j2.xml
    new file mode 100644
    index 0000000..30f7603
    --- /dev/null
    +++ b/hcatalog/src/test/e2e/templeton/deployers/config/hive/hive-log4j2.xml
    @@ -0,0 +1,111 @@
    +<?xml version="1.0" encoding="UTF-8"?>
    +<!--
    + Licensed to the Apache Software Foundation (ASF) under one or more
    + contributor license agreements. See the NOTICE file distributed with
    + this work for additional information regarding copyright ownership.
    + The ASF licenses this file to You under the Apache License, Version 2.0
    + (the "License"); you may not use this file except in compliance with
    + the License. You may obtain a copy of the License at
    +
    + http://www.apache.org/licenses/LICENSE-2.0
    +
    + Unless required by applicable law or agreed to in writing, software
    + distributed under the License is distributed on an "AS IS" BASIS,
    + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    + See the License for the specific language governing permissions and
    + limitations under the License.
    +-->
    +
    +<Configuration status="info" strict="true" name="HCatE2ELog4j2"
    + packages="org.apache.hadoop.hive.ql.log">
    +
    + <Properties>
    + <Property name="hive.log.threshold">ALL</Property>
    + <Property name="hive.log.level">DEBUG</Property>
    + <Property name="hive.root.logger">DRFA</Property>
    + <Property name="hive.log.dir">${sys:java.io.tmpdir}/${sys:user.name}</Property>
    + <Property name="hive.log.file">hive.log</Property>
    + </Properties>
    +
    + <Appenders>
    + <Console name="console" target="SYSTEM_ERR">
    + <PatternLayout pattern="%d{yy/MM/dd HH:mm:ss} [%t]: %p %c{2}: %m%n"/>
    + </Console>
    +
    + <!-- Regular File Appender -->
    + <!-- NOTE: if enabling multiple file appender make sure to use different file names -->
    + <!-- <File name="FA" fileName="${sys:hive.log.dir}/${sys:hive.log.file}">
    + <PatternLayout pattern="%d{ISO8601} %-5p [%t]: %c{2} (%F:%M(%L)) - %m%n" />
    + </File> -->
    +
    + <!-- Daily Rolling File Appender -->
    + <!-- NOTE: if enabling multiple file appender make sure to use different file names -->
    + <!-- Use %pid in the filePattern to append <process-id>@<host-name> to the filename if you want separate log files for different CLI session -->
    + <RollingFile name="DRFA" fileName="${sys:hive.log.dir}/${sys:hive.log.file}"
    + filePattern="${sys:hive.log.dir}/${sys:hive.log.file}.%d{yyyy-MM-dd}">
    + <PatternLayout pattern="%d{ISO8601} %-5p [%t]: %c{2} (%F:%M(%L)) - %m%n" />
    + <Policies>
    + <!-- rollover@mignight (interval = 1 means daily) -->
    + <TimeBasedTriggeringPolicy interval="1" modulate="true"/>
    + </Policies>
    + <!-- 30-day backup -->
    + <!-- <DefaultRolloverStrategy max="30"/> -->
    + </RollingFile>
    +
    + <!-- Size based Rolling File Appender -->
    + <!-- NOTE: if enabling multiple file appender make sure to use different file names -->
    + <!-- <RollingFile name="RFA" fileName="${sys:hive.log.dir}/${sys:hive.log.file}"
    + filePattern="${sys:hive.log.dir}/${sys:hive.log.file}.%i">
    + <PatternLayout pattern="%d{ISO8601} %-5p [%t]: %c{2} (%F:%M(%L)) - %m%n" />
    + <Policies>
    + <SizeBasedTriggeringPolicy size="256 MB" />
    + </Policies>
    + <DefaultRolloverStrategy max="10"/>
    + </RollingFile> -->
    +
    + <!-- hiveeventcounter appender is loaded from configuration packages attribute.sends counts of logging messages@different severity levels to hadoop metrics. -->
    + <HiveEventCounter name="EventCounter"/>
    + </Appenders>
    +
    + <Loggers>
    + <Root level="${sys:hive.log.threshold}">
    + <AppenderRef ref="${sys:hive.root.logger}" level="${sys:hive.log.level}"/>
    + <AppenderRef ref="EventCounter" />
    + </Root>
    +
    + <Logger name="org.apache.zookeeper.server.NIOServerCnxn" level="WARN">
    + <AppenderRef ref="${sys:hive.root.logger}"/>
    + </Logger>
    + <Logger name="org.apache.zookeeper.ClientCnxnSocketNIO" level="WARN">
    + <AppenderRef ref="${sys:hive.root.logger}"/>
    + </Logger>
    + <Logger name="DataNucleus" level="ERROR">
    + <AppenderRef ref="${sys:hive.root.logger}"/>
    + </Logger>
    + <Logger name="Datastore" level="ERROR">
    + <AppenderRef ref="${sys:hive.root.logger}"/>
    + </Logger>
    + <Logger name="Datastore.Schema" level="ERROR">
    + <AppenderRef ref="${sys:hive.root.logger}"/>
    + </Logger>
    + <Logger name="JPOX.Datastore" level="ERROR">
    + <AppenderRef ref="${sys:hive.root.logger}"/>
    + </Logger>
    + <Logger name="JPOX.Plugin" level="ERROR">
    + <AppenderRef ref="${sys:hive.root.logger}"/>
    + </Logger>
    + <Logger name="JPOX.Metadata" level="ERROR">
    + <AppenderRef ref="${sys:hive.root.logger}"/>
    + </Logger>
    + <Logger name="JPOX.Query" level="ERROR">
    + <AppenderRef ref="${sys:hive.root.logger}"/>
    + </Logger>
    + <Logger name="JPOX.General" level="ERROR">
    + <AppenderRef ref="${sys:hive.root.logger}"/>
    + </Logger>
    + <Logger name="JPOX.Enhancer" level="ERROR">
    + <AppenderRef ref="${sys:hive.root.logger}"/>
    + </Logger>
    + </Loggers>
    +
    +</Configuration>

    http://git-wip-us.apache.org/repos/asf/hive/blob/c93d6c77/hcatalog/src/test/e2e/templeton/deployers/start_hive_services.sh
    ----------------------------------------------------------------------
    diff --git a/hcatalog/src/test/e2e/templeton/deployers/start_hive_services.sh b/hcatalog/src/test/e2e/templeton/deployers/start_hive_services.sh
    index 8cc9353..e59177c 100755
    --- a/hcatalog/src/test/e2e/templeton/deployers/start_hive_services.sh
    +++ b/hcatalog/src/test/e2e/templeton/deployers/start_hive_services.sh
    @@ -31,7 +31,7 @@ cp ${PROJ_HOME}/hcatalog/src/test/e2e/templeton/deployers/config/hive/hive-site.
      #cp ${PROJ_HOME}/hcatalog/src/test/e2e/templeton/deployers/config/hive/hive-site.mssql.xml ${HIVE_HOME}/conf/hive-site.xml

      cp ${PROJ_HOME}/hcatalog/src/test/e2e/templeton/deployers/config/webhcat/webhcat-site.xml ${HIVE_HOME}/hcatalog/etc/webhcat/webhcat-site.xml
    -cp ${PROJ_HOME}/hcatalog/src/test/e2e/templeton/deployers/config/hive/hive-log4j.properties ${HIVE_HOME}/conf/hive-log4j.properties
    +cp ${PROJ_HOME}/hcatalog/src/test/e2e/templeton/deployers/config/hive/hive-log4j2.xml ${HIVE_HOME}/conf/hive-log4j2.xml

      if [ -f ${MYSQL_CLIENT_JAR} ]; then
        cp ${MYSQL_CLIENT_JAR} ${HIVE_HOME}/lib

    http://git-wip-us.apache.org/repos/asf/hive/blob/c93d6c77/hcatalog/webhcat/svr/src/main/bin/webhcat_server.sh
    ----------------------------------------------------------------------
    diff --git a/hcatalog/webhcat/svr/src/main/bin/webhcat_server.sh b/hcatalog/webhcat/svr/src/main/bin/webhcat_server.sh
    index 0be8dde..c80fdd5 100644
    --- a/hcatalog/webhcat/svr/src/main/bin/webhcat_server.sh
    +++ b/hcatalog/webhcat/svr/src/main/bin/webhcat_server.sh
    @@ -215,11 +215,11 @@ else
      fi

      if [[ -z "$WEBHCAT_LOG4J" ]]; then
    - WEBHCAT_LOG4J="file://$base_dir/etc/webhcat/webhcat-log4j.properties";
    + WEBHCAT_LOG4J="file://$base_dir/etc/webhcat/webhcat-log4j2.xml";
      fi

      export HADOOP_USER_CLASSPATH_FIRST=true
    -export HADOOP_OPTS="${HADOOP_OPTS} -Dwebhcat.log.dir=$WEBHCAT_LOG_DIR -Dlog4j.configuration=$WEBHCAT_LOG4J"
    +export HADOOP_OPTS="${HADOOP_OPTS} -Dwebhcat.log.dir=$WEBHCAT_LOG_DIR -Dlog4j.configurationFile=$WEBHCAT_LOG4J"

      start_cmd="$HADOOP_PREFIX/bin/hadoop jar $JAR org.apache.hive.hcatalog.templeton.Main "


    http://git-wip-us.apache.org/repos/asf/hive/blob/c93d6c77/hcatalog/webhcat/svr/src/main/config/webhcat-log4j.properties
    ----------------------------------------------------------------------
    diff --git a/hcatalog/webhcat/svr/src/main/config/webhcat-log4j.properties b/hcatalog/webhcat/svr/src/main/config/webhcat-log4j.properties
    deleted file mode 100644
    index 866052c..0000000
    --- a/hcatalog/webhcat/svr/src/main/config/webhcat-log4j.properties
    +++ /dev/null
    @@ -1,45 +0,0 @@
    -# Licensed to the Apache Software Foundation (ASF) under one
    -# or more contributor license agreements. See the NOTICE file
    -# distributed with this work for additional information
    -# regarding copyright ownership. The ASF licenses this file
    -# to you under the Apache License, Version 2.0 (the
    -# "License"); you may not use this file except in compliance
    -# with the License. You may obtain a copy of the License at
    -#
    -# http://www.apache.org/licenses/LICENSE-2.0
    -#
    -# Unless required by applicable law or agreed to in writing,
    -# software distributed under the License is distributed on an
    -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
    -# KIND, either express or implied. See the License for the
    -# specific language governing permissions and limitations
    -# under the License.
    -
    -# Define some default values that can be overridden by system properties
    -webhcat.root.logger = INFO, standard
    -webhcat.log.dir = .
    -webhcat.log.file = webhcat.log
    -
    -log4j.rootLogger = ${webhcat.root.logger}
    -
    -# Logging Threshold
    -log4j.threshhold = DEBUG
    -
    -log4j.appender.standard = org.apache.log4j.DailyRollingFileAppender
    -log4j.appender.standard.File = ${webhcat.log.dir}/${webhcat.log.file}
    -
    -# Rollver at midnight
    -log4j.appender.DRFA.DatePattern = .yyyy-MM-dd
    -
    -log4j.appender.DRFA.layout = org.apache.log4j.PatternLayout
    -
    -log4j.appender.standard.layout = org.apache.log4j.PatternLayout
    -log4j.appender.standard.layout.conversionPattern = %-5p | %d{DATE} | %c | %m%n
    -
    -# Class logging settings
    -log4j.logger.com.sun.jersey = DEBUG
    -log4j.logger.com.sun.jersey.spi.container.servlet.WebComponent = ERROR
    -log4j.logger.org.apache.hadoop = INFO
    -log4j.logger.org.apache.hadoop.conf = WARN
    -log4j.logger.org.apache.zookeeper = WARN
    -log4j.logger.org.eclipse.jetty = INFO

    http://git-wip-us.apache.org/repos/asf/hive/blob/c93d6c77/hcatalog/webhcat/svr/src/main/config/webhcat-log4j2.xml
    ----------------------------------------------------------------------
    diff --git a/hcatalog/webhcat/svr/src/main/config/webhcat-log4j2.xml b/hcatalog/webhcat/svr/src/main/config/webhcat-log4j2.xml
    new file mode 100644
    index 0000000..40da974
    --- /dev/null
    +++ b/hcatalog/webhcat/svr/src/main/config/webhcat-log4j2.xml
    @@ -0,0 +1,75 @@
    +<?xml version="1.0" encoding="UTF-8"?>
    +<!--
    + Licensed to the Apache Software Foundation (ASF) under one or more
    + contributor license agreements. See the NOTICE file distributed with
    + this work for additional information regarding copyright ownership.
    + The ASF licenses this file to You under the Apache License, Version 2.0
    + (the "License"); you may not use this file except in compliance with
    + the License. You may obtain a copy of the License at
    +
    + http://www.apache.org/licenses/LICENSE-2.0
    +
    + Unless required by applicable law or agreed to in writing, software
    + distributed under the License is distributed on an "AS IS" BASIS,
    + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    + See the License for the specific language governing permissions and
    + limitations under the License.
    +-->
    +
    +<Configuration status="info" strict="true" name="WebhcatLog4j2"
    + packages="org.apache.hadoop.hive.ql.log">
    +
    + <Properties>
    + <Property name="webhcat.log.threshold">ALL</Property>
    + <Property name="webhcat.log.level">INFO</Property>
    + <Property name="webhcat.root.logger">standard</Property>
    + <Property name="webhcat.log.dir">.</Property>
    + <Property name="webhcat.log.file">webhcat.log</Property>
    + </Properties>
    +
    + <Appenders>
    + <Console name="console" target="SYSTEM_ERR">
    + <PatternLayout pattern="%d{yy/MM/dd HH:mm:ss} [%t]: %p %c{2}: %m%n"/>
    + </Console>
    +
    + <!-- Daily Rolling File Appender -->
    + <!-- Use %pid in the filePattern to append <process-id>@<host-name> to the filename if you want separate log files for different CLI session -->
    + <RollingFile name="standard" fileName="${sys:webhcat.log.dir}/${sys:webhcat.log.file}"
    + filePattern="${sys:webhcat.log.dir}/${sys:webhcat.log.file}.%d{yyyy-MM-dd}">
    + <PatternLayout pattern="%-5p | %d{DATE} | %c | %m%n" />
    + <Policies>
    + <!-- rollover@mignight (interval = 1 means daily) -->
    + <TimeBasedTriggeringPolicy interval="1" modulate="true"/>
    + </Policies>
    + <!-- 30-day backup -->
    + <!-- <DefaultRolloverStrategy max="30"/> -->
    + </RollingFile>
    +
    + </Appenders>
    +
    + <Loggers>
    + <Root level="${sys:webhcat.log.threshold}">
    + <AppenderRef ref="${sys:webhcat.root.logger}" level="${sys:webhcat.log.level}"/>
    + </Root>
    +
    + <Logger name="com.sun.jersey" level="DEBUG">
    + <AppenderRef ref="${sys:webhcat.root.logger}"/>
    + </Logger>
    + <Logger name="com.sun.jersey.spi.container.servlet.WebComponent" level="ERROR">
    + <AppenderRef ref="${sys:webhcat.root.logger}"/>
    + </Logger>
    + <Logger name="org.apache.hadoop" level="INFO">
    + <AppenderRef ref="${sys:webhcat.root.logger}"/>
    + </Logger>
    + <Logger name="org.apache.hadoop.conf" level="WARN">
    + <AppenderRef ref="${sys:webhcat.root.logger}"/>
    + </Logger>
    + <Logger name="org.apache.zookeeper" level="WARN">
    + <AppenderRef ref="${sys:webhcat.root.logger}"/>
    + </Logger>
    + <Logger name="org.eclipse.jetty" level="INFO">
    + <AppenderRef ref="${sys:webhcat.root.logger}"/>
    + </Logger>
    + </Loggers>
    +
    +</Configuration>

    http://git-wip-us.apache.org/repos/asf/hive/blob/c93d6c77/hplsql/src/main/java/org/apache/hive/hplsql/Exec.java
    ----------------------------------------------------------------------
    diff --git a/hplsql/src/main/java/org/apache/hive/hplsql/Exec.java b/hplsql/src/main/java/org/apache/hive/hplsql/Exec.java
    index 73f470c..268c218 100644
    --- a/hplsql/src/main/java/org/apache/hive/hplsql/Exec.java
    +++ b/hplsql/src/main/java/org/apache/hive/hplsql/Exec.java
    @@ -670,6 +670,8 @@ public class Exec extends HplsqlBaseVisitor<Integer> {
          if (!parseArguments(args)) {
            return 1;
          }
    + // specify the default log4j2 properties file.
    + System.setProperty("log4j.configurationFile", "hive-log4j2.xml");
          conf = new Conf();
          conf.init();
          conn = new Conn(this);

    http://git-wip-us.apache.org/repos/asf/hive/blob/c93d6c77/itests/hive-unit/src/test/java/org/apache/hive/service/cli/operation/TestOperationLoggingLayout.java
    ----------------------------------------------------------------------
    diff --git a/itests/hive-unit/src/test/java/org/apache/hive/service/cli/operation/TestOperationLoggingLayout.java b/itests/hive-unit/src/test/java/org/apache/hive/service/cli/operation/TestOperationLoggingLayout.java
    new file mode 100644
    index 0000000..93c16de
    --- /dev/null
    +++ b/itests/hive-unit/src/test/java/org/apache/hive/service/cli/operation/TestOperationLoggingLayout.java
    @@ -0,0 +1,136 @@
    +package org.apache.hive.service.cli.operation;
    +
    +import java.io.File;
    +import java.util.HashMap;
    +import java.util.Iterator;
    +import java.util.Map;
    +
    +import org.apache.hadoop.hive.conf.HiveConf;
    +import org.apache.hive.jdbc.miniHS2.MiniHS2;
    +import org.apache.hive.service.cli.CLIServiceClient;
    +import org.apache.hive.service.cli.FetchOrientation;
    +import org.apache.hive.service.cli.FetchType;
    +import org.apache.hive.service.cli.OperationHandle;
    +import org.apache.hive.service.cli.RowSet;
    +import org.apache.hive.service.cli.SessionHandle;
    +import org.junit.After;
    +import org.junit.AfterClass;
    +import org.junit.Assert;
    +import org.junit.Before;
    +import org.junit.BeforeClass;
    +import org.junit.Test;
    +
    +/**
    + * Tests to verify operation logging layout for different modes.
    + */
    +public class TestOperationLoggingLayout {
    + protected static HiveConf hiveConf;
    + protected static String tableName;
    + private File dataFile;
    + protected CLIServiceClient client;
    + protected static MiniHS2 miniHS2 = null;
    + protected static Map<String, String> confOverlay;
    + protected SessionHandle sessionHandle;
    + protected final String sql = "select * from " + tableName;
    + private final String sqlCntStar = "select count(*) from " + tableName;
    +
    + @BeforeClass
    + public static void setUpBeforeClass() throws Exception {
    + tableName = "TestOperationLoggingLayout_table";
    + hiveConf = new HiveConf();
    + hiveConf.set(HiveConf.ConfVars.HIVE_SERVER2_LOGGING_OPERATION_LEVEL.varname, "execution");
    + // We need to set the below parameter to test performance level logging
    + hiveConf.set("hive.ql.log.PerfLogger.level", "INFO,DRFA");
    + miniHS2 = new MiniHS2(hiveConf);
    + confOverlay = new HashMap<String, String>();
    + confOverlay.put(HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY.varname, "false");
    + miniHS2.start(confOverlay);
    + }
    +
    + /**
    + * Open a session, and create a table for cases usage
    + *
    + * @throws Exception
    + */
    + @Before
    + public void setUp() throws Exception {
    + dataFile = new File(hiveConf.get("test.data.files"), "kv1.txt");
    + client = miniHS2.getServiceClient();
    + sessionHandle = setupSession();
    + }
    +
    + @After
    + public void tearDown() throws Exception {
    + // Cleanup
    + String queryString = "DROP TABLE " + tableName;
    + client.executeStatement(sessionHandle, queryString, null);
    +
    + client.closeSession(sessionHandle);
    + }
    +
    + @AfterClass
    + public static void tearDownAfterClass() throws Exception {
    + miniHS2.stop();
    + }
    +
    + @Test
    + public void testSwitchLogLayout() throws Exception {
    + // verify whether the sql operation log is generated and fetch correctly.
    + OperationHandle operationHandle = client.executeStatement(sessionHandle, sqlCntStar, null);
    + RowSet rowSetLog = client.fetchResults(operationHandle, FetchOrientation.FETCH_FIRST, 1000,
    + FetchType.LOG);
    + Iterator<Object[]> iter = rowSetLog.iterator();
    + // non-verbose pattern is %-5p : %m%n. Look for " : "
    + while (iter.hasNext()) {
    + String row = iter.next()[0].toString();
    + Assert.assertEquals(true, row.matches("^(FATAL|ERROR|WARN|INFO|DEBUG|TRACE).*$"));
    + }
    +
    + String queryString = "set hive.server2.logging.operation.level=verbose";
    + client.executeStatement(sessionHandle, queryString, null);
    + operationHandle = client.executeStatement(sessionHandle, sqlCntStar, null);
    + // just check for first few lines, some log lines are multi-line strings which can break format
    + // checks below
    + rowSetLog = client.fetchResults(operationHandle, FetchOrientation.FETCH_FIRST, 10,
    + FetchType.LOG);
    + iter = rowSetLog.iterator();
    + // verbose pattern is "%d{yy/MM/dd HH:mm:ss} %p %c{2}: %m%n"
    + while (iter.hasNext()) {
    + String row = iter.next()[0].toString();
    + // just check if the log line starts with date
    + Assert.assertEquals(true,
    + row.matches("^\\d{2}[/](0[1-9]|1[012])[/](0[1-9]|[12][0-9]|3[01]).*$"));
    + }
    + }
    +
    + private SessionHandle setupSession() throws Exception {
    + // Open a session
    + SessionHandle sessionHandle = client.openSession(null, null, null);
    +
    + // Change lock manager to embedded mode
    + String queryString = "SET hive.lock.manager=" +
    + "org.apache.hadoop.hive.ql.lockmgr.EmbeddedLockManager";
    + client.executeStatement(sessionHandle, queryString, null);
    +
    + // Drop the table if it exists
    + queryString = "DROP TABLE IF EXISTS " + tableName;
    + client.executeStatement(sessionHandle, queryString, null);
    +
    + // Create a test table
    + queryString = "create table " + tableName + " (key int, value string)";
    + client.executeStatement(sessionHandle, queryString, null);
    +
    + // Load data
    + queryString = "load data local inpath '" + dataFile + "' into table " + tableName;
    + client.executeStatement(sessionHandle, queryString, null);
    +
    + // Precondition check: verify whether the table is created and data is fetched correctly.
    + OperationHandle operationHandle = client.executeStatement(sessionHandle, sql, null);
    + RowSet rowSetResult = client.fetchResults(operationHandle);
    + Assert.assertEquals(500, rowSetResult.numRows());
    + Assert.assertEquals(238, rowSetResult.iterator().next()[0]);
    + Assert.assertEquals("val_238", rowSetResult.iterator().next()[1]);
    +
    + return sessionHandle;
    + }
    +}
  • Sershe at Aug 25, 2015 at 9:28 pm
    HIVE-11562: Typo in hive-log4j2.xml throws unknown level exception (Prasanth Jayachandran reviewed by Sergey Shelukhin)


    Project: http://git-wip-us.apache.org/repos/asf/hive/repo
    Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/d307abbf
    Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/d307abbf
    Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/d307abbf

    Branch: refs/heads/llap
    Commit: d307abbf1093ffdc2599489bd6f67bdb8dcb3e14
    Parents: c93d6c7
    Author: Prasanth Jayachandran <j.prasanth.j@gmail.com>
    Authored: Fri Aug 14 14:01:57 2015 -0700
    Committer: Prasanth Jayachandran <j.prasanth.j@gmail.com>
    Committed: Fri Aug 14 14:01:57 2015 -0700

    ----------------------------------------------------------------------
      data/conf/hive-log4j2.xml | 2 +-
      1 file changed, 1 insertion(+), 1 deletion(-)
    ----------------------------------------------------------------------


    http://git-wip-us.apache.org/repos/asf/hive/blob/d307abbf/data/conf/hive-log4j2.xml
    ----------------------------------------------------------------------
    diff --git a/data/conf/hive-log4j2.xml b/data/conf/hive-log4j2.xml
    index c9adfa2..a40be2c 100644
    --- a/data/conf/hive-log4j2.xml
    +++ b/data/conf/hive-log4j2.xml
    @@ -70,7 +70,7 @@

        <Loggers>
          <Root level="${sys:hive.log.threshold}">
    - <AppenderRef ref="${sys:hive.root.logger}" level="{sys:hive.log.level}"/>
    + <AppenderRef ref="${sys:hive.root.logger}" level="${sys:hive.log.level}"/>
            <AppenderRef ref="EventCounter" />
          </Root>
  • Sershe at Aug 25, 2015 at 9:28 pm
    HIVE-11441: No DDL allowed on table if user accidentally set table location wrong (Daniel Dai reviewed by Thejas Nair)


    Project: http://git-wip-us.apache.org/repos/asf/hive/repo
    Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/2ccd0616
    Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/2ccd0616
    Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/2ccd0616

    Branch: refs/heads/llap
    Commit: 2ccd061691cd52ed9fa341b61590edb2c022b031
    Parents: 17e95c7
    Author: Daniel Dai <daijy@hortonworks.com>
    Authored: Fri Aug 14 15:28:44 2015 -0700
    Committer: Daniel Dai <daijy@hortonworks.com>
    Committed: Fri Aug 14 15:28:44 2015 -0700

    ----------------------------------------------------------------------
      .../apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java | 11 +++++++++++
      .../queries/clientnegative/alter_table_wrong_location.q | 4 ++++
      .../clientnegative/alter_table_wrong_location.q.out | 9 +++++++++
      3 files changed, 24 insertions(+)
    ----------------------------------------------------------------------


    http://git-wip-us.apache.org/repos/asf/hive/blob/2ccd0616/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java
    ----------------------------------------------------------------------
    diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java
    index 21625bc..9f8c756 100644
    --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java
    +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java
    @@ -23,6 +23,7 @@ import org.antlr.runtime.tree.CommonTree;
      import org.antlr.runtime.tree.Tree;
      import org.apache.commons.logging.Log;
      import org.apache.commons.logging.LogFactory;
    +import org.apache.hadoop.fs.FileSystem;
      import org.apache.hadoop.fs.Path;
      import org.apache.hadoop.hive.common.JavaUtils;
      import org.apache.hadoop.hive.conf.HiveConf;
    @@ -137,6 +138,7 @@ import org.apache.hadoop.mapred.InputFormat;
      import org.apache.hadoop.mapred.TextInputFormat;
      import org.apache.hadoop.util.StringUtils;

    +import java.io.FileNotFoundException;
      import java.io.Serializable;
      import java.lang.reflect.Constructor;
      import java.lang.reflect.InvocationTargetException;
    @@ -1464,6 +1466,15 @@ public class DDLSemanticAnalyzer extends BaseSemanticAnalyzer {
            HashMap<String, String> partSpec) throws SemanticException {

          String newLocation = unescapeSQLString(ast.getChild(0).getText());
    + try {
    + // To make sure host/port pair is valid, the status of the location
    + // does not matter
    + FileSystem.get(new URI(newLocation), conf).getFileStatus(new Path(newLocation));
    + } catch (FileNotFoundException e) {
    + // Only check host/port pair is valid, wheter the file exist or not does not matter
    + } catch (Exception e) {
    + throw new SemanticException("Cannot connect to namenode, please check if host/port pair for " + newLocation + " is valid", e);
    + }
          addLocationToOutputs(newLocation);
          AlterTableDesc alterTblDesc = new AlterTableDesc(tableName, newLocation, partSpec);


    http://git-wip-us.apache.org/repos/asf/hive/blob/2ccd0616/ql/src/test/queries/clientnegative/alter_table_wrong_location.q
    ----------------------------------------------------------------------
    diff --git a/ql/src/test/queries/clientnegative/alter_table_wrong_location.q b/ql/src/test/queries/clientnegative/alter_table_wrong_location.q
    new file mode 100644
    index 0000000..3721867
    --- /dev/null
    +++ b/ql/src/test/queries/clientnegative/alter_table_wrong_location.q
    @@ -0,0 +1,4 @@
    +create table testwrongloc(id int);
    +
    +-- Assume port 12345 is not open
    +alter table testwrongloc set location "hdfs://localhost:12345/tmp/testwrongloc";

    http://git-wip-us.apache.org/repos/asf/hive/blob/2ccd0616/ql/src/test/results/clientnegative/alter_table_wrong_location.q.out
    ----------------------------------------------------------------------
    diff --git a/ql/src/test/results/clientnegative/alter_table_wrong_location.q.out b/ql/src/test/results/clientnegative/alter_table_wrong_location.q.out
    new file mode 100644
    index 0000000..d788d55
    --- /dev/null
    +++ b/ql/src/test/results/clientnegative/alter_table_wrong_location.q.out
    @@ -0,0 +1,9 @@
    +PREHOOK: query: create table testwrongloc(id int)
    +PREHOOK: type: CREATETABLE
    +PREHOOK: Output: database:default
    +PREHOOK: Output: default@testwrongloc
    +POSTHOOK: query: create table testwrongloc(id int)
    +POSTHOOK: type: CREATETABLE
    +POSTHOOK: Output: database:default
    +POSTHOOK: Output: default@testwrongloc
    +#### A masked pattern was here ####
  • Sershe at Aug 25, 2015 at 9:28 pm
    HIVE-11556: HiveFilter.copy should take the condition given as a parameter (Jesus Camacho Rodriguez, reviewed by Ashutosh Chauhan)


    Project: http://git-wip-us.apache.org/repos/asf/hive/repo
    Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/147347a3
    Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/147347a3
    Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/147347a3

    Branch: refs/heads/llap
    Commit: 147347a31dd021ac3a6f956c7c02a77b505bce7b
    Parents: 3071ce9
    Author: Jesus Camacho Rodriguez <jcamacho@apache.org>
    Authored: Sat Aug 15 10:49:55 2015 +0300
    Committer: Jesus Camacho Rodriguez <jcamacho@apache.org>
    Committed: Sat Aug 15 10:49:55 2015 +0300

    ----------------------------------------------------------------------
      .../hadoop/hive/ql/optimizer/calcite/reloperators/HiveFilter.java | 2 +-
      1 file changed, 1 insertion(+), 1 deletion(-)
    ----------------------------------------------------------------------


    http://git-wip-us.apache.org/repos/asf/hive/blob/147347a3/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveFilter.java
    ----------------------------------------------------------------------
    diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveFilter.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveFilter.java
    index de61e48..eb97bec 100644
    --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveFilter.java
    +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveFilter.java
    @@ -39,7 +39,7 @@ public class HiveFilter extends Filter implements HiveRelNode {
        @Override
        public Filter copy(RelTraitSet traitSet, RelNode input, RexNode condition) {
          assert traitSet.containsIfApplicable(HiveRelNode.CONVENTION);
    - return new HiveFilter(getCluster(), traitSet, input, getCondition());
    + return new HiveFilter(getCluster(), traitSet, input, condition);
        }

        @Override
  • Sershe at Aug 25, 2015 at 9:28 pm
    HIVE-11542 : port fileId support on shims and splits from llap branch (Sergey Shelukhin, reviewed by Prasanth Jayachandran)


    Project: http://git-wip-us.apache.org/repos/asf/hive/repo
    Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/3b6825b5
    Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/3b6825b5
    Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/3b6825b5

    Branch: refs/heads/llap
    Commit: 3b6825b5b61e943e8e41743f5cbf6d640e0ebdf5
    Parents: e059409
    Author: Sergey Shelukhin <sershe@apache.org>
    Authored: Mon Aug 17 15:16:57 2015 -0700
    Committer: Sergey Shelukhin <sershe@apache.org>
    Committed: Mon Aug 17 15:16:57 2015 -0700

    ----------------------------------------------------------------------
      .../org/apache/hadoop/hive/conf/HiveConf.java | 2 +
      .../org/apache/hadoop/hive/ql/io/AcidUtils.java | 183 ++++++++++++++-----
      .../hadoop/hive/ql/io/orc/OrcInputFormat.java | 97 +++++++---
      .../apache/hadoop/hive/ql/io/orc/OrcSplit.java | 25 ++-
      .../hive/ql/txn/compactor/CompactorMR.java | 13 +-
      .../hadoop/hive/ql/txn/compactor/Initiator.java | 9 +-
      .../apache/hadoop/hive/ql/io/TestAcidUtils.java | 27 +--
      .../hive/ql/io/orc/TestInputOutputFormat.java | 6 +-
      .../hadoop/hive/shims/Hadoop20SShims.java | 11 ++
      .../apache/hadoop/hive/shims/Hadoop23Shims.java | 66 +++++++
      .../apache/hadoop/hive/shims/HadoopShims.java | 15 ++
      11 files changed, 348 insertions(+), 106 deletions(-)
    ----------------------------------------------------------------------


    http://git-wip-us.apache.org/repos/asf/hive/blob/3b6825b5/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
    ----------------------------------------------------------------------
    diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
    index 9a6781b..da171b1 100644
    --- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
    +++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
    @@ -1024,6 +1024,8 @@ public class HiveConf extends Configuration {
              "data is read remotely (from the client or HS2 machine) and sent to all the tasks."),
          HIVE_ORC_CACHE_STRIPE_DETAILS_SIZE("hive.orc.cache.stripe.details.size", 10000,
              "Max cache size for keeping meta info about orc splits cached in the client."),
    + HIVE_ORC_INCLUDE_FILE_ID_IN_SPLITS("hive.orc.splits.include.fileid", true,
    + "Include file ID in splits on file systems thaty support it."),
          HIVE_ORC_COMPUTE_SPLITS_NUM_THREADS("hive.orc.compute.splits.num.threads", 10,
              "How many threads orc should use to create splits in parallel."),
          HIVE_ORC_SKIP_CORRUPT_DATA("hive.exec.orc.skip.corrupt.data", false,

    http://git-wip-us.apache.org/repos/asf/hive/blob/3b6825b5/ql/src/java/org/apache/hadoop/hive/ql/io/AcidUtils.java
    ----------------------------------------------------------------------
    diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/AcidUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/io/AcidUtils.java
    index c7e0780..30db513 100644
    --- a/ql/src/java/org/apache/hadoop/hive/ql/io/AcidUtils.java
    +++ b/ql/src/java/org/apache/hadoop/hive/ql/io/AcidUtils.java
    @@ -28,6 +28,9 @@ import org.apache.hadoop.fs.PathFilter;
      import org.apache.hadoop.hive.common.ValidTxnList;
      import org.apache.hadoop.hive.shims.HadoopShims;
      import org.apache.hadoop.hive.shims.ShimLoader;
    +import org.apache.hadoop.hive.shims.HadoopShims.HdfsFileStatusWithId;
    +
    +import com.google.common.annotations.VisibleForTesting;

      import java.io.IOException;
      import java.util.ArrayList;
    @@ -221,7 +224,7 @@ public class AcidUtils {
           * Get the list of original files.
           * @return the list of original files (eg. 000000_0)
           */
    - List<FileStatus> getOriginalFiles();
    + List<HdfsFileStatusWithId> getOriginalFiles();

          /**
           * Get the list of base and delta directories that are valid and not
    @@ -423,6 +426,20 @@ public class AcidUtils {
          return false;
        }

    + @VisibleForTesting
    + public static Directory getAcidState(Path directory,
    + Configuration conf,
    + ValidTxnList txnList
    + ) throws IOException {
    + return getAcidState(directory, conf, txnList, false);
    + }
    +
    + /** State class for getChildState; cannot modify 2 things in a method. */
    + private static class TxnBase {
    + private FileStatus status;
    + private long txn;
    + }
    +
        /**
         * Get the ACID state of the given directory. It finds the minimal set of
         * base and diff directories. Note that because major compactions don't
    @@ -436,51 +453,40 @@ public class AcidUtils {
         */
        public static Directory getAcidState(Path directory,
                                             Configuration conf,
    - ValidTxnList txnList
    + ValidTxnList txnList,
    + boolean useFileIds
                                             ) throws IOException {
          FileSystem fs = directory.getFileSystem(conf);
    - FileStatus bestBase = null;
    - long bestBaseTxn = 0;
          final List<ParsedDelta> deltas = new ArrayList<ParsedDelta>();
          List<ParsedDelta> working = new ArrayList<ParsedDelta>();
          List<FileStatus> originalDirectories = new ArrayList<FileStatus>();
          final List<FileStatus> obsolete = new ArrayList<FileStatus>();
    - List<FileStatus> children = SHIMS.listLocatedStatus(fs, directory,
    - hiddenFileFilter);
    - for(FileStatus child: children) {
    - Path p = child.getPath();
    - String fn = p.getName();
    - if (fn.startsWith(BASE_PREFIX) && child.isDir()) {
    - long txn = parseBase(p);
    - if (bestBase == null) {
    - bestBase = child;
    - bestBaseTxn = txn;
    - } else if (bestBaseTxn < txn) {
    - obsolete.add(bestBase);
    - bestBase = child;
    - bestBaseTxn = txn;
    - } else {
    - obsolete.add(child);
    - }
    - } else if (fn.startsWith(DELTA_PREFIX) && child.isDir()) {
    - ParsedDelta delta = parseDelta(child);
    - if (txnList.isTxnRangeValid(delta.minTransaction,
    - delta.maxTransaction) !=
    - ValidTxnList.RangeResponse.NONE) {
    - working.add(delta);
    - }
    - } else {
    - // This is just the directory. We need to recurse and find the actual files. But don't
    - // do this until we have determined there is no base. This saves time. Plus,
    - // it is possible that the cleaner is running and removing these original files,
    - // in which case recursing through them could cause us to get an error.
    - originalDirectories.add(child);
    + List<HdfsFileStatusWithId> childrenWithId = null;
    + if (useFileIds) {
    + try {
    + childrenWithId = SHIMS.listLocatedHdfsStatus(fs, directory, hiddenFileFilter);
    + } catch (Throwable t) {
    + LOG.error("Failed to get files with ID; using regular API", t);
    + useFileIds = false;
    + }
    + }
    + TxnBase bestBase = new TxnBase();
    + final List<HdfsFileStatusWithId> original = new ArrayList<>();
    + if (childrenWithId != null) {
    + for (HdfsFileStatusWithId child : childrenWithId) {
    + getChildState(child.getFileStatus(), child, txnList, working,
    + originalDirectories, original, obsolete, bestBase);
    + }
    + } else {
    + List<FileStatus> children = SHIMS.listLocatedStatus(fs, directory, hiddenFileFilter);
    + for (FileStatus child : children) {
    + getChildState(
    + child, null, txnList, working, originalDirectories, original, obsolete, bestBase);
            }
          }

    - final List<FileStatus> original = new ArrayList<FileStatus>();
    - // if we have a base, the original files are obsolete.
    - if (bestBase != null) {
    + // If we have a base, the original files are obsolete.
    + if (bestBase.status != null) {
            // remove the entries so we don't get confused later and think we should
            // use them.
            original.clear();
    @@ -488,12 +494,12 @@ public class AcidUtils {
            // Okay, we're going to need these originals. Recurse through them and figure out what we
            // really need.
            for (FileStatus origDir : originalDirectories) {
    - findOriginals(fs, origDir, original);
    + findOriginals(fs, origDir, original, useFileIds);
            }
          }

          Collections.sort(working);
    - long current = bestBaseTxn;
    + long current = bestBase.txn;
          int lastStmtId = -1;
          for(ParsedDelta next: working) {
            if (next.maxTransaction > current) {
    @@ -516,7 +522,7 @@ public class AcidUtils {
            }
          }

    - final Path base = bestBase == null ? null : bestBase.getPath();
    + final Path base = bestBase.status == null ? null : bestBase.status.getPath();
          LOG.debug("in directory " + directory.toUri().toString() + " base = " + base + " deltas = " +
              deltas.size());

    @@ -528,7 +534,7 @@ public class AcidUtils {
            }

            @Override
    - public List<FileStatus> getOriginalFiles() {
    + public List<HdfsFileStatusWithId> getOriginalFiles() {
              return original;
            }

    @@ -544,23 +550,100 @@ public class AcidUtils {
          };
        }

    + private static void getChildState(FileStatus child, HdfsFileStatusWithId childWithId,
    + ValidTxnList txnList, List<ParsedDelta> working, List<FileStatus> originalDirectories,
    + List<HdfsFileStatusWithId> original, List<FileStatus> obsolete, TxnBase bestBase) {
    + Path p = child.getPath();
    + String fn = p.getName();
    + if (fn.startsWith(BASE_PREFIX) && child.isDir()) {
    + long txn = parseBase(p);
    + if (bestBase.status == null) {
    + bestBase.status = child;
    + bestBase.txn = txn;
    + } else if (bestBase.txn < txn) {
    + obsolete.add(bestBase.status);
    + bestBase.status = child;
    + bestBase.txn = txn;
    + } else {
    + obsolete.add(child);
    + }
    + } else if (fn.startsWith(DELTA_PREFIX) && child.isDir()) {
    + ParsedDelta delta = parseDelta(child);
    + if (txnList.isTxnRangeValid(delta.minTransaction,
    + delta.maxTransaction) !=
    + ValidTxnList.RangeResponse.NONE) {
    + working.add(delta);
    + }
    + } else if (child.isDir()) {
    + // This is just the directory. We need to recurse and find the actual files. But don't
    + // do this until we have determined there is no base. This saves time. Plus,
    + // it is possible that the cleaner is running and removing these original files,
    + // in which case recursing through them could cause us to get an error.
    + originalDirectories.add(child);
    + } else {
    + original.add(createOriginalObj(childWithId, child));
    + }
    + }
    +
    + public static HdfsFileStatusWithId createOriginalObj(
    + HdfsFileStatusWithId childWithId, FileStatus child) {
    + return childWithId != null ? childWithId : new HdfsFileStatusWithoutId(child);
    + }
    +
    + private static class HdfsFileStatusWithoutId implements HdfsFileStatusWithId {
    + private FileStatus fs;
    +
    + public HdfsFileStatusWithoutId(FileStatus fs) {
    + this.fs = fs;
    + }
    +
    + @Override
    + public FileStatus getFileStatus() {
    + return fs;
    + }
    +
    + @Override
    + public Long getFileId() {
    + return null;
    + }
    + }
    +
        /**
    - * Find the original files (non-ACID layout) recursively under the partition
    - * directory.
    + * Find the original files (non-ACID layout) recursively under the partition directory.
         * @param fs the file system
    - * @param stat the file/directory to add
    + * @param stat the directory to add
         * @param original the list of original files
         * @throws IOException
         */
        private static void findOriginals(FileSystem fs, FileStatus stat,
    - List<FileStatus> original
    - ) throws IOException {
    - if (stat.isDir()) {
    - for(FileStatus child: SHIMS.listLocatedStatus(fs, stat.getPath(), hiddenFileFilter)) {
    - findOriginals(fs, child, original);
    + List<HdfsFileStatusWithId> original, boolean useFileIds) throws IOException {
    + assert stat.isDir();
    + List<HdfsFileStatusWithId> childrenWithId = null;
    + if (useFileIds) {
    + try {
    + childrenWithId = SHIMS.listLocatedHdfsStatus(fs, stat.getPath(), hiddenFileFilter);
    + } catch (Throwable t) {
    + LOG.error("Failed to get files with ID; using regular API", t);
    + useFileIds = false;
    + }
    + }
    + if (childrenWithId != null) {
    + for (HdfsFileStatusWithId child : childrenWithId) {
    + if (child.getFileStatus().isDir()) {
    + findOriginals(fs, child.getFileStatus(), original, useFileIds);
    + } else {
    + original.add(child);
    + }
            }
          } else {
    - original.add(stat);
    + List<FileStatus> children = SHIMS.listLocatedStatus(fs, stat.getPath(), hiddenFileFilter);
    + for (FileStatus child : children) {
    + if (child.isDir()) {
    + findOriginals(fs, child, original, useFileIds);
    + } else {
    + original.add(createOriginalObj(null, child));
    + }
    + }
          }
        }
      }

    http://git-wip-us.apache.org/repos/asf/hive/blob/3b6825b5/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java
    ----------------------------------------------------------------------
    diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java
    index 6ed7872..fd6d2ad 100644
    --- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java
    +++ b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java
    @@ -48,12 +48,14 @@ import org.apache.hadoop.hive.metastore.api.hive_metastoreConstants;
      import org.apache.hadoop.hive.ql.exec.Utilities;
      import org.apache.hadoop.hive.ql.exec.vector.VectorizedInputFormatInterface;
      import org.apache.hadoop.hive.ql.io.AcidInputFormat;
    +import org.apache.hadoop.hive.ql.io.AcidInputFormat.DeltaMetaData;
      import org.apache.hadoop.hive.ql.io.AcidOutputFormat;
      import org.apache.hadoop.hive.ql.io.AcidUtils;
      import org.apache.hadoop.hive.ql.io.CombineHiveInputFormat;
      import org.apache.hadoop.hive.ql.io.InputFormatChecker;
      import org.apache.hadoop.hive.ql.io.RecordIdentifier;
      import org.apache.hadoop.hive.ql.io.StatsProvidingRecordReader;
    +import org.apache.hadoop.hive.ql.io.orc.OrcInputFormat.Context;
      import org.apache.hadoop.hive.ql.io.sarg.ConvertAstToSearchArg;
      import org.apache.hadoop.hive.ql.io.sarg.PredicateLeaf;
      import org.apache.hadoop.hive.ql.io.sarg.SearchArgument;
    @@ -63,6 +65,7 @@ import org.apache.hadoop.hive.serde2.SerDeStats;
      import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
      import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
      import org.apache.hadoop.hive.shims.HadoopShims;
    +import org.apache.hadoop.hive.shims.HadoopShims.HdfsFileStatusWithId;
      import org.apache.hadoop.hive.shims.ShimLoader;
      import org.apache.hadoop.io.LongWritable;
      import org.apache.hadoop.io.NullWritable;
    @@ -73,6 +76,7 @@ import org.apache.hadoop.mapred.JobConf;
      import org.apache.hadoop.mapred.Reporter;
      import org.apache.hadoop.util.StringUtils;

    +import com.google.common.annotations.VisibleForTesting;
      import com.google.common.cache.Cache;
      import com.google.common.cache.CacheBuilder;
      import com.google.common.collect.Lists;
    @@ -436,26 +440,34 @@ public class OrcInputFormat implements InputFormat<NullWritable, OrcStruct>,
        static final class SplitInfo extends ACIDSplitStrategy {
          private final Context context;
          private final FileSystem fs;
    - private final FileStatus file;
    + private final HdfsFileStatusWithId fileWithId;
          private final FileInfo fileInfo;
          private final boolean isOriginal;
          private final List<DeltaMetaData> deltas;
          private final boolean hasBase;

          SplitInfo(Context context, FileSystem fs,
    - FileStatus file, FileInfo fileInfo,
    + HdfsFileStatusWithId fileWithId, FileInfo fileInfo,
              boolean isOriginal,
              List<DeltaMetaData> deltas,
              boolean hasBase, Path dir, boolean[] covered) throws IOException {
            super(dir, context.numBuckets, deltas, covered);
            this.context = context;
            this.fs = fs;
    - this.file = file;
    + this.fileWithId = fileWithId;
            this.fileInfo = fileInfo;
            this.isOriginal = isOriginal;
            this.deltas = deltas;
            this.hasBase = hasBase;
          }
    +
    + @VisibleForTesting
    + public SplitInfo(Context context, FileSystem fs, FileStatus fileStatus, FileInfo fileInfo,
    + boolean isOriginal, ArrayList<DeltaMetaData> deltas, boolean hasBase, Path dir,
    + boolean[] covered) throws IOException {
    + this(context, fs, AcidUtils.createOriginalObj(null, fileStatus),
    + fileInfo, isOriginal, deltas, hasBase, dir, covered);
    + }
        }

        /**
    @@ -465,14 +477,15 @@ public class OrcInputFormat implements InputFormat<NullWritable, OrcStruct>,
        static final class ETLSplitStrategy implements SplitStrategy<SplitInfo> {
          Context context;
          FileSystem fs;
    - List<FileStatus> files;
    + List<HdfsFileStatusWithId> files;
          boolean isOriginal;
          List<DeltaMetaData> deltas;
          Path dir;
          boolean[] covered;

    - public ETLSplitStrategy(Context context, FileSystem fs, Path dir, List<FileStatus> children,
    - boolean isOriginal, List<DeltaMetaData> deltas, boolean[] covered) {
    + public ETLSplitStrategy(Context context, FileSystem fs, Path dir,
    + List<HdfsFileStatusWithId> children, boolean isOriginal, List<DeltaMetaData> deltas,
    + boolean[] covered) {
            this.context = context;
            this.dir = dir;
            this.fs = fs;
    @@ -516,14 +529,15 @@ public class OrcInputFormat implements InputFormat<NullWritable, OrcStruct>,
          @Override
          public List<SplitInfo> getSplits() throws IOException {
            List<SplitInfo> result = Lists.newArrayList();
    - for (FileStatus file : files) {
    + for (HdfsFileStatusWithId file : files) {
              FileInfo info = null;
              if (context.cacheStripeDetails) {
    - info = verifyCachedFileInfo(file);
    + info = verifyCachedFileInfo(file.getFileStatus());
              }
              // ignore files of 0 length
    - if (file.getLen() > 0) {
    - result.add(new SplitInfo(context, fs, file, info, isOriginal, deltas, true, dir, covered));
    + if (file.getFileStatus().getLen() > 0) {
    + result.add(new SplitInfo(
    + context, fs, file, info, isOriginal, deltas, true, dir, covered));
              }
            }
            return result;
    @@ -540,7 +554,7 @@ public class OrcInputFormat implements InputFormat<NullWritable, OrcStruct>,
         * as opposed to query execution (split generation does not read or cache file footers).
         */
        static final class BISplitStrategy extends ACIDSplitStrategy {
    - List<FileStatus> fileStatuses;
    + List<HdfsFileStatusWithId> fileStatuses;
          boolean isOriginal;
          List<DeltaMetaData> deltas;
          FileSystem fs;
    @@ -548,7 +562,7 @@ public class OrcInputFormat implements InputFormat<NullWritable, OrcStruct>,
          Path dir;

          public BISplitStrategy(Context context, FileSystem fs,
    - Path dir, List<FileStatus> fileStatuses, boolean isOriginal,
    + Path dir, List<HdfsFileStatusWithId> fileStatuses, boolean isOriginal,
              List<DeltaMetaData> deltas, boolean[] covered) {
            super(dir, context.numBuckets, deltas, covered);
            this.context = context;
    @@ -562,11 +576,12 @@ public class OrcInputFormat implements InputFormat<NullWritable, OrcStruct>,
          @Override
          public List<OrcSplit> getSplits() throws IOException {
            List<OrcSplit> splits = Lists.newArrayList();
    - for (FileStatus fileStatus : fileStatuses) {
    + for (HdfsFileStatusWithId file : fileStatuses) {
    + FileStatus fileStatus = file.getFileStatus();
              String[] hosts = SHIMS.getLocationsWithOffset(fs, fileStatus).firstEntry().getValue()
                  .getHosts();
    - OrcSplit orcSplit = new OrcSplit(fileStatus.getPath(), 0, fileStatus.getLen(), hosts,
    - null, isOriginal, true, deltas, -1);
    + OrcSplit orcSplit = new OrcSplit(fileStatus.getPath(), file.getFileId(), 0,
    + fileStatus.getLen(), hosts, null, isOriginal, true, deltas, -1);
              splits.add(orcSplit);
            }

    @@ -606,7 +621,7 @@ public class OrcInputFormat implements InputFormat<NullWritable, OrcStruct>,
            if (!deltas.isEmpty()) {
              for (int b = 0; b < numBuckets; ++b) {
                if (!covered[b]) {
    - splits.add(new OrcSplit(dir, b, 0, new String[0], null, false, false, deltas, -1));
    + splits.add(new OrcSplit(dir, null, b, 0, new String[0], null, false, false, deltas, -1));
                }
              }
            }
    @@ -627,21 +642,23 @@ public class OrcInputFormat implements InputFormat<NullWritable, OrcStruct>,
          private final Context context;
          private final FileSystem fs;
          private final Path dir;
    + private final boolean useFileIds;

    - FileGenerator(Context context, FileSystem fs, Path dir) {
    + FileGenerator(Context context, FileSystem fs, Path dir, boolean useFileIds) {
            this.context = context;
            this.fs = fs;
            this.dir = dir;
    + this.useFileIds = useFileIds;
          }

          @Override
          public SplitStrategy call() throws IOException {
            final SplitStrategy splitStrategy;
            AcidUtils.Directory dirInfo = AcidUtils.getAcidState(dir,
    - context.conf, context.transactionList);
    + context.conf, context.transactionList, useFileIds);
            List<DeltaMetaData> deltas = AcidUtils.serializeDeltas(dirInfo.getCurrentDirectories());
            Path base = dirInfo.getBaseDirectory();
    - List<FileStatus> original = dirInfo.getOriginalFiles();
    + List<HdfsFileStatusWithId> original = dirInfo.getOriginalFiles();
            boolean[] covered = new boolean[context.numBuckets];
            boolean isOriginal = base == null;

    @@ -649,17 +666,16 @@ public class OrcInputFormat implements InputFormat<NullWritable, OrcStruct>,
            if (base != null || !original.isEmpty()) {

              // find the base files (original or new style)
    - List<FileStatus> children = original;
    + List<HdfsFileStatusWithId> children = original;
              if (base != null) {
    - children = SHIMS.listLocatedStatus(fs, base,
    - AcidUtils.hiddenFileFilter);
    + children = findBaseFiles(base, useFileIds);
              }

              long totalFileSize = 0;
    - for (FileStatus child : children) {
    - totalFileSize += child.getLen();
    + for (HdfsFileStatusWithId child : children) {
    + totalFileSize += child.getFileStatus().getLen();
                AcidOutputFormat.Options opts = AcidUtils.parseBaseBucketFilename
    - (child.getPath(), context.conf);
    + (child.getFileStatus().getPath(), context.conf);
                int b = opts.getBucket();
                // If the bucket is in the valid range, mark it as covered.
                // I wish Hive actually enforced bucketing all of the time.
    @@ -700,6 +716,24 @@ public class OrcInputFormat implements InputFormat<NullWritable, OrcStruct>,

            return splitStrategy;
          }
    +
    + private List<HdfsFileStatusWithId> findBaseFiles(
    + Path base, boolean useFileIds) throws IOException {
    + if (useFileIds) {
    + try {
    + return SHIMS.listLocatedHdfsStatus(fs, base, AcidUtils.hiddenFileFilter);
    + } catch (Throwable t) {
    + LOG.error("Failed to get files with ID; using regular API", t);
    + }
    + }
    + // Fall back to regular API and create states without ID.
    + List<FileStatus> children = SHIMS.listLocatedStatus(fs, base, AcidUtils.hiddenFileFilter);
    + List<HdfsFileStatusWithId> result = new ArrayList<>(children.size());
    + for (FileStatus child : children) {
    + result.add(AcidUtils.createOriginalObj(null, child));
    + }
    + return result;
    + }
        }

        /**
    @@ -709,6 +743,7 @@ public class OrcInputFormat implements InputFormat<NullWritable, OrcStruct>,
        static final class SplitGenerator implements Callable<List<OrcSplit>> {
          private final Context context;
          private final FileSystem fs;
    + private final HdfsFileStatusWithId fileWithId;
          private final FileStatus file;
          private final long blockSize;
          private final TreeMap<Long, BlockLocation> locations;
    @@ -728,8 +763,9 @@ public class OrcInputFormat implements InputFormat<NullWritable, OrcStruct>,
          public SplitGenerator(SplitInfo splitInfo) throws IOException {
            this.context = splitInfo.context;
            this.fs = splitInfo.fs;
    - this.file = splitInfo.file;
    - this.blockSize = file.getBlockSize();
    + this.fileWithId = splitInfo.fileWithId;
    + this.file = this.fileWithId.getFileStatus();
    + this.blockSize = this.file.getBlockSize();
            this.fileInfo = splitInfo.fileInfo;
            locations = SHIMS.getLocationsWithOffset(fs, file);
            this.isOriginal = splitInfo.isOriginal;
    @@ -837,8 +873,8 @@ public class OrcInputFormat implements InputFormat<NullWritable, OrcStruct>,
            final double splitRatio = (double) length / (double) fileLen;
            final long scaledProjSize = projColsUncompressedSize > 0 ?
                (long) (splitRatio * projColsUncompressedSize) : fileLen;
    - return new OrcSplit(file.getPath(), offset, length, hosts, fileMetaInfo,
    - isOriginal, hasBase, deltas, scaledProjSize);
    + return new OrcSplit(file.getPath(), fileWithId.getFileId(), offset, length, hosts,
    + fileMetaInfo, isOriginal, hasBase, deltas, scaledProjSize);
          }

          /**
    @@ -1020,9 +1056,10 @@ public class OrcInputFormat implements InputFormat<NullWritable, OrcStruct>,
          List<Future<?>> splitFutures = Lists.newArrayList();

          // multi-threaded file statuses and split strategy
    + boolean useFileIds = HiveConf.getBoolVar(conf, ConfVars.HIVE_ORC_INCLUDE_FILE_ID_IN_SPLITS);
          for (Path dir : getInputPaths(conf)) {
            FileSystem fs = dir.getFileSystem(conf);
    - FileGenerator fileGenerator = new FileGenerator(context, fs, dir);
    + FileGenerator fileGenerator = new FileGenerator(context, fs, dir, useFileIds);
            pathFutures.add(context.threadPool.submit(fileGenerator));
          }


    http://git-wip-us.apache.org/repos/asf/hive/blob/3b6825b5/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcSplit.java
    ----------------------------------------------------------------------
    diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcSplit.java b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcSplit.java
    index 8cf4cc0..cc03df7 100644
    --- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcSplit.java
    +++ b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcSplit.java
    @@ -25,6 +25,8 @@ import java.nio.ByteBuffer;
      import java.util.ArrayList;
      import java.util.List;

    +import org.apache.commons.logging.Log;
    +import org.apache.commons.logging.LogFactory;
      import org.apache.hadoop.fs.Path;
      import org.apache.hadoop.hive.ql.io.AcidInputFormat;
      import org.apache.hadoop.hive.ql.io.AcidUtils;
    @@ -39,6 +41,8 @@ import org.apache.hadoop.mapred.FileSplit;
       *
       */
      public class OrcSplit extends FileSplit {
    + private static final Log LOG = LogFactory.getLog(OrcSplit.class);
    +
        private ReaderImpl.FileMetaInfo fileMetaInfo;
        private boolean hasFooter;
        private boolean isOriginal;
    @@ -46,7 +50,9 @@ public class OrcSplit extends FileSplit {
        private final List<AcidInputFormat.DeltaMetaData> deltas = new ArrayList<>();
        private OrcFile.WriterVersion writerVersion;
        private long projColsUncompressedSize;
    + private transient Long fileId;

    + static final int HAS_FILEID_FLAG = 8;
        static final int BASE_FLAG = 4;
        static final int ORIGINAL_FLAG = 2;
        static final int FOOTER_FLAG = 1;
    @@ -58,10 +64,13 @@ public class OrcSplit extends FileSplit {
          super(null, 0, 0, (String[]) null);
        }

    - public OrcSplit(Path path, long offset, long length, String[] hosts,
    + public OrcSplit(Path path, Long fileId, long offset, long length, String[] hosts,
            ReaderImpl.FileMetaInfo fileMetaInfo, boolean isOriginal, boolean hasBase,
            List<AcidInputFormat.DeltaMetaData> deltas, long projectedDataSize) {
          super(path, offset, length, hosts);
    + // We could avoid serializing file ID and just replace the path with inode-based path.
    + // However, that breaks bunch of stuff because Hive later looks up things by split path.
    + this.fileId = fileId;
          this.fileMetaInfo = fileMetaInfo;
          hasFooter = this.fileMetaInfo != null;
          this.isOriginal = isOriginal;
    @@ -77,7 +86,8 @@ public class OrcSplit extends FileSplit {

          int flags = (hasBase ? BASE_FLAG : 0) |
              (isOriginal ? ORIGINAL_FLAG : 0) |
    - (hasFooter ? FOOTER_FLAG : 0);
    + (hasFooter ? FOOTER_FLAG : 0) |
    + (fileId != null ? HAS_FILEID_FLAG : 0);
          out.writeByte(flags);
          out.writeInt(deltas.size());
          for(AcidInputFormat.DeltaMetaData delta: deltas) {
    @@ -99,6 +109,9 @@ public class OrcSplit extends FileSplit {
                footerBuff.limit() - footerBuff.position());
            WritableUtils.writeVInt(out, fileMetaInfo.writerVersion.getId());
          }
    + if (fileId != null) {
    + out.writeLong(fileId.longValue());
    + }
        }

        @Override
    @@ -110,6 +123,7 @@ public class OrcSplit extends FileSplit {
          hasFooter = (FOOTER_FLAG & flags) != 0;
          isOriginal = (ORIGINAL_FLAG & flags) != 0;
          hasBase = (BASE_FLAG & flags) != 0;
    + boolean hasFileId = (HAS_FILEID_FLAG & flags) != 0;

          deltas.clear();
          int numDeltas = in.readInt();
    @@ -134,6 +148,9 @@ public class OrcSplit extends FileSplit {
            fileMetaInfo = new ReaderImpl.FileMetaInfo(compressionType, bufferSize,
                metadataSize, footerBuff, writerVersion);
          }
    + if (hasFileId) {
    + fileId = in.readLong();
    + }
        }

        ReaderImpl.FileMetaInfo getFileMetaInfo(){
    @@ -159,4 +176,8 @@ public class OrcSplit extends FileSplit {
        public long getProjectedColumnsUncompressedSize() {
          return projColsUncompressedSize;
        }
    +
    + public Long getFileId() {
    + return fileId;
    + }
      }

    http://git-wip-us.apache.org/repos/asf/hive/blob/3b6825b5/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorMR.java
    ----------------------------------------------------------------------
    diff --git a/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorMR.java b/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorMR.java
    index 8e431b2..02fa725 100644
    --- a/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorMR.java
    +++ b/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorMR.java
    @@ -39,6 +39,7 @@ import org.apache.hadoop.hive.ql.io.AcidUtils;
      import org.apache.hadoop.hive.ql.io.RecordIdentifier;
      import org.apache.hadoop.hive.serde.serdeConstants;
      import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
    +import org.apache.hadoop.hive.shims.HadoopShims.HdfsFileStatusWithId;
      import org.apache.hadoop.hive.shims.ShimLoader;
      import org.apache.hadoop.io.NullWritable;
      import org.apache.hadoop.io.Writable;
    @@ -133,7 +134,8 @@ public class CompactorMR {
          // and discovering that in getSplits is too late as we then have no way to pass it to our
          // mapper.

    - AcidUtils.Directory dir = AcidUtils.getAcidState(new Path(sd.getLocation()), conf, txns);
    + AcidUtils.Directory dir = AcidUtils.getAcidState(
    + new Path(sd.getLocation()), conf, txns, false);
          StringableList dirsToSearch = new StringableList();
          Path baseDir = null;
          if (isMajor) {
    @@ -141,12 +143,13 @@ public class CompactorMR {
            // partition is just now being converted to ACID.
            baseDir = dir.getBaseDirectory();
            if (baseDir == null) {
    - List<FileStatus> originalFiles = dir.getOriginalFiles();
    + List<HdfsFileStatusWithId> originalFiles = dir.getOriginalFiles();
              if (!(originalFiles == null) && !(originalFiles.size() == 0)) {
                // There are original format files
    - for (FileStatus stat : originalFiles) {
    - dirsToSearch.add(stat.getPath());
    - LOG.debug("Adding original file " + stat.getPath().toString() + " to dirs to search");
    + for (HdfsFileStatusWithId stat : originalFiles) {
    + Path path = stat.getFileStatus().getPath();
    + dirsToSearch.add(path);
    + LOG.debug("Adding original file " + path + " to dirs to search");
                }
                // Set base to the location so that the input format reads the original files.
                baseDir = new Path(sd.getLocation());

    http://git-wip-us.apache.org/repos/asf/hive/blob/3b6825b5/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Initiator.java
    ----------------------------------------------------------------------
    diff --git a/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Initiator.java b/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Initiator.java
    index 73715c6..9bf725d 100644
    --- a/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Initiator.java
    +++ b/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Initiator.java
    @@ -40,6 +40,7 @@ import org.apache.hadoop.hive.metastore.txn.CompactionInfo;
      import org.apache.hadoop.hive.metastore.txn.CompactionTxnHandler;
      import org.apache.hadoop.hive.metastore.txn.TxnHandler;
      import org.apache.hadoop.hive.ql.io.AcidUtils;
    +import org.apache.hadoop.hive.shims.HadoopShims.HdfsFileStatusWithId;
      import org.apache.hadoop.security.UserGroupInformation;
      import org.apache.hadoop.util.StringUtils;

    @@ -223,7 +224,7 @@ public class Initiator extends CompactorThread {
          boolean noBase = false;
          Path location = new Path(sd.getLocation());
          FileSystem fs = location.getFileSystem(conf);
    - AcidUtils.Directory dir = AcidUtils.getAcidState(location, conf, txns);
    + AcidUtils.Directory dir = AcidUtils.getAcidState(location, conf, txns, false);
          Path base = dir.getBaseDirectory();
          long baseSize = 0;
          FileStatus stat = null;
    @@ -236,9 +237,9 @@ public class Initiator extends CompactorThread {
            baseSize = sumDirSize(fs, base);
          }

    - List<FileStatus> originals = dir.getOriginalFiles();
    - for (FileStatus origStat : originals) {
    - baseSize += origStat.getLen();
    + List<HdfsFileStatusWithId> originals = dir.getOriginalFiles();
    + for (HdfsFileStatusWithId origStat : originals) {
    + baseSize += origStat.getFileStatus().getLen();
          }

          long deltaSize = 0;

    http://git-wip-us.apache.org/repos/asf/hive/blob/3b6825b5/ql/src/test/org/apache/hadoop/hive/ql/io/TestAcidUtils.java
    ----------------------------------------------------------------------
    diff --git a/ql/src/test/org/apache/hadoop/hive/ql/io/TestAcidUtils.java b/ql/src/test/org/apache/hadoop/hive/ql/io/TestAcidUtils.java
    index f8ded12..b6ba862 100644
    --- a/ql/src/test/org/apache/hadoop/hive/ql/io/TestAcidUtils.java
    +++ b/ql/src/test/org/apache/hadoop/hive/ql/io/TestAcidUtils.java
    @@ -26,6 +26,7 @@ import org.apache.hadoop.hive.ql.io.orc.TestInputOutputFormat;
      import org.apache.hadoop.hive.ql.io.orc.TestInputOutputFormat.MockFile;
      import org.apache.hadoop.hive.ql.io.orc.TestInputOutputFormat.MockFileSystem;
      import org.apache.hadoop.hive.ql.io.orc.TestInputOutputFormat.MockPath;
    +import org.apache.hadoop.hive.shims.HadoopShims.HdfsFileStatusWithId;
      import org.junit.Test;

      import java.util.List;
    @@ -102,13 +103,14 @@ public class TestAcidUtils {
          assertEquals(null, dir.getBaseDirectory());
          assertEquals(0, dir.getCurrentDirectories().size());
          assertEquals(0, dir.getObsolete().size());
    - List<FileStatus> result = dir.getOriginalFiles();
    + List<HdfsFileStatusWithId> result = dir.getOriginalFiles();
          assertEquals(5, result.size());
    - assertEquals("mock:/tbl/part1/000000_0", result.get(0).getPath().toString());
    - assertEquals("mock:/tbl/part1/000001_1", result.get(1).getPath().toString());
    - assertEquals("mock:/tbl/part1/000002_0", result.get(2).getPath().toString());
    - assertEquals("mock:/tbl/part1/random", result.get(3).getPath().toString());
    - assertEquals("mock:/tbl/part1/subdir/000000_0", result.get(4).getPath().toString());
    + assertEquals("mock:/tbl/part1/000000_0", result.get(0).getFileStatus().getPath().toString());
    + assertEquals("mock:/tbl/part1/000001_1", result.get(1).getFileStatus().getPath().toString());
    + assertEquals("mock:/tbl/part1/000002_0", result.get(2).getFileStatus().getPath().toString());
    + assertEquals("mock:/tbl/part1/random", result.get(3).getFileStatus().getPath().toString());
    + assertEquals("mock:/tbl/part1/subdir/000000_0",
    + result.get(4).getFileStatus().getPath().toString());
        }

        @Test
    @@ -136,13 +138,14 @@ public class TestAcidUtils {
              obsolete.get(0).getPath().toString());
          assertEquals("mock:/tbl/part1/delta_029_029",
              obsolete.get(1).getPath().toString());
    - List<FileStatus> result = dir.getOriginalFiles();
    + List<HdfsFileStatusWithId> result = dir.getOriginalFiles();
          assertEquals(5, result.size());
    - assertEquals("mock:/tbl/part1/000000_0", result.get(0).getPath().toString());
    - assertEquals("mock:/tbl/part1/000001_1", result.get(1).getPath().toString());
    - assertEquals("mock:/tbl/part1/000002_0", result.get(2).getPath().toString());
    - assertEquals("mock:/tbl/part1/random", result.get(3).getPath().toString());
    - assertEquals("mock:/tbl/part1/subdir/000000_0", result.get(4).getPath().toString());
    + assertEquals("mock:/tbl/part1/000000_0", result.get(0).getFileStatus().getPath().toString());
    + assertEquals("mock:/tbl/part1/000001_1", result.get(1).getFileStatus().getPath().toString());
    + assertEquals("mock:/tbl/part1/000002_0", result.get(2).getFileStatus().getPath().toString());
    + assertEquals("mock:/tbl/part1/random", result.get(3).getFileStatus().getPath().toString());
    + assertEquals("mock:/tbl/part1/subdir/000000_0",
    + result.get(4).getFileStatus().getPath().toString());
          List<AcidUtils.ParsedDelta> deltas = dir.getCurrentDirectories();
          assertEquals(2, deltas.size());
          AcidUtils.ParsedDelta delt = deltas.get(0);

    http://git-wip-us.apache.org/repos/asf/hive/blob/3b6825b5/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestInputOutputFormat.java
    ----------------------------------------------------------------------
    diff --git a/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestInputOutputFormat.java b/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestInputOutputFormat.java
    index 0c12c89..547e799 100644
    --- a/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestInputOutputFormat.java
    +++ b/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestInputOutputFormat.java
    @@ -483,7 +483,7 @@ public class TestInputOutputFormat {
                final OrcInputFormat.Context context = new OrcInputFormat.Context(
                    conf, n);
                OrcInputFormat.FileGenerator gen = new OrcInputFormat.FileGenerator(
    - context, fs, new MockPath(fs, "mock:/a/b"));
    + context, fs, new MockPath(fs, "mock:/a/b"), false);
                final SplitStrategy splitStrategy = gen.call();
                assertTrue(
                    String.format(
    @@ -507,7 +507,7 @@ public class TestInputOutputFormat {
              new MockFile("mock:/a/b/part-04", 1000, new byte[0]));
          OrcInputFormat.FileGenerator gen =
            new OrcInputFormat.FileGenerator(context, fs,
    - new MockPath(fs, "mock:/a/b"));
    + new MockPath(fs, "mock:/a/b"), false);
          SplitStrategy splitStrategy = gen.call();
          assertEquals(true, splitStrategy instanceof OrcInputFormat.BISplitStrategy);

    @@ -520,7 +520,7 @@ public class TestInputOutputFormat {
              new MockFile("mock:/a/b/.part-03", 1000, new byte[1000]),
              new MockFile("mock:/a/b/part-04", 1000, new byte[1000]));
          gen = new OrcInputFormat.FileGenerator(context, fs,
    - new MockPath(fs, "mock:/a/b"));
    + new MockPath(fs, "mock:/a/b"), false);
          splitStrategy = gen.call();
          assertEquals(true, splitStrategy instanceof OrcInputFormat.ETLSplitStrategy);


    http://git-wip-us.apache.org/repos/asf/hive/blob/3b6825b5/shims/0.20S/src/main/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java
    ----------------------------------------------------------------------
    diff --git a/shims/0.20S/src/main/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java b/shims/0.20S/src/main/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java
    index ffffcb7..a56309f 100644
    --- a/shims/0.20S/src/main/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java
    +++ b/shims/0.20S/src/main/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java
    @@ -722,4 +722,15 @@ public class Hadoop20SShims extends HadoopShimsSecure {
          Token<?> fsToken = fs.getDelegationToken(uname);
          cred.addToken(fsToken.getService(), fsToken);
        }
    +
    + @Override
    + public List<HdfsFileStatusWithId> listLocatedHdfsStatus(
    + FileSystem fs, Path path, PathFilter filter) throws IOException {
    + throw new UnsupportedOperationException("Not supported on old version");
    + }
    +
    + @Override
    + public long getFileId(FileSystem fs, String path) throws IOException {
    + throw new UnsupportedOperationException("Not supported on old version");
    + }
      }

    http://git-wip-us.apache.org/repos/asf/hive/blob/3b6825b5/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java
    ----------------------------------------------------------------------
    diff --git a/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java b/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java
    index 9eae0ac..e5be8d6 100644
    --- a/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java
    +++ b/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java
    @@ -61,10 +61,13 @@ import org.apache.hadoop.fs.permission.AclEntryType;
      import org.apache.hadoop.fs.permission.AclStatus;
      import org.apache.hadoop.fs.permission.FsAction;
      import org.apache.hadoop.fs.permission.FsPermission;
    +import org.apache.hadoop.hdfs.DFSClient;
      import org.apache.hadoop.hdfs.DistributedFileSystem;
      import org.apache.hadoop.hdfs.MiniDFSCluster;
      import org.apache.hadoop.hdfs.protocol.BlockStoragePolicy;
    +import org.apache.hadoop.hdfs.protocol.DirectoryListing;
      import org.apache.hadoop.hdfs.protocol.HdfsConstants;
    +import org.apache.hadoop.hdfs.protocol.HdfsLocatedFileStatus;
      import org.apache.hadoop.hdfs.client.HdfsAdmin;
      import org.apache.hadoop.hdfs.protocol.EncryptionZone;
      import org.apache.hadoop.io.LongWritable;
    @@ -662,6 +665,64 @@ public class Hadoop23Shims extends HadoopShimsSecure {
          return result;
        }

    + private static final class HdfsFileStatusWithIdImpl implements HdfsFileStatusWithId {
    + private final LocatedFileStatus lfs;
    + private final long fileId;
    +
    + public HdfsFileStatusWithIdImpl(LocatedFileStatus lfs, long fileId) {
    + this.lfs = lfs;
    + this.fileId = fileId;
    + }
    +
    + @Override
    + public FileStatus getFileStatus() {
    + return lfs;
    + }
    +
    + @Override
    + public Long getFileId() {
    + return fileId;
    + }
    + }
    +
    + @Override
    + public List<HdfsFileStatusWithId> listLocatedHdfsStatus(
    + FileSystem fs, Path p, PathFilter filter) throws IOException {
    + DistributedFileSystem dfs = ensureDfs(fs);
    + DFSClient dfsc = dfs.getClient();
    + final String src = p.toUri().getPath();
    + DirectoryListing current = dfsc.listPaths(src,
    + org.apache.hadoop.hdfs.protocol.HdfsFileStatus.EMPTY_NAME, true);
    + if (current == null) { // the directory does not exist
    + throw new FileNotFoundException("File " + p + " does not exist.");
    + }
    + final URI fsUri = fs.getUri();
    + List<HdfsFileStatusWithId> result = new ArrayList<HdfsFileStatusWithId>(
    + current.getPartialListing().length);
    + while (current != null) {
    + org.apache.hadoop.hdfs.protocol.HdfsFileStatus[] hfss = current.getPartialListing();
    + for (int i = 0; i < hfss.length; ++i) {
    + HdfsLocatedFileStatus next = (HdfsLocatedFileStatus)(hfss[i]);
    + if (filter != null) {
    + Path filterPath = next.getFullPath(p).makeQualified(fsUri, null);
    + if (!filter.accept(filterPath)) continue;
    + }
    + LocatedFileStatus lfs = next.makeQualifiedLocated(fsUri, p);
    + result.add(new HdfsFileStatusWithIdImpl(lfs, next.getFileId()));
    + }
    + current = current.hasMore() ? dfsc.listPaths(src, current.getLastName(), true) : null;
    + }
    + return result;
    + }
    +
    + private DistributedFileSystem ensureDfs(FileSystem fs) {
    + if (!(fs instanceof DistributedFileSystem)) {
    + throw new UnsupportedOperationException("Only supported for DFS; got " + fs.getClass());
    + }
    + DistributedFileSystem dfs = (DistributedFileSystem)fs;
    + return dfs;
    + }
    +
        @Override
        public BlockLocation[] getLocations(FileSystem fs,
                                            FileStatus status) throws IOException {
    @@ -1352,4 +1413,9 @@ public class Hadoop23Shims extends HadoopShimsSecure {
          // Use method addDelegationTokens instead of getDelegationToken to get all the tokens including KMS.
          fs.addDelegationTokens(uname, cred);
        }
    +
    + @Override
    + public long getFileId(FileSystem fs, String path) throws IOException {
    + return ensureDfs(fs).getClient().getFileInfo(path).getFileId();
    + }
      }

    http://git-wip-us.apache.org/repos/asf/hive/blob/3b6825b5/shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShims.java
    ----------------------------------------------------------------------
    diff --git a/shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShims.java b/shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShims.java
    index 74785e5..2b6f322 100644
    --- a/shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShims.java
    +++ b/shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShims.java
    @@ -256,6 +256,10 @@ public interface HadoopShims {
        List<FileStatus> listLocatedStatus(FileSystem fs, Path path,
                                           PathFilter filter) throws IOException;

    +
    + List<HdfsFileStatusWithId> listLocatedHdfsStatus(
    + FileSystem fs, Path path, PathFilter filter) throws IOException;
    +
        /**
         * For file status returned by listLocatedStatus, convert them into a list
         * of block locations.
    @@ -316,6 +320,11 @@ public interface HadoopShims {
          public void debugLog();
        }

    + public interface HdfsFileStatusWithId {
    + public FileStatus getFileStatus();
    + public Long getFileId();
    + }
    +
        public HCatHadoopShims getHCatShim();
        public interface HCatHadoopShims {

    @@ -731,4 +740,10 @@ public interface HadoopShims {
         * @throws IOException If an error occurred on adding the token.
         */
        public void addDelegationTokens(FileSystem fs, Credentials cred, String uname) throws IOException;
    +
    + /**
    + * Gets file ID. Only supported on hadoop-2.
    + * @return inode ID of the file.
    + */
    + long getFileId(FileSystem fs, String path) throws IOException;
      }
  • Sershe at Aug 25, 2015 at 9:28 pm
    HIVE-11567: Some trace logs seeped through with new log4j2 changes (Prasanth Jayachandran reviewed by Sergey Shelukhin)


    Project: http://git-wip-us.apache.org/repos/asf/hive/repo
    Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/cf0481fc
    Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/cf0481fc
    Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/cf0481fc

    Branch: refs/heads/llap
    Commit: cf0481fcf26087dc2cd2de8b10bc2b13befa96ac
    Parents: 0fab86c
    Author: Prasanth Jayachandran <j.prasanth.j@gmail.com>
    Authored: Fri Aug 14 14:04:10 2015 -0700
    Committer: Prasanth Jayachandran <j.prasanth.j@gmail.com>
    Committed: Fri Aug 14 14:04:10 2015 -0700

    ----------------------------------------------------------------------
      beeline/src/main/resources/beeline-log4j2.xml | 2 +-
      common/src/test/resources/hive-exec-log4j2-test.xml | 2 +-
      common/src/test/resources/hive-log4j2-test.xml | 2 +-
      data/conf/hive-log4j2.xml | 2 +-
      hcatalog/webhcat/svr/src/main/config/webhcat-log4j2.xml | 2 +-
      ql/src/main/resources/hive-exec-log4j2.xml | 2 +-
      testutils/ptest2/src/main/resources/log4j2.xml | 2 +-
      7 files changed, 7 insertions(+), 7 deletions(-)
    ----------------------------------------------------------------------


    http://git-wip-us.apache.org/repos/asf/hive/blob/cf0481fc/beeline/src/main/resources/beeline-log4j2.xml
    ----------------------------------------------------------------------
    diff --git a/beeline/src/main/resources/beeline-log4j2.xml b/beeline/src/main/resources/beeline-log4j2.xml
    index 5f09741..2349c5a 100644
    --- a/beeline/src/main/resources/beeline-log4j2.xml
    +++ b/beeline/src/main/resources/beeline-log4j2.xml
    @@ -20,7 +20,7 @@
       packages="org.apache.hadoop.hive.ql.log">

        <Properties>
    - <Property name="hive.log.threshold">ALL</Property>
    + <Property name="hive.log.threshold">DEBUG</Property>
          <Property name="hive.log.level">WARN</Property>
          <Property name="hive.root.logger">console</Property>
        </Properties>

    http://git-wip-us.apache.org/repos/asf/hive/blob/cf0481fc/common/src/test/resources/hive-exec-log4j2-test.xml
    ----------------------------------------------------------------------
    diff --git a/common/src/test/resources/hive-exec-log4j2-test.xml b/common/src/test/resources/hive-exec-log4j2-test.xml
    index b5f2cb4..1d91b26 100644
    --- a/common/src/test/resources/hive-exec-log4j2-test.xml
    +++ b/common/src/test/resources/hive-exec-log4j2-test.xml
    @@ -20,7 +20,7 @@
       packages="org.apache.hadoop.hive.ql.log">

        <Properties>
    - <Property name="hive.log.threshold">ALL</Property>
    + <Property name="hive.log.threshold">DEBUG</Property>
          <Property name="hive.log.level">INFO</Property>
          <Property name="hive.root.logger">FA</Property>
          <Property name="hive.log.dir">${sys:test.tmp.dir}/${sys:user.name}-TestHiveLogging</Property>

    http://git-wip-us.apache.org/repos/asf/hive/blob/cf0481fc/common/src/test/resources/hive-log4j2-test.xml
    ----------------------------------------------------------------------
    diff --git a/common/src/test/resources/hive-log4j2-test.xml b/common/src/test/resources/hive-log4j2-test.xml
    index 63b46c8..98ca6f8 100644
    --- a/common/src/test/resources/hive-log4j2-test.xml
    +++ b/common/src/test/resources/hive-log4j2-test.xml
    @@ -20,7 +20,7 @@
       packages="org.apache.hadoop.hive.ql.log">

        <Properties>
    - <Property name="hive.log.threshold">ALL</Property>
    + <Property name="hive.log.threshold">DEBUG</Property>
          <Property name="hive.log.level">WARN</Property>
          <Property name="hive.root.logger">DRFA</Property>
          <Property name="hive.log.dir">${sys:test.tmp.dir}/${sys:user.name}-TestHiveLogging</Property>

    http://git-wip-us.apache.org/repos/asf/hive/blob/cf0481fc/data/conf/hive-log4j2.xml
    ----------------------------------------------------------------------
    diff --git a/data/conf/hive-log4j2.xml b/data/conf/hive-log4j2.xml
    index ff8e4d3..51173a0 100644
    --- a/data/conf/hive-log4j2.xml
    +++ b/data/conf/hive-log4j2.xml
    @@ -20,7 +20,7 @@
       packages="org.apache.hadoop.hive.ql.log">

        <Properties>
    - <Property name="hive.log.threshold">ALL</Property>
    + <Property name="hive.log.threshold">DEBUG</Property>
          <Property name="hive.log.level">DEBUG</Property>
          <Property name="hive.root.logger">DRFA</Property>
          <Property name="hive.log.dir">${sys:test.tmp.dir}/log</Property>

    http://git-wip-us.apache.org/repos/asf/hive/blob/cf0481fc/hcatalog/webhcat/svr/src/main/config/webhcat-log4j2.xml
    ----------------------------------------------------------------------
    diff --git a/hcatalog/webhcat/svr/src/main/config/webhcat-log4j2.xml b/hcatalog/webhcat/svr/src/main/config/webhcat-log4j2.xml
    index 40da974..96f0974 100644
    --- a/hcatalog/webhcat/svr/src/main/config/webhcat-log4j2.xml
    +++ b/hcatalog/webhcat/svr/src/main/config/webhcat-log4j2.xml
    @@ -20,7 +20,7 @@
       packages="org.apache.hadoop.hive.ql.log">

        <Properties>
    - <Property name="webhcat.log.threshold">ALL</Property>
    + <Property name="webhcat.log.threshold">DEBUG</Property>
          <Property name="webhcat.log.level">INFO</Property>
          <Property name="webhcat.root.logger">standard</Property>
          <Property name="webhcat.log.dir">.</Property>

    http://git-wip-us.apache.org/repos/asf/hive/blob/cf0481fc/ql/src/main/resources/hive-exec-log4j2.xml
    ----------------------------------------------------------------------
    diff --git a/ql/src/main/resources/hive-exec-log4j2.xml b/ql/src/main/resources/hive-exec-log4j2.xml
    index c93437c..8b520a2 100644
    --- a/ql/src/main/resources/hive-exec-log4j2.xml
    +++ b/ql/src/main/resources/hive-exec-log4j2.xml
    @@ -20,7 +20,7 @@
       packages="org.apache.hadoop.hive.ql.log">

        <Properties>
    - <Property name="hive.log.threshold">ALL</Property>
    + <Property name="hive.log.threshold">DEBUG</Property>
          <Property name="hive.log.level">INFO</Property>
          <Property name="hive.root.logger">FA</Property>
          <Property name="hive.log.dir">${sys:java.io.tmpdir}/${sys:user.name}</Property>

    http://git-wip-us.apache.org/repos/asf/hive/blob/cf0481fc/testutils/ptest2/src/main/resources/log4j2.xml
    ----------------------------------------------------------------------
    diff --git a/testutils/ptest2/src/main/resources/log4j2.xml b/testutils/ptest2/src/main/resources/log4j2.xml
    index 992462e..6502ad1 100644
    --- a/testutils/ptest2/src/main/resources/log4j2.xml
    +++ b/testutils/ptest2/src/main/resources/log4j2.xml
    @@ -20,7 +20,7 @@
       packages="org.apache.hadoop.hive.ql.log">

        <Properties>
    - <Property name="hive.ptest.log.threshold">ALL</Property>
    + <Property name="hive.ptest.log.threshold">DEBUG</Property>
          <Property name="hive.ptest.log.level">DEBUG</Property>
          <Property name="hive.ptest.root.logger">FILE</Property>
          <Property name="hive.ptest.log.dir">target</Property>
  • Sershe at Aug 25, 2015 at 9:28 pm
    HIVE-11571: Fix Hive PTest2 logging configuration (Gopal V reviewed by Sergey Shelukhin)


    Project: http://git-wip-us.apache.org/repos/asf/hive/repo
    Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/3071ce96
    Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/3071ce96
    Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/3071ce96

    Branch: refs/heads/llap
    Commit: 3071ce96b6b8635f668d0698c18a727bea1b1de1
    Parents: fe1efe5
    Author: Prasanth Jayachandran <j.prasanth.j@gmail.com>
    Authored: Fri Aug 14 15:40:48 2015 -0700
    Committer: Prasanth Jayachandran <j.prasanth.j@gmail.com>
    Committed: Fri Aug 14 15:40:48 2015 -0700

    ----------------------------------------------------------------------
      testutils/ptest2/src/main/resources/log4j2.xml | 1 +
      1 file changed, 1 insertion(+)
    ----------------------------------------------------------------------


    http://git-wip-us.apache.org/repos/asf/hive/blob/3071ce96/testutils/ptest2/src/main/resources/log4j2.xml
    ----------------------------------------------------------------------
    diff --git a/testutils/ptest2/src/main/resources/log4j2.xml b/testutils/ptest2/src/main/resources/log4j2.xml
    index 6502ad1..42141b7 100644
    --- a/testutils/ptest2/src/main/resources/log4j2.xml
    +++ b/testutils/ptest2/src/main/resources/log4j2.xml
    @@ -75,5 +75,6 @@
          <Logger name="org.apache.zookeeper.ClientCnxnSocketNIO" level="WARN">
            <AppenderRef ref="${sys:hive.ptest.root.logger}"/>
          </Logger>
    + </Loggers>

      </Configuration>
  • Sershe at Aug 25, 2015 at 9:28 pm
    HIVE-11575: Fix test failures in master due to log4j changes (Prasanth Jayachandran reviewed by Sergey Shelukhin)


    Project: http://git-wip-us.apache.org/repos/asf/hive/repo
    Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/e0594099
    Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/e0594099
    Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/e0594099

    Branch: refs/heads/llap
    Commit: e0594099c2ea1652e49ff041c6dfb569f47a5912
    Parents: 9d8515d
    Author: Prasanth Jayachandran <j.prasanth.j@gmail.com>
    Authored: Mon Aug 17 13:47:09 2015 -0700
    Committer: Prasanth Jayachandran <j.prasanth.j@gmail.com>
    Committed: Mon Aug 17 13:47:09 2015 -0700

    ----------------------------------------------------------------------
      data/conf/hive-log4j2.xml | 5 ++---
      data/conf/hive-site.xml | 6 ------
      .../service/cli/operation/TestOperationLoggingAPIWithMr.java | 2 --
      .../service/cli/operation/TestOperationLoggingAPIWithTez.java | 2 --
      .../hive/service/cli/operation/TestOperationLoggingLayout.java | 2 --
      5 files changed, 2 insertions(+), 15 deletions(-)
    ----------------------------------------------------------------------


    http://git-wip-us.apache.org/repos/asf/hive/blob/e0594099/data/conf/hive-log4j2.xml
    ----------------------------------------------------------------------
    diff --git a/data/conf/hive-log4j2.xml b/data/conf/hive-log4j2.xml
    index 51173a0..11c8e79 100644
    --- a/data/conf/hive-log4j2.xml
    +++ b/data/conf/hive-log4j2.xml
    @@ -25,7 +25,6 @@
          <Property name="hive.root.logger">DRFA</Property>
          <Property name="hive.log.dir">${sys:test.tmp.dir}/log</Property>
          <Property name="hive.log.file">hive.log</Property>
    - <Property name="hive.ql.log.PerfLogger.level">INFO</Property>
        </Properties>

        <Appenders>
    @@ -95,8 +94,8 @@
          <Logger name="org.apache.zookeeper.ClientCnxnSocketNIO" level="WARN">
            <AppenderRef ref="${sys:hive.root.logger}"/>
          </Logger>
    - <Logger name="org.apache.hadoop.hive.ql.log.PerfLogger" level="${sys:hive.ql.log.PerfLogger.level}" additivity="false">
    - <AppenderRef ref="${sys:hive.root.logger}"/>
    + <Logger name="org.apache.hadoop.hive.ql.log.PerfLogger" level="${sys:hive.ql.log.PerfLogger.level}">
    + <AppenderRef ref="${sys:hive.ql.log.PerfLogger.logger}"/>
          </Logger>
          <Logger name="org.apache.hadoop.hive.ql.exec.Operator" level="INFO">
            <AppenderRef ref="${sys:hive.root.logger}"/>

    http://git-wip-us.apache.org/repos/asf/hive/blob/e0594099/data/conf/hive-site.xml
    ----------------------------------------------------------------------
    diff --git a/data/conf/hive-site.xml b/data/conf/hive-site.xml
    index 8f7fb28..a58017e 100644
    --- a/data/conf/hive-site.xml
    +++ b/data/conf/hive-site.xml
    @@ -241,12 +241,6 @@
      </property>

      <property>
    - <name>hive.ql.log.PerfLogger.level</name>
    - <value>WARN,DRFA</value>
    - <description>Used to change the perflogger level</description>
    -</property>
    -
    -<property>
        <name>hive.fetch.task.conversion</name>
        <value>minimal</value>
      </property>

    http://git-wip-us.apache.org/repos/asf/hive/blob/e0594099/itests/hive-unit/src/test/java/org/apache/hive/service/cli/operation/TestOperationLoggingAPIWithMr.java
    ----------------------------------------------------------------------
    diff --git a/itests/hive-unit/src/test/java/org/apache/hive/service/cli/operation/TestOperationLoggingAPIWithMr.java b/itests/hive-unit/src/test/java/org/apache/hive/service/cli/operation/TestOperationLoggingAPIWithMr.java
    index 920b563..0155b75 100644
    --- a/itests/hive-unit/src/test/java/org/apache/hive/service/cli/operation/TestOperationLoggingAPIWithMr.java
    +++ b/itests/hive-unit/src/test/java/org/apache/hive/service/cli/operation/TestOperationLoggingAPIWithMr.java
    @@ -62,8 +62,6 @@ public class TestOperationLoggingAPIWithMr extends OperationLoggingAPITestBase{
          };
          hiveConf = new HiveConf();
          hiveConf.set(ConfVars.HIVE_SERVER2_LOGGING_OPERATION_LEVEL.varname, "verbose");
    - // We need to set the below parameter to test performance level logging
    - hiveConf.set("hive.ql.log.PerfLogger.level", "INFO,DRFA");
          miniHS2 = new MiniHS2(hiveConf);
          confOverlay = new HashMap<String, String>();
          confOverlay.put(ConfVars.HIVE_SUPPORT_CONCURRENCY.varname, "false");

    http://git-wip-us.apache.org/repos/asf/hive/blob/e0594099/itests/hive-unit/src/test/java/org/apache/hive/service/cli/operation/TestOperationLoggingAPIWithTez.java
    ----------------------------------------------------------------------
    diff --git a/itests/hive-unit/src/test/java/org/apache/hive/service/cli/operation/TestOperationLoggingAPIWithTez.java b/itests/hive-unit/src/test/java/org/apache/hive/service/cli/operation/TestOperationLoggingAPIWithTez.java
    index 31f34b2..ab29861 100644
    --- a/itests/hive-unit/src/test/java/org/apache/hive/service/cli/operation/TestOperationLoggingAPIWithTez.java
    +++ b/itests/hive-unit/src/test/java/org/apache/hive/service/cli/operation/TestOperationLoggingAPIWithTez.java
    @@ -40,8 +40,6 @@ public class TestOperationLoggingAPIWithTez extends OperationLoggingAPITestBase
          };
          hiveConf = new HiveConf();
          hiveConf.set(ConfVars.HIVE_SERVER2_LOGGING_OPERATION_LEVEL.varname, "verbose");
    - // We need to set the below parameter to test performance level logging
    - hiveConf.set("hive.ql.log.PerfLogger.level", "INFO,DRFA");
          // Change the engine to tez
          hiveConf.setVar(ConfVars.HIVE_EXECUTION_ENGINE, "tez");
          // Set tez execution summary to false.

    http://git-wip-us.apache.org/repos/asf/hive/blob/e0594099/itests/hive-unit/src/test/java/org/apache/hive/service/cli/operation/TestOperationLoggingLayout.java
    ----------------------------------------------------------------------
    diff --git a/itests/hive-unit/src/test/java/org/apache/hive/service/cli/operation/TestOperationLoggingLayout.java b/itests/hive-unit/src/test/java/org/apache/hive/service/cli/operation/TestOperationLoggingLayout.java
    index 93c16de..56f6a31 100644
    --- a/itests/hive-unit/src/test/java/org/apache/hive/service/cli/operation/TestOperationLoggingLayout.java
    +++ b/itests/hive-unit/src/test/java/org/apache/hive/service/cli/operation/TestOperationLoggingLayout.java
    @@ -39,8 +39,6 @@ public class TestOperationLoggingLayout {
          tableName = "TestOperationLoggingLayout_table";
          hiveConf = new HiveConf();
          hiveConf.set(HiveConf.ConfVars.HIVE_SERVER2_LOGGING_OPERATION_LEVEL.varname, "execution");
    - // We need to set the below parameter to test performance level logging
    - hiveConf.set("hive.ql.log.PerfLogger.level", "INFO,DRFA");
          miniHS2 = new MiniHS2(hiveConf);
          confOverlay = new HashMap<String, String>();
          confOverlay.put(HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY.varname, "false");
  • Sershe at Aug 25, 2015 at 9:28 pm
    HIVE-11557: CBO (Calcite Return Path): Convert to flat AND/OR (Jesus Camacho Rodriguez, reviewed by Ashutosh Chauhan)


    Project: http://git-wip-us.apache.org/repos/asf/hive/repo
    Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/e8329ee0
    Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/e8329ee0
    Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/e8329ee0

    Branch: refs/heads/llap
    Commit: e8329ee0068dbfe2ca92399d0109f308d6cf7541
    Parents: 147347a
    Author: Jesus Camacho Rodriguez <jcamacho@apache.org>
    Authored: Sat Aug 15 11:11:42 2015 +0300
    Committer: Jesus Camacho Rodriguez <jcamacho@apache.org>
    Committed: Sat Aug 15 11:11:42 2015 +0300

    ----------------------------------------------------------------------
      .../calcite/translator/ExprNodeConverter.java | 26 --------------------
      1 file changed, 26 deletions(-)
    ----------------------------------------------------------------------


    http://git-wip-us.apache.org/repos/asf/hive/blob/e8329ee0/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/ExprNodeConverter.java
    ----------------------------------------------------------------------
    diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/ExprNodeConverter.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/ExprNodeConverter.java
    index 00bf009..ec22f1a 100644
    --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/ExprNodeConverter.java
    +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/ExprNodeConverter.java
    @@ -20,7 +20,6 @@ package org.apache.hadoop.hive.ql.optimizer.calcite.translator;
      import java.math.BigDecimal;
      import java.sql.Date;
      import java.sql.Timestamp;
    -import java.util.ArrayList;
      import java.util.Calendar;
      import java.util.LinkedList;
      import java.util.List;
    @@ -43,11 +42,9 @@ import org.apache.calcite.sql.SqlKind;
      import org.apache.calcite.sql.type.SqlTypeUtil;
      import org.apache.commons.logging.Log;
      import org.apache.commons.logging.LogFactory;
    -import org.apache.hadoop.hive.common.type.HiveChar;
      import org.apache.hadoop.hive.common.type.HiveDecimal;
      import org.apache.hadoop.hive.common.type.HiveIntervalDayTime;
      import org.apache.hadoop.hive.common.type.HiveIntervalYearMonth;
    -import org.apache.hadoop.hive.common.type.HiveVarchar;
      import org.apache.hadoop.hive.ql.exec.UDFArgumentException;
      import org.apache.hadoop.hive.ql.optimizer.calcite.translator.ASTConverter.RexVisitor;
      import org.apache.hadoop.hive.ql.optimizer.calcite.translator.ASTConverter.Schema;
    @@ -139,29 +136,6 @@ public class ExprNodeConverter extends RexVisitorImpl<ExprNodeDesc> {
              && SqlTypeUtil.equalSansNullability(dTFactory, call.getType(),
                  call.operands.get(0).getType())) {
            return args.get(0);
    - } else if (ASTConverter.isFlat(call)) {
    - // If Expr is flat (and[p,q,r,s] or[p,q,r,s]) then recursively build the
    - // exprnode
    - GenericUDF hiveUdf = SqlFunctionConverter.getHiveUDF(call.getOperator(), call.getType(), 2);
    - ArrayList<ExprNodeDesc> tmpExprArgs = new ArrayList<ExprNodeDesc>();
    - tmpExprArgs.addAll(args.subList(0, 2));
    - try {
    - gfDesc = ExprNodeGenericFuncDesc.newInstance(hiveUdf, tmpExprArgs);
    - } catch (UDFArgumentException e) {
    - LOG.error(e);
    - throw new RuntimeException(e);
    - }
    - for (int i = 2; i < call.operands.size(); i++) {
    - tmpExprArgs = new ArrayList<ExprNodeDesc>();
    - tmpExprArgs.add(gfDesc);
    - tmpExprArgs.add(args.get(i));
    - try {
    - gfDesc = ExprNodeGenericFuncDesc.newInstance(hiveUdf, tmpExprArgs);
    - } catch (UDFArgumentException e) {
    - LOG.error(e);
    - throw new RuntimeException(e);
    - }
    - }
          } else {
            GenericUDF hiveUdf = SqlFunctionConverter.getHiveUDF(call.getOperator(), call.getType(),
                args.size());
  • Sershe at Aug 25, 2015 at 9:28 pm
    HIVE-11597 : [CBO new return path] Handling of strings of zero-length (Ashutosh Chauhan via Jesus Camacho Rodriguez)


    Project: http://git-wip-us.apache.org/repos/asf/hive/repo
    Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/c3c2bdac
    Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/c3c2bdac
    Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/c3c2bdac

    Branch: refs/heads/llap
    Commit: c3c2bdacdbfbcad5a92959e05313c6ff677af89c
    Parents: e1b88a1
    Author: Ashutosh Chauhan <hashutosh@apache.org>
    Authored: Wed Aug 19 09:15:18 2015 -0700
    Committer: Ashutosh Chauhan <hashutosh@apache.org>
    Committed: Wed Aug 19 09:15:18 2015 -0700

    ----------------------------------------------------------------------
      .../ql/optimizer/calcite/translator/TypeConverter.java | 13 ++++++++++---
      1 file changed, 10 insertions(+), 3 deletions(-)
    ----------------------------------------------------------------------


    http://git-wip-us.apache.org/repos/asf/hive/blob/c3c2bdac/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/TypeConverter.java
    ----------------------------------------------------------------------
    diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/TypeConverter.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/TypeConverter.java
    index 13c8c23..2825f77 100644
    --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/TypeConverter.java
    +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/TypeConverter.java
    @@ -31,6 +31,8 @@ import org.apache.calcite.rex.RexBuilder;
      import org.apache.calcite.sql.parser.SqlParserPos;
      import org.apache.calcite.sql.SqlIntervalQualifier;
      import org.apache.calcite.sql.type.SqlTypeName;
    +import org.apache.hadoop.hive.common.type.HiveChar;
    +import org.apache.hadoop.hive.common.type.HiveVarchar;
      import org.apache.hadoop.hive.ql.exec.ColumnInfo;
      import org.apache.hadoop.hive.ql.exec.RowSchema;
      import org.apache.hadoop.hive.ql.optimizer.calcite.CalciteSemanticException;
    @@ -300,12 +302,17 @@ public class TypeConverter {
          case DECIMAL:
            return TypeInfoFactory.getDecimalTypeInfo(rType.getPrecision(), rType.getScale());
          case VARCHAR:
    - if (rType.getPrecision() == Integer.MAX_VALUE)
    + int varcharLength = rType.getPrecision();
    + if (varcharLength < 1 || varcharLength > HiveVarchar.MAX_VARCHAR_LENGTH)
              return TypeInfoFactory.getPrimitiveTypeInfo(serdeConstants.STRING_TYPE_NAME);
            else
    - return TypeInfoFactory.getVarcharTypeInfo(rType.getPrecision());
    + return TypeInfoFactory.getVarcharTypeInfo(varcharLength);
          case CHAR:
    - return TypeInfoFactory.getCharTypeInfo(rType.getPrecision());
    + int charLength = rType.getPrecision();
    + if (charLength < 1 || charLength > HiveChar.MAX_CHAR_LENGTH)
    + return TypeInfoFactory.getPrimitiveTypeInfo(serdeConstants.STRING_TYPE_NAME);
    + else
    + return TypeInfoFactory.getCharTypeInfo(charLength);
          case OTHER:
          default:
            return TypeInfoFactory.voidTypeInfo;
  • Sershe at Aug 25, 2015 at 9:28 pm
    HIVE-11580: ThriftUnionObjectInspector#toString throws NPE (Jimmy, reviewed by Chao)


    Project: http://git-wip-us.apache.org/repos/asf/hive/repo
    Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/6433c3b0
    Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/6433c3b0
    Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/6433c3b0

    Branch: refs/heads/llap
    Commit: 6433c3b0b8e305c8dddc6ea2fe94cfcd5062e40d
    Parents: 5edbf31
    Author: Jimmy Xiang <jxiang@cloudera.com>
    Authored: Fri Aug 14 15:48:23 2015 -0700
    Committer: Jimmy Xiang <jxiang@cloudera.com>
    Committed: Tue Aug 18 08:49:21 2015 -0700

    ----------------------------------------------------------------------
      .../ThriftUnionObjectInspector.java | 26 ++++++++++----------
      .../TestThriftObjectInspectors.java | 10 ++++++--
      2 files changed, 21 insertions(+), 15 deletions(-)
    ----------------------------------------------------------------------


    http://git-wip-us.apache.org/repos/asf/hive/blob/6433c3b0/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ThriftUnionObjectInspector.java
    ----------------------------------------------------------------------
    diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ThriftUnionObjectInspector.java b/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ThriftUnionObjectInspector.java
    index 17add28..600abbb 100644
    --- a/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ThriftUnionObjectInspector.java
    +++ b/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ThriftUnionObjectInspector.java
    @@ -18,18 +18,18 @@

      package org.apache.hadoop.hive.serde2.objectinspector;

    -import com.google.common.primitives.UnsignedBytes;
    -import org.apache.hadoop.util.ReflectionUtils;
    -import org.apache.thrift.TFieldIdEnum;
    -import org.apache.thrift.TUnion;
    -import org.apache.thrift.meta_data.FieldMetaData;
    -
      import java.lang.reflect.Field;
      import java.lang.reflect.Type;
      import java.util.ArrayList;
      import java.util.List;
      import java.util.Map;

    +import org.apache.thrift.TFieldIdEnum;
    +import org.apache.thrift.TUnion;
    +import org.apache.thrift.meta_data.FieldMetaData;
    +
    +import com.google.common.primitives.UnsignedBytes;
    +
      /**
       * Always use the ObjectInspectorFactory to create new ObjectInspector objects,
       * instead of directly creating an instance of this class.
    @@ -37,7 +37,8 @@ import java.util.Map;
      public class ThriftUnionObjectInspector extends ReflectionStructObjectInspector implements UnionObjectInspector {

        private static final String FIELD_METADATA_MAP = "metaDataMap";
    - private List<ObjectInspector> ois;
    + private List<ObjectInspector> ois;
    + private List<StandardStructObjectInspector.MyField> fields;

        @Override
        public boolean shouldIgnoreField(String name) {
    @@ -88,10 +89,14 @@ public class ThriftUnionObjectInspector extends ReflectionStructObjectInspector

          try {
            final Map<? extends TFieldIdEnum, FieldMetaData> fieldMap = (Map<? extends TFieldIdEnum, FieldMetaData>) fieldMetaData.get(null);
    + fields = new ArrayList<StandardStructObjectInspector.MyField>(fieldMap.size());
            this.ois = new ArrayList<ObjectInspector>();
            for(Map.Entry<? extends TFieldIdEnum, FieldMetaData> metadata : fieldMap.entrySet()) {
    - final Type fieldType = ThriftObjectInspectorUtils.getFieldType(objectClass, metadata.getValue().fieldName);
    + int fieldId = metadata.getKey().getThriftFieldId();
    + String fieldName = metadata.getValue().fieldName;
    + final Type fieldType = ThriftObjectInspectorUtils.getFieldType(objectClass, fieldName);
              final ObjectInspector reflectionObjectInspector = ObjectInspectorFactory.getReflectionObjectInspector(fieldType, options);
    + fields.add(new StandardStructObjectInspector.MyField(fieldId, fieldName, reflectionObjectInspector));
              this.ois.add(reflectionObjectInspector);
            }
          } catch (IllegalAccessException e) {
    @@ -112,10 +117,5 @@ public class ThriftUnionObjectInspector extends ReflectionStructObjectInspector
        public String getTypeName() {
          return ObjectInspectorUtils.getStandardUnionTypeName(this);
        }
    -
    - @Override
    - public Object create() {
    - return ReflectionUtils.newInstance(objectClass, null);
    - }
      }


    http://git-wip-us.apache.org/repos/asf/hive/blob/6433c3b0/serde/src/test/org/apache/hadoop/hive/serde2/objectinspector/TestThriftObjectInspectors.java
    ----------------------------------------------------------------------
    diff --git a/serde/src/test/org/apache/hadoop/hive/serde2/objectinspector/TestThriftObjectInspectors.java b/serde/src/test/org/apache/hadoop/hive/serde2/objectinspector/TestThriftObjectInspectors.java
    index 968cd22..85f2bd6 100644
    --- a/serde/src/test/org/apache/hadoop/hive/serde2/objectinspector/TestThriftObjectInspectors.java
    +++ b/serde/src/test/org/apache/hadoop/hive/serde2/objectinspector/TestThriftObjectInspectors.java
    @@ -23,14 +23,15 @@ import java.util.HashSet;
      import java.util.List;
      import java.util.Set;

    -import junit.framework.TestCase;
    -
      import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category;
      import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
      import org.apache.hadoop.hive.serde2.thrift.test.Complex;
      import org.apache.hadoop.hive.serde2.thrift.test.IntString;
    +import org.apache.hadoop.hive.serde2.thrift.test.PropValueUnion;
      import org.apache.hadoop.hive.serde2.thrift.test.SetIntString;

    +import junit.framework.TestCase;
    +
      /**
       * TestThriftObjectInspectors.
       *
    @@ -60,6 +61,11 @@ public class TestThriftObjectInspectors extends TestCase {
              assertNull(soi.getStructFieldData(null, fields.get(i)));
            }

    + ObjectInspector oi = ObjectInspectorFactory
    + .getReflectionObjectInspector(PropValueUnion.class,
    + ObjectInspectorFactory.ObjectInspectorOptions.THRIFT);
    + assertNotNull(oi.toString());
    +
            // real object
            Complex c = new Complex();
            c.setAint(1);
  • Sershe at Aug 25, 2015 at 9:28 pm
    HIVE-11490 : Lazily call ASTNode::toStringTree() after tree modification (Hari Subramaniyan, reviewed by Ashutosh Chauhan)


    Project: http://git-wip-us.apache.org/repos/asf/hive/repo
    Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/9d8515df
    Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/9d8515df
    Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/9d8515df

    Branch: refs/heads/llap
    Commit: 9d8515df6ff722cd81b9b42a582c422adeac8849
    Parents: 97a6cd3
    Author: Hari Subramaniyan <harisankar@apache.org>
    Authored: Mon Aug 17 11:32:06 2015 -0700
    Committer: Hari Subramaniyan <harisankar@apache.org>
    Committed: Mon Aug 17 11:32:06 2015 -0700

    ----------------------------------------------------------------------
      .../hadoop/hive/ql/lib/DefaultGraphWalker.java | 2 +-
      .../org/apache/hadoop/hive/ql/parse/ASTNode.java | 18 +++++++++---------
      2 files changed, 10 insertions(+), 10 deletions(-)
    ----------------------------------------------------------------------


    http://git-wip-us.apache.org/repos/asf/hive/blob/9d8515df/ql/src/java/org/apache/hadoop/hive/ql/lib/DefaultGraphWalker.java
    ----------------------------------------------------------------------
    diff --git a/ql/src/java/org/apache/hadoop/hive/ql/lib/DefaultGraphWalker.java b/ql/src/java/org/apache/hadoop/hive/ql/lib/DefaultGraphWalker.java
    index cf9131d..583c113 100644
    --- a/ql/src/java/org/apache/hadoop/hive/ql/lib/DefaultGraphWalker.java
    +++ b/ql/src/java/org/apache/hadoop/hive/ql/lib/DefaultGraphWalker.java
    @@ -108,7 +108,7 @@ public class DefaultGraphWalker implements GraphWalker {
          while (toWalk.size() > 0) {
            Node nd = toWalk.remove(0);
            walk(nd);
    - if (nodeOutput != null) {
    + if (nodeOutput != null && getDispatchedList().contains(nd)) {
              nodeOutput.put(nd, retMap.get(nd));
            }
          }

    http://git-wip-us.apache.org/repos/asf/hive/blob/9d8515df/ql/src/java/org/apache/hadoop/hive/ql/parse/ASTNode.java
    ----------------------------------------------------------------------
    diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/ASTNode.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/ASTNode.java
    index 136d481..b96e2eb 100644
    --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/ASTNode.java
    +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/ASTNode.java
    @@ -143,9 +143,11 @@ public class ASTNode extends CommonTree implements Node,Serializable {
            retNode = (ASTNode) retNode.parent;
          }
          rootNode=retNode;
    - rootNode.astStr = new StringBuffer();
    - rootNode.toStringTree(rootNode);
    - rootNode.isValidASTStr = true;
    + if (!rootNode.isValidASTStr) {
    + rootNode.astStr = new StringBuffer();
    + rootNode.toStringTree(rootNode);
    + rootNode.isValidASTStr = true;
    + }
          return retNode;
        }

    @@ -159,9 +161,6 @@ public class ASTNode extends CommonTree implements Node,Serializable {
            rootNode.astStr = null;
            rootNode.isValidASTStr = false;
          }
    - // The root might have changed because of tree modifications.
    - // Compute the new root for this tree and set the astStr.
    - getRootNodeWithValidASTStr(false);
        }

        private int getMemoizedStringLen() {
    @@ -225,9 +224,10 @@ public class ASTNode extends CommonTree implements Node,Serializable {

        @Override
        public String toStringTree() {
    - // The tree modifier functions invalidate the old astStr, rootNode, etc.
    - // Hence, we can use the memoized root node and string values here.
    - ASTNode rootNode = (ASTNode)this.getRootNodeWithValidASTStr(true);
    +
    + // The root might have changed because of tree modifications.
    + // Compute the new root for this tree and set the astStr.
    + getRootNodeWithValidASTStr(true);

          // If rootNotModified is false, then startIndx and endIndx will be stale.
          if (startIndx >= 0 && endIndx <= rootNode.getMemoizedStringLen()) {
  • Sershe at Aug 25, 2015 at 9:28 pm
    HIVE-11429: Increase default JDBC result set fetch size (# rows it fetches in one RPC call) to 1000 from 50 (Vaibhav Gumashta reviewed by Thejas Nair)


    Project: http://git-wip-us.apache.org/repos/asf/hive/repo
    Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/5edbf31a
    Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/5edbf31a
    Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/5edbf31a

    Branch: refs/heads/llap
    Commit: 5edbf31a755668b213c2a21fb8a4a2e902e081f6
    Parents: 3b6825b
    Author: Vaibhav Gumashta <vgumashta@apache.org>
    Authored: Tue Aug 18 02:12:35 2015 -0700
    Committer: Vaibhav Gumashta <vgumashta@apache.org>
    Committed: Tue Aug 18 02:12:35 2015 -0700

    ----------------------------------------------------------------------
      jdbc/src/java/org/apache/hive/jdbc/HiveStatement.java | 2 +-
      1 file changed, 1 insertion(+), 1 deletion(-)
    ----------------------------------------------------------------------


    http://git-wip-us.apache.org/repos/asf/hive/blob/5edbf31a/jdbc/src/java/org/apache/hive/jdbc/HiveStatement.java
    ----------------------------------------------------------------------
    diff --git a/jdbc/src/java/org/apache/hive/jdbc/HiveStatement.java b/jdbc/src/java/org/apache/hive/jdbc/HiveStatement.java
    index 170fc53..d4041bb 100644
    --- a/jdbc/src/java/org/apache/hive/jdbc/HiveStatement.java
    +++ b/jdbc/src/java/org/apache/hive/jdbc/HiveStatement.java
    @@ -59,7 +59,7 @@ public class HiveStatement implements java.sql.Statement {
        private TOperationHandle stmtHandle = null;
        private final TSessionHandle sessHandle;
        Map<String,String> sessConf = new HashMap<String,String>();
    - private int fetchSize = 50;
    + private int fetchSize = 1000;
        private boolean isScrollableResultset = false;
        /**
         * We need to keep a reference to the result set to support the following:
  • Sershe at Aug 25, 2015 at 9:28 pm
    HIVE-11176 : Caused by: java.lang.ClassCastException: org.apache.hadoop.hive.serde2.lazybinary.LazyBinaryStruct cannot be cast to [Ljava.lang.Object; (Navis via Ashutosh Chauhan)


    Project: http://git-wip-us.apache.org/repos/asf/hive/repo
    Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/a16bbd48
    Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/a16bbd48
    Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/a16bbd48

    Branch: refs/heads/llap
    Commit: a16bbd4877da11871656ed082a84d6809139740f
    Parents: f7ebff0
    Author: Navis Ryu <navis@apache.org>
    Authored: Tue Aug 11 17:34:00 2015 -0800
    Committer: Ashutosh Chauhan <hashutosh@apache.org>
    Committed: Mon Aug 24 00:30:11 2015 -0700

    ----------------------------------------------------------------------
      .../serde2/objectinspector/StandardStructObjectInspector.java | 7 ++++++-
      1 file changed, 6 insertions(+), 1 deletion(-)
    ----------------------------------------------------------------------


    http://git-wip-us.apache.org/repos/asf/hive/blob/a16bbd48/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/StandardStructObjectInspector.java
    ----------------------------------------------------------------------
    diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/StandardStructObjectInspector.java b/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/StandardStructObjectInspector.java
    index c96fc2d..87a072c 100644
    --- a/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/StandardStructObjectInspector.java
    +++ b/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/StandardStructObjectInspector.java
    @@ -163,8 +163,13 @@ public class StandardStructObjectInspector extends
          }
          // We support both List<Object> and Object[]
          // so we have to do differently.
    - boolean isArray = ! (data instanceof List);
    + boolean isArray = data.getClass().isArray();
          if (!isArray && !(data instanceof List)) {
    + if (!warned) {
    + LOG.warn("Invalid type for struct " + data.getClass());
    + LOG.warn("ignoring similar errors.");
    + warned = true;
    + }
            return data;
          }
          int listSize = (isArray ? ((Object[]) data).length : ((List<Object>) data)
  • Sershe at Aug 25, 2015 at 9:28 pm
    HIVE-11602: Support Struct with different field types in query (Jesus Camacho Rodriguez, reviewed by Hari Sankar Sivarama Subramaniyan)


    Project: http://git-wip-us.apache.org/repos/asf/hive/repo
    Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/0012864b
    Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/0012864b
    Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/0012864b

    Branch: refs/heads/llap
    Commit: 0012864b8d370f948ac2cb875e4264100390d1d4
    Parents: ab03dc9
    Author: Jesus Camacho Rodriguez <jcamacho@apache.org>
    Authored: Wed Aug 19 12:52:16 2015 +0300
    Committer: Jesus Camacho Rodriguez <jcamacho@apache.org>
    Committed: Thu Aug 20 15:17:40 2015 +0300

    ----------------------------------------------------------------------
      .../hadoop/hive/ql/exec/FunctionRegistry.java | 61 ++++++++++++++++++--
      .../ql/udf/generic/GenericUDFBaseNumeric.java | 4 +-
      ql/src/test/queries/clientpositive/structin.q | 6 ++
      .../test/results/clientpositive/structin.q.out | 44 ++++++++++++++
      4 files changed, 109 insertions(+), 6 deletions(-)
    ----------------------------------------------------------------------


    http://git-wip-us.apache.org/repos/asf/hive/blob/0012864b/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java
    ----------------------------------------------------------------------
    diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java
    index 9edcc4d..4c1c53e 100644
    --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java
    +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java
    @@ -132,6 +132,7 @@ import org.apache.hadoop.hive.serde2.typeinfo.HiveDecimalUtils;
      import org.apache.hadoop.hive.serde2.typeinfo.ListTypeInfo;
      import org.apache.hadoop.hive.serde2.typeinfo.MapTypeInfo;
      import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo;
    +import org.apache.hadoop.hive.serde2.typeinfo.StructTypeInfo;
      import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
      import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
      import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils;
    @@ -765,10 +766,11 @@ public final class FunctionRegistry {
          return null;
        }

    - public static PrimitiveCategory getCommonCategory(TypeInfo a, TypeInfo b) {
    + public static PrimitiveCategory getPrimitiveCommonCategory(TypeInfo a, TypeInfo b) {
          if (a.getCategory() != Category.PRIMITIVE || b.getCategory() != Category.PRIMITIVE) {
            return null;
          }
    +
          PrimitiveCategory pcA = ((PrimitiveTypeInfo)a).getPrimitiveCategory();
          PrimitiveCategory pcB = ((PrimitiveTypeInfo)b).getPrimitiveCategory();

    @@ -802,10 +804,61 @@ public final class FunctionRegistry {
            return a;
          }

    - PrimitiveCategory commonCat = getCommonCategory(a, b);
    - if (commonCat == null)
    + // We try to infer a common primitive category
    + PrimitiveCategory commonCat = getPrimitiveCommonCategory(a, b);
    + if (commonCat != null) {
    + return getTypeInfoForPrimitiveCategory((PrimitiveTypeInfo)a, (PrimitiveTypeInfo)b, commonCat);
    + }
    + // It is not primitive; check if it is a struct and we can infer a common class
    + if (a.getCategory() == Category.STRUCT && b.getCategory() == Category.STRUCT) {
    + return getCommonClassForStruct((StructTypeInfo)a, (StructTypeInfo)b);
    + }
    + return null;
    + }
    +
    + /**
    + * Find a common class that objects of both StructTypeInfo a and StructTypeInfo b can
    + * convert to. This is used for places other than comparison.
    + *
    + * @return null if no common class could be found.
    + */
    + public static TypeInfo getCommonClassForStruct(StructTypeInfo a, StructTypeInfo b) {
    + if (a == b || a.equals(b)) {
    + return a;
    + }
    +
    + List<String> names = new ArrayList<String>();
    + List<TypeInfo> typeInfos = new ArrayList<TypeInfo>();
    +
    + Iterator<String> namesIterator = a.getAllStructFieldNames().iterator();
    + Iterator<String> otherNamesIterator = b.getAllStructFieldNames().iterator();
    +
    + // Compare the field names using ignore-case semantics
    + while (namesIterator.hasNext() && otherNamesIterator.hasNext()) {
    + String name = namesIterator.next();
    + if (!name.equalsIgnoreCase(otherNamesIterator.next())) {
    + return null;
    + }
    + names.add(name);
    + }
    +
    + // Different number of field names
    + if (namesIterator.hasNext() || otherNamesIterator.hasNext()) {
            return null;
    - return getTypeInfoForPrimitiveCategory((PrimitiveTypeInfo)a, (PrimitiveTypeInfo)b, commonCat);
    + }
    +
    + // Compare the field types
    + ArrayList<TypeInfo> fromTypes = a.getAllStructFieldTypeInfos();
    + ArrayList<TypeInfo> toTypes = b.getAllStructFieldTypeInfos();
    + for (int i = 0; i < fromTypes.size(); i++) {
    + TypeInfo commonType = getCommonClass(fromTypes.get(i), toTypes.get(i));
    + if (commonType == null) {
    + return null;
    + }
    + typeInfos.add(commonType);
    + }
    +
    + return TypeInfoFactory.getStructTypeInfo(names, typeInfos);
        }

        public static boolean implicitConvertible(PrimitiveCategory from, PrimitiveCategory to) {

    http://git-wip-us.apache.org/repos/asf/hive/blob/0012864b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFBaseNumeric.java
    ----------------------------------------------------------------------
    diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFBaseNumeric.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFBaseNumeric.java
    index ca5c459..ef6ef11 100644
    --- a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFBaseNumeric.java
    +++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFBaseNumeric.java
    @@ -245,7 +245,7 @@ public abstract class GenericUDFBaseNumeric extends GenericUDFBaseBinary {
          }

          // Use type promotion
    - PrimitiveCategory commonCat = FunctionRegistry.getCommonCategory(left, right);
    + PrimitiveCategory commonCat = FunctionRegistry.getPrimitiveCommonCategory(left, right);
          if (commonCat == PrimitiveCategory.DECIMAL) {
            // Hive 0.12 behavior where double * decimal -> decimal is gone.
            return TypeInfoFactory.doubleTypeInfo;
    @@ -267,7 +267,7 @@ public abstract class GenericUDFBaseNumeric extends GenericUDFBaseBinary {
          PrimitiveTypeInfo right = (PrimitiveTypeInfo) TypeInfoUtils.getTypeInfoFromObjectInspector(rightOI);

          // Now we are handling exact types. Base implementation handles type promotion.
    - PrimitiveCategory commonCat = FunctionRegistry.getCommonCategory(left, right);
    + PrimitiveCategory commonCat = FunctionRegistry.getPrimitiveCommonCategory(left, right);
          if (commonCat == PrimitiveCategory.DECIMAL) {
            return deriveResultDecimalTypeInfo();
          } else {

    http://git-wip-us.apache.org/repos/asf/hive/blob/0012864b/ql/src/test/queries/clientpositive/structin.q
    ----------------------------------------------------------------------
    diff --git a/ql/src/test/queries/clientpositive/structin.q b/ql/src/test/queries/clientpositive/structin.q
    index 48b31f3..35498bf 100644
    --- a/ql/src/test/queries/clientpositive/structin.q
    +++ b/ql/src/test/queries/clientpositive/structin.q
    @@ -15,3 +15,9 @@ struct('1234-1111-0074019112','1'),
      struct('1234-1111-0074019610','1'),
      struct('1234-1111-0074022106','1')
      );
    +
    +explain select * from t11 where struct(`id`, `lineid`)
    +IN (
    +struct('1234-1111-0074578664','3'),
    +struct('1234-1111-0074578695',1)
    +);

    http://git-wip-us.apache.org/repos/asf/hive/blob/0012864b/ql/src/test/results/clientpositive/structin.q.out
    ----------------------------------------------------------------------
    diff --git a/ql/src/test/results/clientpositive/structin.q.out b/ql/src/test/results/clientpositive/structin.q.out
    index 81c792a..f176e79 100644
    --- a/ql/src/test/results/clientpositive/structin.q.out
    +++ b/ql/src/test/results/clientpositive/structin.q.out
    @@ -64,3 +64,47 @@ STAGE PLANS:
            Processor Tree:
              ListSink

    +PREHOOK: query: explain select * from t11 where struct(`id`, `lineid`)
    +IN (
    +struct('1234-1111-0074578664','3'),
    +struct('1234-1111-0074578695',1)
    +)
    +PREHOOK: type: QUERY
    +POSTHOOK: query: explain select * from t11 where struct(`id`, `lineid`)
    +IN (
    +struct('1234-1111-0074578664','3'),
    +struct('1234-1111-0074578695',1)
    +)
    +POSTHOOK: type: QUERY
    +STAGE DEPENDENCIES:
    + Stage-1 is a root stage
    + Stage-0 depends on stages: Stage-1
    +
    +STAGE PLANS:
    + Stage: Stage-1
    + Map Reduce
    + Map Operator Tree:
    + TableScan
    + alias: t11
    + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
    + Filter Operator
    + predicate: (struct(id,lineid)) IN (const struct('1234-1111-0074578664','3'), const struct('1234-1111-0074578695',1)) (type: boolean)
    + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
    + Select Operator
    + expressions: id (type: string), lineid (type: string)
    + outputColumnNames: _col0, _col1
    + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
    + File Output Operator
    + compressed: false
    + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
    + table:
    + input format: org.apache.hadoop.mapred.TextInputFormat
    + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
    + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
    +
    + Stage: Stage-0
    + Fetch Operator
    + limit: -1
    + Processor Tree:
    + ListSink
    +
  • Sershe at Aug 25, 2015 at 9:28 pm
    HIVE-11596: nvl(x, y) throws NPE if type x and type y doesn't match, rather than throwing the meaningful error (Aihua Xu, reviewed by Chao Sun)


    Project: http://git-wip-us.apache.org/repos/asf/hive/repo
    Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/3c1eae0c
    Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/3c1eae0c
    Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/3c1eae0c

    Branch: refs/heads/llap
    Commit: 3c1eae0c44450ebbf26c88066de871ea7f479981
    Parents: 1030ae7
    Author: Aihua Xu <aihuaxu@gmail.com>
    Authored: Fri Aug 21 15:23:53 2015 -0700
    Committer: Chao Sun <sunchao@apache.org>
    Committed: Fri Aug 21 15:23:53 2015 -0700

    ----------------------------------------------------------------------
      .../hive/ql/udf/generic/GenericUDFBasePad.java | 8 ++--
      .../hive/ql/udf/generic/GenericUDFNvl.java | 2 +-
      .../queries/clientnegative/nvl_mismatch_type.q | 20 +++++++++
      .../clientnegative/char_pad_convert_fail0.q.out | 2 +-
      .../clientnegative/char_pad_convert_fail1.q.out | 2 +-
      .../clientnegative/char_pad_convert_fail3.q.out | 2 +-
      .../clientnegative/nvl_mismatch_type.q.out | 43 ++++++++++++++++++++
      7 files changed, 71 insertions(+), 8 deletions(-)
    ----------------------------------------------------------------------


    http://git-wip-us.apache.org/repos/asf/hive/blob/3c1eae0c/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFBasePad.java
    ----------------------------------------------------------------------
    diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFBasePad.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFBasePad.java
    index 3904ca4..33839f7 100644
    --- a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFBasePad.java
    +++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFBasePad.java
    @@ -94,8 +94,8 @@ public abstract class GenericUDFBasePad extends GenericUDF {
        private Converter checkTextArguments(ObjectInspector[] arguments, int i)
          throws UDFArgumentException {
          if (arguments[i].getCategory() != ObjectInspector.Category.PRIMITIVE) {
    - throw new UDFArgumentTypeException(i + 1, "Only primitive type arguments are accepted but "
    - + arguments[i].getTypeName() + " is passed. as arguments");
    + throw new UDFArgumentTypeException(i, "Only primitive type arguments are accepted but "
    + + arguments[i].getTypeName() + " is passed.");
          }

          Converter converter = ObjectInspectorConverters.getConverter((PrimitiveObjectInspector) arguments[i],
    @@ -107,8 +107,8 @@ public abstract class GenericUDFBasePad extends GenericUDF {
        private Converter checkIntArguments(ObjectInspector[] arguments, int i)
          throws UDFArgumentException {
          if (arguments[i].getCategory() != ObjectInspector.Category.PRIMITIVE) {
    - throw new UDFArgumentTypeException(i + 1, "Only primitive type arguments are accepted but "
    - + arguments[i].getTypeName() + " is passed. as arguments");
    + throw new UDFArgumentTypeException(i, "Only primitive type arguments are accepted but "
    + + arguments[i].getTypeName() + " is passed.");
          }
          PrimitiveCategory inputType = ((PrimitiveObjectInspector) arguments[i]).getPrimitiveCategory();
          Converter converter;

    http://git-wip-us.apache.org/repos/asf/hive/blob/3c1eae0c/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFNvl.java
    ----------------------------------------------------------------------
    diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFNvl.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFNvl.java
    index 625b466..0a16da8 100644
    --- a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFNvl.java
    +++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFNvl.java
    @@ -42,7 +42,7 @@ public class GenericUDFNvl extends GenericUDF{
          returnOIResolver = new GenericUDFUtils.ReturnObjectInspectorResolver(true);
          if (!(returnOIResolver.update(arguments[0]) && returnOIResolver
              .update(arguments[1]))) {
    - throw new UDFArgumentTypeException(2,
    + throw new UDFArgumentTypeException(1,
                "The first and seconds arguments of function NLV should have the same type, "
                + "but they are different: \"" + arguments[0].getTypeName()
                + "\" and \"" + arguments[1].getTypeName() + "\"");

    http://git-wip-us.apache.org/repos/asf/hive/blob/3c1eae0c/ql/src/test/queries/clientnegative/nvl_mismatch_type.q
    ----------------------------------------------------------------------
    diff --git a/ql/src/test/queries/clientnegative/nvl_mismatch_type.q b/ql/src/test/queries/clientnegative/nvl_mismatch_type.q
    new file mode 100644
    index 0000000..ace180c
    --- /dev/null
    +++ b/ql/src/test/queries/clientnegative/nvl_mismatch_type.q
    @@ -0,0 +1,20 @@
    +
    +create table over1k(
    + t tinyint,
    + si smallint,
    + i int,
    + b bigint,
    + f float,
    + d double,
    + bo boolean,
    + s string,
    + ts timestamp,
    + dec decimal(4,2),
    + bin binary)
    + row format delimited
    + fields terminated by '|';
    +
    +load data local inpath '../../data/files/over1k' into table over1k;
    +
    +-- Integers
    +select nvl(t, true) from over1k limit 5;

    http://git-wip-us.apache.org/repos/asf/hive/blob/3c1eae0c/ql/src/test/results/clientnegative/char_pad_convert_fail0.q.out
    ----------------------------------------------------------------------
    diff --git a/ql/src/test/results/clientnegative/char_pad_convert_fail0.q.out b/ql/src/test/results/clientnegative/char_pad_convert_fail0.q.out
    index d6f5502..ed2f516 100644
    --- a/ql/src/test/results/clientnegative/char_pad_convert_fail0.q.out
    +++ b/ql/src/test/results/clientnegative/char_pad_convert_fail0.q.out
    @@ -40,4 +40,4 @@ POSTHOOK: query: load data local inpath '../../data/files/over1k' into table ove
      POSTHOOK: type: LOAD
      #### A masked pattern was here ####
      POSTHOOK: Output: default@over1k
    -FAILED: SemanticException [Error 10016]: Line 7:29 Argument type mismatch '' '': Only primitive type arguments are accepted but array<int> is passed. as arguments
    +FAILED: SemanticException [Error 10016]: Line 7:15 Argument type mismatch '3': Only primitive type arguments are accepted but array<int> is passed.

    http://git-wip-us.apache.org/repos/asf/hive/blob/3c1eae0c/ql/src/test/results/clientnegative/char_pad_convert_fail1.q.out
    ----------------------------------------------------------------------
    diff --git a/ql/src/test/results/clientnegative/char_pad_convert_fail1.q.out b/ql/src/test/results/clientnegative/char_pad_convert_fail1.q.out
    index 81ceeec..5f17cfb 100644
    --- a/ql/src/test/results/clientnegative/char_pad_convert_fail1.q.out
    +++ b/ql/src/test/results/clientnegative/char_pad_convert_fail1.q.out
    @@ -40,4 +40,4 @@ POSTHOOK: query: load data local inpath '../../data/files/over1k' into table ove
      POSTHOOK: type: LOAD
      #### A masked pattern was here ####
      POSTHOOK: Output: default@over1k
    -FAILED: SemanticException [Error 10016]: Line 7:26 Argument type mismatch '4': Only primitive type arguments are accepted but array<int> is passed. as arguments
    +FAILED: SemanticException [Error 10016]: Line 7:12 Argument type mismatch '3': Only primitive type arguments are accepted but array<int> is passed.

    http://git-wip-us.apache.org/repos/asf/hive/blob/3c1eae0c/ql/src/test/results/clientnegative/char_pad_convert_fail3.q.out
    ----------------------------------------------------------------------
    diff --git a/ql/src/test/results/clientnegative/char_pad_convert_fail3.q.out b/ql/src/test/results/clientnegative/char_pad_convert_fail3.q.out
    index 9896eae..a25ea15 100644
    --- a/ql/src/test/results/clientnegative/char_pad_convert_fail3.q.out
    +++ b/ql/src/test/results/clientnegative/char_pad_convert_fail3.q.out
    @@ -40,4 +40,4 @@ POSTHOOK: query: load data local inpath '../../data/files/over1k' into table ove
      POSTHOOK: type: LOAD
      #### A masked pattern was here ####
      POSTHOOK: Output: default@over1k
    -FAILED: SemanticException [Error 10016]: Line 7:38 Argument type mismatch '4': Only primitive type arguments are accepted but uniontype<string> is passed. as arguments
    +FAILED: SemanticException [Error 10016]: Line 7:12 Argument type mismatch ''Union'': Only primitive type arguments are accepted but uniontype<string> is passed.

    http://git-wip-us.apache.org/repos/asf/hive/blob/3c1eae0c/ql/src/test/results/clientnegative/nvl_mismatch_type.q.out
    ----------------------------------------------------------------------
    diff --git a/ql/src/test/results/clientnegative/nvl_mismatch_type.q.out b/ql/src/test/results/clientnegative/nvl_mismatch_type.q.out
    new file mode 100644
    index 0000000..6e88385
    --- /dev/null
    +++ b/ql/src/test/results/clientnegative/nvl_mismatch_type.q.out
    @@ -0,0 +1,43 @@
    +PREHOOK: query: create table over1k(
    + t tinyint,
    + si smallint,
    + i int,
    + b bigint,
    + f float,
    + d double,
    + bo boolean,
    + s string,
    + ts timestamp,
    + dec decimal(4,2),
    + bin binary)
    + row format delimited
    + fields terminated by '|'
    +PREHOOK: type: CREATETABLE
    +PREHOOK: Output: database:default
    +PREHOOK: Output: default@over1k
    +POSTHOOK: query: create table over1k(
    + t tinyint,
    + si smallint,
    + i int,
    + b bigint,
    + f float,
    + d double,
    + bo boolean,
    + s string,
    + ts timestamp,
    + dec decimal(4,2),
    + bin binary)
    + row format delimited
    + fields terminated by '|'
    +POSTHOOK: type: CREATETABLE
    +POSTHOOK: Output: database:default
    +POSTHOOK: Output: default@over1k
    +PREHOOK: query: load data local inpath '../../data/files/over1k' into table over1k
    +PREHOOK: type: LOAD
    +#### A masked pattern was here ####
    +PREHOOK: Output: default@over1k
    +POSTHOOK: query: load data local inpath '../../data/files/over1k' into table over1k
    +POSTHOOK: type: LOAD
    +#### A masked pattern was here ####
    +POSTHOOK: Output: default@over1k
    +FAILED: SemanticException [Error 10016]: Line 4:14 Argument type mismatch 'true': The first and seconds arguments of function NLV should have the same type, but they are different: "tinyint" and "boolean"
  • Sershe at Aug 25, 2015 at 9:28 pm
    HIVE-11506 : Casting varchar/char type to string cannot be vectorized (Navis via Ashutosh Chauhan)


    Project: http://git-wip-us.apache.org/repos/asf/hive/repo
    Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/2de6fda0
    Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/2de6fda0
    Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/2de6fda0

    Branch: refs/heads/llap
    Commit: 2de6fda0f81072a6ad1af103f2ddec49b73f4eab
    Parents: e2d148b
    Author: Navis Ryu <navis@apache.org>
    Authored: Mon Aug 10 17:36:00 2015 -0800
    Committer: Ashutosh Chauhan <hashutosh@apache.org>
    Committed: Sun Aug 23 21:32:00 2015 -0700

    ----------------------------------------------------------------------
      .../ql/exec/vector/VectorizationContext.java | 7 +-
      .../expressions/CastStringGroupToString.java | 40 ++++++++
      .../queries/clientpositive/vectorized_casts.q | 6 ++
      .../tez/vector_char_mapjoin1.q.out | 1 +
      .../tez/vector_varchar_mapjoin1.q.out | 1 +
      .../clientpositive/tez/vectorized_casts.q.out | 99 +++++++++++++-------
      .../clientpositive/vector_char_mapjoin1.q.out | 1 +
      .../vector_varchar_mapjoin1.q.out | 1 +
      .../clientpositive/vectorized_casts.q.out | 66 +++++++------
      9 files changed, 155 insertions(+), 67 deletions(-)
    ----------------------------------------------------------------------


    http://git-wip-us.apache.org/repos/asf/hive/blob/2de6fda0/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java
    ----------------------------------------------------------------------
    diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java
    index 5b702bb..34f9329 100644
    --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java
    +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java
    @@ -50,7 +50,6 @@ import org.apache.hadoop.hive.ql.exec.vector.ColumnVector.Type;
      import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor.InputExpressionType;
      import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor.Mode;
      import org.apache.hadoop.hive.ql.exec.vector.expressions.*;
    -import org.apache.hadoop.hive.ql.exec.vector.AggregateDefinition;
      import org.apache.hadoop.hive.ql.exec.vector.expressions.aggregates.VectorAggregateExpression;
      import org.apache.hadoop.hive.ql.exec.vector.expressions.aggregates.VectorUDAFAvgDecimal;
      import org.apache.hadoop.hive.ql.exec.vector.expressions.aggregates.VectorUDAFCount;
    @@ -1023,9 +1022,7 @@ public class VectorizationContext {
          VectorExpressionDescriptor.Descriptor descriptor = builder.build();
          Class<?> vclass = this.vMap.getVectorExpressionClass(udfClass, descriptor);
          if (vclass == null) {
    - if (LOG.isDebugEnabled()) {
    - LOG.debug("No vector udf found for "+udfClass.getSimpleName() + ", descriptor: "+descriptor);
    - }
    + LOG.info("No vector udf found for " + udfClass.getSimpleName() + ", descriptor: " + descriptor);
            return null;
          }
          Mode childrenMode = getChildrenMode(mode, udfClass);
    @@ -1536,6 +1533,8 @@ public class VectorizationContext {
            return createVectorExpression(CastDecimalToString.class, childExpr, Mode.PROJECTION, returnType);
          } else if (isDateFamily(inputType)) {
            return createVectorExpression(CastDateToString.class, childExpr, Mode.PROJECTION, returnType);
    + } else if (isStringFamily(inputType)) {
    + return createVectorExpression(CastStringGroupToString.class, childExpr, Mode.PROJECTION, returnType);
          }
          /* The string type is deliberately omitted -- the planner removes string to string casts.
           * Timestamp, float, and double types are handled by the legacy code path. See isLegacyPathUDF.

    http://git-wip-us.apache.org/repos/asf/hive/blob/2de6fda0/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/CastStringGroupToString.java
    ----------------------------------------------------------------------
    diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/CastStringGroupToString.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/CastStringGroupToString.java
    new file mode 100644
    index 0000000..bbc770c
    --- /dev/null
    +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/CastStringGroupToString.java
    @@ -0,0 +1,40 @@
    +/**
    + * Licensed to the Apache Software Foundation (ASF) under one
    + * or more contributor license agreements. See the NOTICE file
    + * distributed with this work for additional information
    + * regarding copyright ownership. The ASF licenses this file
    + * to you under the Apache License, Version 2.0 (the
    + * "License"); you may not use this file except in compliance
    + * with the License. You may obtain a copy of the License at
    + *
    + * http://www.apache.org/licenses/LICENSE-2.0
    + *
    + * Unless required by applicable law or agreed to in writing, software
    + * distributed under the License is distributed on an "AS IS" BASIS,
    + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    + * See the License for the specific language governing permissions and
    + * limitations under the License.
    + */
    +
    +package org.apache.hadoop.hive.ql.exec.vector.expressions;
    +
    +import org.apache.hadoop.hive.ql.exec.vector.BytesColumnVector;
    +
    +// cast string group to string (varchar to string, etc.)
    +public class CastStringGroupToString extends StringUnaryUDFDirect {
    +
    + private static final long serialVersionUID = 1L;
    +
    + public CastStringGroupToString() {
    + super();
    + }
    +
    + public CastStringGroupToString(int inputColumn, int outputColumn) {
    + super(inputColumn, outputColumn);
    + }
    +
    + @Override
    + protected void func(BytesColumnVector outV, byte[][] vector, int[] start, int[] length, int i) {
    + outV.setVal(i, vector[i], start[i], length[i]);
    + }
    +}

    http://git-wip-us.apache.org/repos/asf/hive/blob/2de6fda0/ql/src/test/queries/clientpositive/vectorized_casts.q
    ----------------------------------------------------------------------
    diff --git a/ql/src/test/queries/clientpositive/vectorized_casts.q b/ql/src/test/queries/clientpositive/vectorized_casts.q
    index ade3645..8e0bdbc 100644
    --- a/ql/src/test/queries/clientpositive/vectorized_casts.q
    +++ b/ql/src/test/queries/clientpositive/vectorized_casts.q
    @@ -1,8 +1,10 @@
      set hive.explain.user=false;
    +set hive.fetch.task.conversion=none;
      SET hive.vectorized.execution.enabled = true;

      -- SORT_QUERY_RESULTS

    +-- Currently, vectorization is not supported in fetch task (hive.fetch.task.conversion=none)
      -- Test type casting in vectorized mode to verify end-to-end functionality.

      explain
    @@ -68,6 +70,8 @@ select
        ,cast (cbigint * 0 as string)
        ,cast (ctimestamp1 as string)
        ,cast (cstring1 as string)
    + ,cast (cast (cstring1 as char(10)) as string)
    + ,cast (cast (cstring1 as varchar(10)) as string)
      -- nested and expression arguments
        ,cast (cast (cfloat as int) as float)
        ,cast (cint * 2 as double)
    @@ -140,6 +144,8 @@ select
        ,cast (cbigint * 0 as string)
        ,cast (ctimestamp1 as string)
        ,cast (cstring1 as string)
    + ,cast (cast (cstring1 as char(10)) as string)
    + ,cast (cast (cstring1 as varchar(10)) as string)
      -- nested and expression arguments
        ,cast (cast (cfloat as int) as float)
        ,cast (cint * 2 as double)

    http://git-wip-us.apache.org/repos/asf/hive/blob/2de6fda0/ql/src/test/results/clientpositive/tez/vector_char_mapjoin1.q.out
    ----------------------------------------------------------------------
    diff --git a/ql/src/test/results/clientpositive/tez/vector_char_mapjoin1.q.out b/ql/src/test/results/clientpositive/tez/vector_char_mapjoin1.q.out
    index f495f95..cdb24cc 100644
    --- a/ql/src/test/results/clientpositive/tez/vector_char_mapjoin1.q.out
    +++ b/ql/src/test/results/clientpositive/tez/vector_char_mapjoin1.q.out
    @@ -366,6 +366,7 @@ STAGE PLANS:
                                sort order: +
                                Statistics: Num rows: 2 Data size: 215 Basic stats: COMPLETE Column stats: NONE
                                value expressions: _col1 (type: char(10)), _col2 (type: int), _col3 (type: string)
    + Execution mode: vectorized
              Map 3
                  Map Operator Tree:
                      TableScan

    http://git-wip-us.apache.org/repos/asf/hive/blob/2de6fda0/ql/src/test/results/clientpositive/tez/vector_varchar_mapjoin1.q.out
    ----------------------------------------------------------------------
    diff --git a/ql/src/test/results/clientpositive/tez/vector_varchar_mapjoin1.q.out b/ql/src/test/results/clientpositive/tez/vector_varchar_mapjoin1.q.out
    index ae5be44..b453b0e 100644
    --- a/ql/src/test/results/clientpositive/tez/vector_varchar_mapjoin1.q.out
    +++ b/ql/src/test/results/clientpositive/tez/vector_varchar_mapjoin1.q.out
    @@ -354,6 +354,7 @@ STAGE PLANS:
                                sort order: +
                                Statistics: Num rows: 2 Data size: 200 Basic stats: COMPLETE Column stats: NONE
                                value expressions: _col1 (type: varchar(10)), _col2 (type: int), _col3 (type: string)
    + Execution mode: vectorized
              Map 3
                  Map Operator Tree:
                      TableScan

    http://git-wip-us.apache.org/repos/asf/hive/blob/2de6fda0/ql/src/test/results/clientpositive/tez/vectorized_casts.q.out
    ----------------------------------------------------------------------
    diff --git a/ql/src/test/results/clientpositive/tez/vectorized_casts.q.out b/ql/src/test/results/clientpositive/tez/vectorized_casts.q.out
    index 1dd07b1..19c41ed 100644
    --- a/ql/src/test/results/clientpositive/tez/vectorized_casts.q.out
    +++ b/ql/src/test/results/clientpositive/tez/vectorized_casts.q.out
    @@ -1,5 +1,6 @@
      PREHOOK: query: -- SORT_QUERY_RESULTS

    +-- Currently, vectorization is not supported in fetch task (hive.fetch.task.conversion=none)
      -- Test type casting in vectorized mode to verify end-to-end functionality.

      explain
    @@ -65,6 +66,8 @@ select
        ,cast (cbigint * 0 as string)
        ,cast (ctimestamp1 as string)
        ,cast (cstring1 as string)
    + ,cast (cast (cstring1 as char(10)) as string)
    + ,cast (cast (cstring1 as varchar(10)) as string)
      -- nested and expression arguments
        ,cast (cast (cfloat as int) as float)
        ,cast (cint * 2 as double)
    @@ -76,6 +79,7 @@ where cbigint % 250 = 0
      PREHOOK: type: QUERY
      POSTHOOK: query: -- SORT_QUERY_RESULTS

    +-- Currently, vectorization is not supported in fetch task (hive.fetch.task.conversion=none)
      -- Test type casting in vectorized mode to verify end-to-end functionality.

      explain
    @@ -141,6 +145,8 @@ select
        ,cast (cbigint * 0 as string)
        ,cast (ctimestamp1 as string)
        ,cast (cstring1 as string)
    + ,cast (cast (cstring1 as char(10)) as string)
    + ,cast (cast (cstring1 as varchar(10)) as string)
      -- nested and expression arguments
        ,cast (cast (cfloat as int) as float)
        ,cast (cint * 2 as double)
    @@ -151,21 +157,40 @@ from alltypesorc
      where cbigint % 250 = 0
      POSTHOOK: type: QUERY
      STAGE DEPENDENCIES:
    - Stage-0 is a root stage
    + Stage-1 is a root stage
    + Stage-0 depends on stages: Stage-1

      STAGE PLANS:
    + Stage: Stage-1
    + Tez
    +#### A masked pattern was here ####
    + Vertices:
    + Map 1
    + Map Operator Tree:
    + TableScan
    + alias: alltypesorc
    + Statistics: Num rows: 12288 Data size: 2641964 Basic stats: COMPLETE Column stats: NONE
    + Filter Operator
    + predicate: ((cbigint % 250) = 0) (type: boolean)
    + Statistics: Num rows: 6144 Data size: 1320982 Basic stats: COMPLETE Column stats: NONE
    + Select Operator
    + expressions: UDFToBoolean(ctinyint) (type: boolean), UDFToBoolean(csmallint) (type: boolean), UDFToBoolean(cint) (type: boolean), UDFToBoolean(cbigint) (type: boolean), UDFToBoolean(cfloat) (type: boolean), UDFToBoolean(cdouble) (type: boolean), cboolean1 (type: boolean), UDFToBoolean((cbigint * 0)) (type: boolean), UDFToBoolean(ctimestamp1) (type: boolean), UDFToBoolean(cstring1) (type: boolean), UDFToInteger(ctinyint) (type: int), UDFToInteger(csmallint) (type: int), cint (type: int), UDFToInteger(cbigint) (type: int), UDFToInteger(cfloat) (type: int), UDFToInteger(cdouble) (type: int), UDFToInteger(cboolean1) (type: int), UDFToInteger(ctimestamp1) (type: int), UDFToInteger(cstring1) (type: int), UDFToInteger(substr(cstring1, 1, 1)) (type: int), UDFToByte(cfloat) (type: tinyint), UDFToShort(cfloat) (type: smallint), UDFToLong(cfloat) (type: bigint), UDFToDouble(ctinyint) (type: double), UDFToDouble(csmallint) (type: double), UDFToDouble(cint) (type: double),
      UDFToDouble(cbigint) (type: double), UDFToDouble(cfloat) (type: double), cdouble (type: double), UDFToDouble(cboolean1) (type: double), UDFToDouble(ctimestamp1) (type: double), UDFToDouble(cstring1) (type: double), UDFToDouble(substr(cstring1, 1, 1)) (type: double), UDFToFloat(cint) (type: float), UDFToFloat(cdouble) (type: float), CAST( ctinyint AS TIMESTAMP) (type: timestamp), CAST( csmallint AS TIMESTAMP) (type: timestamp), CAST( cint AS TIMESTAMP) (type: timestamp), CAST( cbigint AS TIMESTAMP) (type: timestamp), CAST( cfloat AS TIMESTAMP) (type: timestamp), CAST( cdouble AS TIMESTAMP) (type: timestamp), CAST( cboolean1 AS TIMESTAMP) (type: timestamp), CAST( (cbigint * 0) AS TIMESTAMP) (type: timestamp), ctimestamp1 (type: timestamp), CAST( cstring1 AS TIMESTAMP) (type: timestamp), CAST( substr(cstring1, 1, 1) AS TIMESTAMP) (type: timestamp), UDFToString(ctinyint) (type: string), UDFToString(csmallint) (type: string), UDFToString(cint) (type: string), UDFToString(cbigint) (type:
      string), UDFToString(cfloat) (type: string), UDFToString(cdouble) (type: string), UDFToString(cboolean1) (type: string), UDFToString((cbigint * 0)) (type: string), UDFToString(ctimestamp1) (type: string), cstring1 (type: string), UDFToString(CAST( cstring1 AS CHAR(10)) (type: string), UDFToString(CAST( cstring1 AS varchar(10))) (type: string), UDFToFloat(UDFToInteger(cfloat)) (type: float), UDFToDouble((cint * 2)) (type: double), UDFToString(sin(cfloat)) (type: string), (UDFToDouble(UDFToFloat(cint)) + UDFToDouble(cboolean1)) (type: double)
    + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22, _col23, _col24, _col25, _col26, _col27, _col28, _col29, _col30, _col31, _col32, _col33, _col34, _col35, _col36, _col37, _col38, _col39, _col40, _col41, _col42, _col43, _col44, _col45, _col46, _col47, _col48, _col49, _col50, _col51, _col52, _col53, _col54, _col55, _col56, _col57, _col58, _col59, _col60, _col61
    + Statistics: Num rows: 6144 Data size: 1320982 Basic stats: COMPLETE Column stats: NONE
    + File Output Operator
    + compressed: false
    + Statistics: Num rows: 6144 Data size: 1320982 Basic stats: COMPLETE Column stats: NONE
    + table:
    + input format: org.apache.hadoop.mapred.TextInputFormat
    + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
    + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
    + Execution mode: vectorized
    +
        Stage: Stage-0
          Fetch Operator
            limit: -1
            Processor Tree:
    - TableScan
    - alias: alltypesorc
    - Filter Operator
    - predicate: ((cbigint % 250) = 0) (type: boolean)
    - Select Operator
    - expressions: UDFToBoolean(ctinyint) (type: boolean), UDFToBoolean(csmallint) (type: boolean), UDFToBoolean(cint) (type: boolean), UDFToBoolean(cbigint) (type: boolean), UDFToBoolean(cfloat) (type: boolean), UDFToBoolean(cdouble) (type: boolean), cboolean1 (type: boolean), UDFToBoolean((cbigint * 0)) (type: boolean), UDFToBoolean(ctimestamp1) (type: boolean), UDFToBoolean(cstring1) (type: boolean), UDFToInteger(ctinyint) (type: int), UDFToInteger(csmallint) (type: int), cint (type: int), UDFToInteger(cbigint) (type: int), UDFToInteger(cfloat) (type: int), UDFToInteger(cdouble) (type: int), UDFToInteger(cboolean1) (type: int), UDFToInteger(ctimestamp1) (type: int), UDFToInteger(cstring1) (type: int), UDFToInteger(substr(cstring1, 1, 1)) (type: int), UDFToByte(cfloat) (type: tinyint), UDFToShort(cfloat) (type: smallint), UDFToLong(cfloat) (type: bigint), UDFToDouble(ctinyint) (type: double), UDFToDouble(csmallint) (type: double), UDFToDouble(cint) (type: double), UDFToDou
      ble(cbigint) (type: double), UDFToDouble(cfloat) (type: double), cdouble (type: double), UDFToDouble(cboolean1) (type: double), UDFToDouble(ctimestamp1) (type: double), UDFToDouble(cstring1) (type: double), UDFToDouble(substr(cstring1, 1, 1)) (type: double), UDFToFloat(cint) (type: float), UDFToFloat(cdouble) (type: float), CAST( ctinyint AS TIMESTAMP) (type: timestamp), CAST( csmallint AS TIMESTAMP) (type: timestamp), CAST( cint AS TIMESTAMP) (type: timestamp), CAST( cbigint AS TIMESTAMP) (type: timestamp), CAST( cfloat AS TIMESTAMP) (type: timestamp), CAST( cdouble AS TIMESTAMP) (type: timestamp), CAST( cboolean1 AS TIMESTAMP) (type: timestamp), CAST( (cbigint * 0) AS TIMESTAMP) (type: timestamp), ctimestamp1 (type: timestamp), CAST( cstring1 AS TIMESTAMP) (type: timestamp), CAST( substr(cstring1, 1, 1) AS TIMESTAMP) (type: timestamp), UDFToString(ctinyint) (type: string), UDFToString(csmallint) (type: string), UDFToString(cint) (type: string), UDFToString(cbigint) (type: string),
       UDFToString(cfloat) (type: string), UDFToString(cdouble) (type: string), UDFToString(cboolean1) (type: string), UDFToString((cbigint * 0)) (type: string), UDFToString(ctimestamp1) (type: string), cstring1 (type: string), UDFToFloat(UDFToInteger(cfloat)) (type: float), UDFToDouble((cint * 2)) (type: double), UDFToString(sin(cfloat)) (type: string), (UDFToDouble(UDFToFloat(cint)) + UDFToDouble(cboolean1)) (type: double)
    - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22, _col23, _col24, _col25, _col26, _col27, _col28, _col29, _col30, _col31, _col32, _col33, _col34, _col35, _col36, _col37, _col38, _col39, _col40, _col41, _col42, _col43, _col44, _col45, _col46, _col47, _col48, _col49, _col50, _col51, _col52, _col53, _col54, _col55, _col56, _col57, _col58, _col59
    - ListSink
    + ListSink

      PREHOOK: query: select
      -- to boolean
    @@ -229,6 +254,8 @@ PREHOOK: query: select
        ,cast (cbigint * 0 as string)
        ,cast (ctimestamp1 as string)
        ,cast (cstring1 as string)
    + ,cast (cast (cstring1 as char(10)) as string)
    + ,cast (cast (cstring1 as varchar(10)) as string)
      -- nested and expression arguments
        ,cast (cast (cfloat as int) as float)
        ,cast (cint * 2 as double)
    @@ -302,6 +329,8 @@ POSTHOOK: query: select
        ,cast (cbigint * 0 as string)
        ,cast (ctimestamp1 as string)
        ,cast (cstring1 as string)
    + ,cast (cast (cstring1 as char(10)) as string)
    + ,cast (cast (cstring1 as varchar(10)) as string)
      -- nested and expression arguments
        ,cast (cast (cfloat as int) as float)
        ,cast (cint * 2 as double)
    @@ -313,29 +342,29 @@ where cbigint % 250 = 0
      POSTHOOK: type: QUERY
      POSTHOOK: Input: default@alltypesorc
      #### A masked pattern was here ####
    -true NULL true true true NULL false false true true -51 NULL 773600971 1053923250 -51 NULL 0 8 NULL 2 -51 -51 -51 -51.0 NULL 7.73600971E8 1.05392325E9 -51.0 NULL 0.0 8.451 NULL 2.0 7.7360096E8 NULL 1969-12-31 15:59:59.949 NULL 1970-01-09 14:53:20.971 1970-01-12 20:45:23.25 1969-12-31 15:59:09 NULL 1969-12-31 16:00:00 1969-12-31 16:00:00 1969-12-31 16:00:08.451 NULL NULL -51 NULL 773600971 1053923250 -51.0 NULL FALSE 0 1969-12-31 16:00:08.451 2yK4Bx76O -51.0 1.547201942E9 -0.6702291758433747 7.7360096E8
    -true NULL true true true NULL false false true true 8 NULL -102936434 -1312782750 8 NULL 0 15 NULL NULL 8 8 8 8.0 NULL -1.02936434E8 -1.31278275E9 8.0 NULL 0.0 15.892 NULL NULL -1.02936432E8 NULL 1969-12-31 16:00:00.008 NULL 1969-12-30 11:24:23.566 1969-12-16 11:20:17.25 1969-12-31 16:00:08 NULL 1969-12-31 16:00:00 1969-12-31 16:00:00 1969-12-31 16:00:15.892 NULL NULL 8 NULL -102936434 -1312782750 8.0 NULL FALSE 0 1969-12-31 16:00:15.892 eJROSNhugc3kQR7Pb 8.0 -2.05872868E8 0.9893582466233818 -1.02936432E8
    -true NULL true true true NULL false false true true 8 NULL -661621138 -931392750 8 NULL 0 15 NULL NULL 8 8 8 8.0 NULL -6.61621138E8 -9.3139275E8 8.0 NULL 0.0 15.892 NULL NULL -6.6162112E8 NULL 1969-12-31 16:00:00.008 NULL 1969-12-24 00:12:58.862 1969-12-20 21:16:47.25 1969-12-31 16:00:08 NULL 1969-12-31 16:00:00 1969-12-31 16:00:00 1969-12-31 16:00:15.892 NULL NULL 8 NULL -661621138 -931392750 8.0 NULL FALSE 0 1969-12-31 16:00:15.892 L15l8i5k558tBcDV20 8.0 -1.323242276E9 0.9893582466233818 -6.6162112E8
    -true NULL true true true NULL false false true true 8 NULL -669632311 1588591250 8 NULL 0 15 NULL 3 8 8 8 8.0 NULL -6.69632311E8 1.58859125E9 8.0 NULL 0.0 15.892 NULL 3.0 -6.6963232E8 NULL 1969-12-31 16:00:00.008 NULL 1969-12-23 21:59:27.689 1970-01-19 01:16:31.25 1969-12-31 16:00:08 NULL 1969-12-31 16:00:00 1969-12-31 16:00:00 1969-12-31 16:00:15.892 NULL NULL 8 NULL -669632311 1588591250 8.0 NULL FALSE 0 1969-12-31 16:00:15.892 3r3sDvfUkG0yTP3LnX5mNQRr 8.0 -1.339264622E9 0.9893582466233818 -6.6963232E8
    -true NULL true true true NULL false false true true 8 NULL 805179664 868161500 8 NULL 0 15 NULL NULL 8 8 8 8.0 NULL 8.05179664E8 8.681615E8 8.0 NULL 0.0 15.892 NULL NULL 8.0517965E8 NULL 1969-12-31 16:00:00.008 NULL 1970-01-09 23:39:39.664 1970-01-10 17:09:21.5 1969-12-31 16:00:08 NULL 1969-12-31 16:00:00 1969-12-31 16:00:00 1969-12-31 16:00:15.892 NULL NULL 8 NULL 805179664 868161500 8.0 NULL FALSE 0 1969-12-31 16:00:15.892 e005B5q 8.0 1.610359328E9 0.9893582466233818 8.05179648E8
    -true NULL true true true NULL true false true true -51 NULL 747553882 -1930467250 -51 NULL 1 8 NULL NULL -51 -51 -51 -51.0 NULL 7.47553882E8 -1.93046725E9 -51.0 NULL 1.0 8.451 NULL NULL 7.4755386E8 NULL 1969-12-31 15:59:59.949 NULL 1970-01-09 07:39:13.882 1969-12-09 07:45:32.75 1969-12-31 15:59:09 NULL 1969-12-31 16:00:00.001 1969-12-31 16:00:00 1969-12-31 16:00:08.451 NULL NULL -51 NULL 747553882 -1930467250 -51.0 NULL TRUE 0 1969-12-31 16:00:08.451 q8M86Fx0r -51.0 1.495107764E9 -0.6702291758433747 7.47553857E8
    -true NULL true true true NULL true false true true 11 NULL -335450417 1233327000 11 NULL 1 2 NULL NULL 11 11 11 11.0 NULL -3.35450417E8 1.233327E9 11.0 NULL 1.0 2.351 NULL NULL -3.35450432E8 NULL 1969-12-31 16:00:00.011 NULL 1969-12-27 18:49:09.583 1970-01-14 22:35:27 1969-12-31 16:00:11 NULL 1969-12-31 16:00:00.001 1969-12-31 16:00:00 1969-12-31 16:00:02.351 NULL NULL 11 NULL -335450417 1233327000 11.0 NULL TRUE 0 1969-12-31 16:00:02.351 dOYnqgaXoJ1P3ERwxe5N7 11.0 -6.70900834E8 -0.9999902065507035 -3.35450431E8
    -true NULL true true true NULL true false true true 11 NULL -64615982 1803053750 11 NULL 1 2 NULL 8 11 11 11 11.0 NULL -6.4615982E7 1.80305375E9 11.0 NULL 1.0 2.351 NULL 8.0 -6.4615984E7 NULL 1969-12-31 16:00:00.011 NULL 1969-12-30 22:03:04.018 1970-01-21 12:50:53.75 1969-12-31 16:00:11 NULL 1969-12-31 16:00:00.001 1969-12-31 16:00:00 1969-12-31 16:00:02.351 NULL NULL 11 NULL -64615982 1803053750 11.0 NULL TRUE 0 1969-12-31 16:00:02.351 8J5OB7K26PEV7kdbeHr3 11.0 -1.29231964E8 -0.9999902065507035 -6.4615983E7
    -true NULL true true true NULL true false true true 8 NULL 890988972 -1862301000 8 NULL 1 15 NULL NULL 8 8 8 8.0 NULL 8.90988972E8 -1.862301E9 8.0 NULL 1.0 15.892 NULL NULL 8.9098899E8 NULL 1969-12-31 16:00:00.008 NULL 1970-01-10 23:29:48.972 1969-12-10 02:41:39 1969-12-31 16:00:08 NULL 1969-12-31 16:00:00.001 1969-12-31 16:00:00 1969-12-31 16:00:15.892 NULL NULL 8 NULL 890988972 -1862301000 8.0 NULL TRUE 0 1969-12-31 16:00:15.892 XylAH4 8.0 1.781977944E9 0.9893582466233818 8.90988993E8
    -true NULL true true true NULL true false true true 8 NULL 930867246 1205399250 8 NULL 1 15 NULL NULL 8 8 8 8.0 NULL 9.30867246E8 1.20539925E9 8.0 NULL 1.0 15.892 NULL NULL 9.3086726E8 NULL 1969-12-31 16:00:00.008 NULL 1970-01-11 10:34:27.246 1970-01-14 14:49:59.25 1969-12-31 16:00:08 NULL 1969-12-31 16:00:00.001 1969-12-31 16:00:00 1969-12-31 16:00:15.892 NULL NULL 8 NULL 930867246 1205399250 8.0 NULL TRUE 0 1969-12-31 16:00:15.892 c1V8o1A 8.0 1.861734492E9 0.9893582466233818 9.30867265E8
    -true true NULL true true true NULL false true NULL -14 -7196 NULL -1552199500 -14 -7196 NULL 11 NULL NULL -14 -14 -14 -14.0 -7196.0 NULL -1.5521995E9 -14.0 -7196.0 NULL 11.065 NULL NULL NULL -7196.0 1969-12-31 15:59:59.986 1969-12-31 15:59:52.804 NULL 1969-12-13 16:50:00.5 1969-12-31 15:59:46 1969-12-31 14:00:04 NULL 1969-12-31 16:00:00 1969-12-31 16:00:11.065 NULL NULL -14 -7196 NULL -1552199500 -14.0 -7196.0 NULL 0 1969-12-31 16:00:11.065 NULL -14.0 NULL -0.9906073556948704 NULL
    -true true NULL true true true NULL false true NULL -21 -7196 NULL 1542429000 -21 -7196 NULL -5 NULL NULL -21 -21 -21 -21.0 -7196.0 NULL 1.542429E9 -21.0 -7196.0 NULL -4.1 NULL NULL NULL -7196.0 1969-12-31 15:59:59.979 1969-12-31 15:59:52.804 NULL 1970-01-18 12:27:09 1969-12-31 15:59:39 1969-12-31 14:00:04 NULL 1969-12-31 16:00:00 1969-12-31 15:59:55.9 NULL NULL -21 -7196 NULL 1542429000 -21.0 -7196.0 NULL 0 1969-12-31 15:59:55.9 NULL -21.0 NULL -0.8366556385360561 NULL
    -true true NULL true true true NULL false true NULL -24 -7196 NULL 829111000 -24 -7196 NULL -7 NULL NULL -24 -24 -24 -24.0 -7196.0 NULL 8.29111E8 -24.0 -7196.0 NULL -6.855 NULL NULL NULL -7196.0 1969-12-31 15:59:59.976 1969-12-31 15:59:52.804 NULL 1970-01-10 06:18:31 1969-12-31 15:59:36 1969-12-31 14:00:04 NULL 1969-12-31 16:00:00 1969-12-31 15:59:53.145 NULL NULL -24 -7196 NULL 829111000 -24.0 -7196.0 NULL 0 1969-12-31 15:59:53.145 NULL -24.0 NULL 0.9055783620066238 NULL
    -true true NULL true true true NULL false true NULL -30 -200 NULL 1429852250 -30 -200 NULL 12 NULL NULL -30 -30 -30 -30.0 -200.0 NULL 1.42985225E9 -30.0 -200.0 NULL 12.935 NULL NULL NULL -200.0 1969-12-31 15:59:59.97 1969-12-31 15:59:59.8 NULL 1970-01-17 05:10:52.25 1969-12-31 15:59:30 1969-12-31 15:56:40 NULL 1969-12-31 16:00:00 1969-12-31 16:00:12.935 NULL NULL -30 -200 NULL 1429852250 -30.0 -200.0 NULL 0 1969-12-31 16:00:12.935 NULL -30.0 NULL 0.9880316240928618 NULL
    -true true NULL true true true NULL false true NULL -36 -200 NULL -2006216750 -36 -200 NULL -15 NULL NULL -36 -36 -36 -36.0 -200.0 NULL -2.00621675E9 -36.0 -200.0 NULL -14.252 NULL NULL NULL -200.0 1969-12-31 15:59:59.964 1969-12-31 15:59:59.8 NULL 1969-12-08 10:43:03.25 1969-12-31 15:59:24 1969-12-31 15:56:40 NULL 1969-12-31 16:00:00 1969-12-31 15:59:45.748 NULL NULL -36 -200 NULL -2006216750 -36.0 -200.0 NULL 0 1969-12-31 15:59:45.748 NULL -36.0 NULL 0.9917788534431158 NULL
    -true true NULL true true true NULL false true NULL -36 -200 NULL 1599879000 -36 -200 NULL -7 NULL NULL -36 -36 -36 -36.0 -200.0 NULL 1.599879E9 -36.0 -200.0 NULL -6.183 NULL NULL NULL -200.0 1969-12-31 15:59:59.964 1969-12-31 15:59:59.8 NULL 1970-01-19 04:24:39 1969-12-31 15:59:24 1969-12-31 15:56:40 NULL 1969-12-31 16:00:00 1969-12-31 15:59:53.817 NULL NULL -36 -200 NULL 1599879000 -36.0 -200.0 NULL 0 1969-12-31 15:59:53.817 NULL -36.0 NULL 0.9917788534431158 NULL
    -true true NULL true true true NULL false true NULL -38 15601 NULL -1858689000 -38 15601 NULL -2 NULL NULL -38 -38 -38 -38.0 15601.0 NULL -1.858689E9 -38.0 15601.0 NULL -1.3860000000000001 NULL NULL NULL 15601.0 1969-12-31 15:59:59.962 1969-12-31 16:00:15.601 NULL 1969-12-10 03:41:51 1969-12-31 15:59:22 1969-12-31 20:20:01 NULL 1969-12-31 16:00:00 1969-12-31 15:59:58.614 NULL NULL -38 15601 NULL -1858689000 -38.0 15601.0 NULL 0 1969-12-31 15:59:58.614 NULL -38.0 NULL -0.2963685787093853 NULL
    -true true NULL true true true NULL false true NULL -5 15601 NULL 612416000 -5 15601 NULL 4 NULL NULL -5 -5 -5 -5.0 15601.0 NULL 6.12416E8 -5.0 15601.0 NULL 4.679 NULL NULL NULL 15601.0 1969-12-31 15:59:59.995 1969-12-31 16:00:15.601 NULL 1970-01-07 18:06:56 1969-12-31 15:59:55 1969-12-31 20:20:01 NULL 1969-12-31 16:00:00 1969-12-31 16:00:04.679 NULL NULL -5 15601 NULL 612416000 -5.0 15601.0 NULL 0 1969-12-31 16:00:04.679 NULL -5.0 NULL 0.9589242746631385 NULL
    -true true NULL true true true NULL false true NULL -50 -7196 NULL -1031187250 -50 -7196 NULL -6 NULL NULL -50 -50 -50 -50.0 -7196.0 NULL -1.03118725E9 -50.0 -7196.0 NULL -5.267 NULL NULL NULL -7196.0 1969-12-31 15:59:59.95 1969-12-31 15:59:52.804 NULL 1969-12-19 17:33:32.75 1969-12-31 15:59:10 1969-12-31 14:00:04 NULL 1969-12-31 16:00:00 1969-12-31 15:59:54.733 NULL NULL -50 -7196 NULL -1031187250 -50.0 -7196.0 NULL 0 1969-12-31 15:59:54.733 NULL -50.0 NULL 0.26237485370392877 NULL
    -true true NULL true true true NULL false true NULL -59 -7196 NULL -1604890000 -59 -7196 NULL 13 NULL NULL -59 -59 -59 -59.0 -7196.0 NULL -1.60489E9 -59.0 -7196.0 NULL 13.15 NULL NULL NULL -7196.0 1969-12-31 15:59:59.941 1969-12-31 15:59:52.804 NULL 1969-12-13 02:11:50 1969-12-31 15:59:01 1969-12-31 14:00:04 NULL 1969-12-31 16:00:00 1969-12-31 16:00:13.15 NULL NULL -59 -7196 NULL -1604890000 -59.0 -7196.0 NULL 0 1969-12-31 16:00:13.15 NULL -59.0 NULL -0.6367380071391379 NULL
    -true true NULL true true true NULL false true NULL -60 -7196 NULL 1516314750 -60 -7196 NULL -8 NULL NULL -60 -60 -60 -60.0 -7196.0 NULL 1.51631475E9 -60.0 -7196.0 NULL -7.592 NULL NULL NULL -7196.0 1969-12-31 15:59:59.94 1969-12-31 15:59:52.804 NULL 1970-01-18 05:11:54.75 1969-12-31 15:59:00 1969-12-31 14:00:04 NULL 1969-12-31 16:00:00 1969-12-31 15:59:52.408 NULL NULL -60 -7196 NULL 1516314750 -60.0 -7196.0 NULL 0 1969-12-31 15:59:52.408 NULL -60.0 NULL 0.3048106211022167 NULL
    -true true NULL true true true NULL false true NULL -8 -7196 NULL -1849991500 -8 -7196 NULL 3 NULL NULL -8 -8 -8 -8.0 -7196.0 NULL -1.8499915E9 -8.0 -7196.0 NULL 3.136 NULL NULL NULL -7196.0 1969-12-31 15:59:59.992 1969-12-31 15:59:52.804 NULL 1969-12-10 06:06:48.5 1969-12-31 15:59:52 1969-12-31 14:00:04 NULL 1969-12-31 16:00:00 1969-12-31 16:00:03.136 NULL NULL -8 -7196 NULL -1849991500 -8.0 -7196.0 NULL 0 1969-12-31 16:00:03.136 NULL -8.0 NULL -0.9893582466233818 NULL
    -true true NULL true true true NULL false true NULL 20 15601 NULL -362433250 20 15601 NULL -15 NULL NULL 20 20 20 20.0 15601.0 NULL -3.6243325E8 20.0 15601.0 NULL -14.871 NULL NULL NULL 15601.0 1969-12-31 16:00:00.02 1969-12-31 16:00:15.601 NULL 1969-12-27 11:19:26.75 1969-12-31 16:00:20 1969-12-31 20:20:01 NULL 1969-12-31 16:00:00 1969-12-31 15:59:45.129 NULL NULL 20 15601 NULL -362433250 20.0 15601.0 NULL 0 1969-12-31 15:59:45.129 NULL 20.0 NULL 0.9129452507276277 NULL
    -true true NULL true true true NULL false true NULL 48 15601 NULL -795361000 48 15601 NULL -10 NULL NULL 48 48 48 48.0 15601.0 NULL -7.95361E8 48.0 15601.0 NULL -9.765 NULL NULL NULL 15601.0 1969-12-31 16:00:00.048 1969-12-31 16:00:15.601 NULL 1969-12-22 11:03:59 1969-12-31 16:00:48 1969-12-31 20:20:01 NULL 1969-12-31 16:00:00 1969-12-31 15:59:50.235 NULL NULL 48 15601 NULL -795361000 48.0 15601.0 NULL 0 1969-12-31 15:59:50.235 NULL 48.0 NULL -0.7682546613236668 NULL
    -true true NULL true true true NULL false true NULL 5 -7196 NULL -1015607500 5 -7196 NULL 10 NULL NULL 5 5 5 5.0 -7196.0 NULL -1.0156075E9 5.0 -7196.0 NULL 10.973 NULL NULL NULL -7196.0 1969-12-31 16:00:00.005 1969-12-31 15:59:52.804 NULL 1969-12-19 21:53:12.5 1969-12-31 16:00:05 1969-12-31 14:00:04 NULL 1969-12-31 16:00:00 1969-12-31 16:00:10.973 NULL NULL 5 -7196 NULL -1015607500 5.0 -7196.0 NULL 0 1969-12-31 16:00:10.973 NULL 5.0 NULL -0.9589242746631385 NULL
    -true true NULL true true true NULL false true NULL 59 -7196 NULL -1137754500 59 -7196 NULL 10 NULL NULL 59 59 59 59.0 -7196.0 NULL -1.1377545E9 59.0 -7196.0 NULL 10.956 NULL NULL NULL -7196.0 1969-12-31 16:00:00.059 1969-12-31 15:59:52.804 NULL 1969-12-18 11:57:25.5 1969-12-31 16:00:59 1969-12-31 14:00:04 NULL 1969-12-31 16:00:00 1969-12-31 16:00:10.956 NULL NULL 59 -7196 NULL -1137754500 59.0 -7196.0 NULL 0 1969-12-31 16:00:10.956 NULL 59.0 NULL 0.6367380071391379 NULL
    +true NULL true true true NULL false false true true -51 NULL 773600971 1053923250 -51 NULL 0 8 NULL 2 -51 -51 -51 -51.0 NULL 7.73600971E8 1.05392325E9 -51.0 NULL 0.0 8.451 NULL 2.0 7.7360096E8 NULL 1969-12-31 15:59:59.949 NULL 1970-01-09 14:53:20.971 1970-01-12 20:45:23.25 1969-12-31 15:59:09 NULL 1969-12-31 16:00:00 1969-12-31 16:00:00 1969-12-31 16:00:08.451 NULL NULL -51 NULL 773600971 1053923250 -51.0 NULL FALSE 0 1969-12-31 16:00:08.451 2yK4Bx76O 2yK4Bx76O 2yK4Bx76O -51.0 1.547201942E9 -0.6702291758433747 7.73600971E8
    +true NULL true true true NULL false false true true 8 NULL -102936434 -1312782750 8 NULL 0 15 NULL NULL 8 8 8 8.0 NULL -1.02936434E8 -1.31278275E9 8.0 NULL 0.0 15.892 NULL NULL -1.02936432E8 NULL 1969-12-31 16:00:00.008 NULL 1969-12-30 11:24:23.566 1969-12-16 11:20:17.25 1969-12-31 16:00:08 NULL 1969-12-31 16:00:00 1969-12-31 16:00:00 1969-12-31 16:00:15.892 NULL NULL 8 NULL -102936434 -1312782750 8.0 NULL FALSE 0 1969-12-31 16:00:15.892 eJROSNhugc3kQR7Pb eJROSNhugc eJROSNhugc 8.0 -2.05872868E8 0.9893582466233818 -1.02936434E8
    +true NULL true true true NULL false false true true 8 NULL -661621138 -931392750 8 NULL 0 15 NULL NULL 8 8 8 8.0 NULL -6.61621138E8 -9.3139275E8 8.0 NULL 0.0 15.892 NULL NULL -6.6162112E8 NULL 1969-12-31 16:00:00.008 NULL 1969-12-24 00:12:58.862 1969-12-20 21:16:47.25 1969-12-31 16:00:08 NULL 1969-12-31 16:00:00 1969-12-31 16:00:00 1969-12-31 16:00:15.892 NULL NULL 8 NULL -661621138 -931392750 8.0 NULL FALSE 0 1969-12-31 16:00:15.892 L15l8i5k558tBcDV20 L15l8i5k55 L15l8i5k55 8.0 -1.323242276E9 0.9893582466233818 -6.61621138E8
    +true NULL true true true NULL false false true true 8 NULL -669632311 1588591250 8 NULL 0 15 NULL 3 8 8 8 8.0 NULL -6.69632311E8 1.58859125E9 8.0 NULL 0.0 15.892 NULL 3.0 -6.6963232E8 NULL 1969-12-31 16:00:00.008 NULL 1969-12-23 21:59:27.689 1970-01-19 01:16:31.25 1969-12-31 16:00:08 NULL 1969-12-31 16:00:00 1969-12-31 16:00:00 1969-12-31 16:00:15.892 NULL NULL 8 NULL -669632311 1588591250 8.0 NULL FALSE 0 1969-12-31 16:00:15.892 3r3sDvfUkG0yTP3LnX5mNQRr 3r3sDvfUkG 3r3sDvfUkG 8.0 -1.339264622E9 0.9893582466233818 -6.69632311E8
    +true NULL true true true NULL false false true true 8 NULL 805179664 868161500 8 NULL 0 15 NULL NULL 8 8 8 8.0 NULL 8.05179664E8 8.681615E8 8.0 NULL 0.0 15.892 NULL NULL 8.0517965E8 NULL 1969-12-31 16:00:00.008 NULL 1970-01-09 23:39:39.664 1970-01-10 17:09:21.5 1969-12-31 16:00:08 NULL 1969-12-31 16:00:00 1969-12-31 16:00:00 1969-12-31 16:00:15.892 NULL NULL 8 NULL 805179664 868161500 8.0 NULL FALSE 0 1969-12-31 16:00:15.892 e005B5q e005B5q e005B5q 8.0 1.610359328E9 0.9893582466233818 8.05179664E8
    +true NULL true true true NULL true false true true -51 NULL 747553882 -1930467250 -51 NULL 1 8 NULL NULL -51 -51 -51 -51.0 NULL 7.47553882E8 -1.93046725E9 -51.0 NULL 1.0 8.451 NULL NULL 7.4755386E8 NULL 1969-12-31 15:59:59.949 NULL 1970-01-09 07:39:13.882 1969-12-09 07:45:32.75 1969-12-31 15:59:09 NULL 1969-12-31 16:00:00.001 1969-12-31 16:00:00 1969-12-31 16:00:08.451 NULL NULL -51 NULL 747553882 -1930467250 -51.0 NULL TRUE 0 1969-12-31 16:00:08.451 q8M86Fx0r q8M86Fx0r q8M86Fx0r -51.0 1.495107764E9 -0.6702291758433747 7.47553883E8
    +true NULL true true true NULL true false true true 11 NULL -335450417 1233327000 11 NULL 1 2 NULL NULL 11 11 11 11.0 NULL -3.35450417E8 1.233327E9 11.0 NULL 1.0 2.351 NULL NULL -3.35450432E8 NULL 1969-12-31 16:00:00.011 NULL 1969-12-27 18:49:09.583 1970-01-14 22:35:27 1969-12-31 16:00:11 NULL 1969-12-31 16:00:00.001 1969-12-31 16:00:00 1969-12-31 16:00:02.351 NULL NULL 11 NULL -335450417 1233327000 11.0 NULL TRUE 0 1969-12-31 16:00:02.351 dOYnqgaXoJ1P3ERwxe5N7 dOYnqgaXoJ dOYnqgaXoJ 11.0 -6.70900834E8 -0.9999902065507035 -3.35450416E8
    +true NULL true true true NULL true false true true 11 NULL -64615982 1803053750 11 NULL 1 2 NULL 8 11 11 11 11.0 NULL -6.4615982E7 1.80305375E9 11.0 NULL 1.0 2.351 NULL 8.0 -6.4615984E7 NULL 1969-12-31 16:00:00.011 NULL 1969-12-30 22:03:04.018 1970-01-21 12:50:53.75 1969-12-31 16:00:11 NULL 1969-12-31 16:00:00.001 1969-12-31 16:00:00 1969-12-31 16:00:02.351 NULL NULL 11 NULL -64615982 1803053750 11.0 NULL TRUE 0 1969-12-31 16:00:02.351 8J5OB7K26PEV7kdbeHr3 8J5OB7K26P 8J5OB7K26P 11.0 -1.29231964E8 -0.9999902065507035 -6.4615981E7
    +true NULL true true true NULL true false true true 8 NULL 890988972 -1862301000 8 NULL 1 15 NULL NULL 8 8 8 8.0 NULL 8.90988972E8 -1.862301E9 8.0 NULL 1.0 15.892 NULL NULL 8.9098899E8 NULL 1969-12-31 16:00:00.008 NULL 1970-01-10 23:29:48.972 1969-12-10 02:41:39 1969-12-31 16:00:08 NULL 1969-12-31 16:00:00.001 1969-12-31 16:00:00 1969-12-31 16:00:15.892 NULL NULL 8 NULL 890988972 -1862301000 8.0 NULL TRUE 0 1969-12-31 16:00:15.892 XylAH4 XylAH4 XylAH4 8.0 1.781977944E9 0.9893582466233818 8.90988973E8
    +true NULL true true true NULL true false true true 8 NULL 930867246 1205399250 8 NULL 1 15 NULL NULL 8 8 8 8.0 NULL 9.30867246E8 1.20539925E9 8.0 NULL 1.0 15.892 NULL NULL 9.3086726E8 NULL 1969-12-31 16:00:00.008 NULL 1970-01-11 10:34:27.246 1970-01-14 14:49:59.25 1969-12-31 16:00:08 NULL 1969-12-31 16:00:00.001 1969-12-31 16:00:00 1969-12-31 16:00:15.892 NULL NULL 8 NULL 930867246 1205399250 8.0 NULL TRUE 0 1969-12-31 16:00:15.892 c1V8o1A c1V8o1A c1V8o1A 8.0 1.861734492E9 0.9893582466233818 9.30867247E8
    +true true NULL true true true NULL false true NULL -14 -7196 NULL -1552199500 -14 -7196 NULL 11 NULL NULL -14 -14 -14 -14.0 -7196.0 NULL -1.5521995E9 -14.0 -7196.0 NULL 11.065 NULL NULL NULL -7196.0 1969-12-31 15:59:59.986 1969-12-31 15:59:52.804 NULL 1969-12-13 16:50:00.5 1969-12-31 15:59:46 1969-12-31 14:00:04 NULL 1969-12-31 16:00:00 1969-12-31 16:00:11.065 NULL NULL -14 -7196 NULL -1552199500 -14.0 -7196.0 NULL 0 1969-12-31 16:00:11.065 NULL NULL NULL -14.0 NULL -0.9906073556948704 NULL
    +true true NULL true true true NULL false true NULL -21 -7196 NULL 1542429000 -21 -7196 NULL -4 NULL NULL -21 -21 -21 -21.0 -7196.0 NULL 1.542429E9 -21.0 -7196.0 NULL -4.1 NULL NULL NULL -7196.0 1969-12-31 15:59:59.979 1969-12-31 15:59:52.804 NULL 1970-01-18 12:27:09 1969-12-31 15:59:39 1969-12-31 14:00:04 NULL 1969-12-31 16:00:00 1969-12-31 15:59:55.9 NULL NULL -21 -7196 NULL 1542429000 -21.0 -7196.0 NULL 0 1969-12-31 15:59:55.9 NULL NULL NULL -21.0 NULL -0.8366556385360561 NULL
    +true true NULL true true true NULL false true NULL -24 -7196 NULL 829111000 -24 -7196 NULL -6 NULL NULL -24 -24 -24 -24.0 -7196.0 NULL 8.29111E8 -24.0 -7196.0 NULL -6.855 NULL NULL NULL -7196.0 1969-12-31 15:59:59.976 1969-12-31 15:59:52.804 NULL 1970-01-10 06:18:31 1969-12-31 15:59:36 1969-12-31 14:00:04 NULL 1969-12-31 16:00:00 1969-12-31 15:59:53.145 NULL NULL -24 -7196 NULL 829111000 -24.0 -7196.0 NULL 0 1969-12-31 15:59:53.145 NULL NULL NULL -24.0 NULL 0.9055783620066238 NULL
    +true true NULL true true true NULL false true NULL -30 -200 NULL 1429852250 -30 -200 NULL 12 NULL NULL -30 -30 -30 -30.0 -200.0 NULL 1.42985225E9 -30.0 -200.0 NULL 12.935 NULL NULL NULL -200.0 1969-12-31 15:59:59.97 1969-12-31 15:59:59.8 NULL 1970-01-17 05:10:52.25 1969-12-31 15:59:30 1969-12-31 15:56:40 NULL 1969-12-31 16:00:00 1969-12-31 16:00:12.935 NULL NULL -30 -200 NULL 1429852250 -30.0 -200.0 NULL 0 1969-12-31 16:00:12.935 NULL NULL NULL -30.0 NULL 0.9880316240928618 NULL
    +true true NULL true true true NULL false true NULL -36 -200 NULL -2006216750 -36 -200 NULL -14 NULL NULL -36 -36 -36 -36.0 -200.0 NULL -2.00621675E9 -36.0 -200.0 NULL -14.252 NULL NULL NULL -200.0 1969-12-31 15:59:59.964 1969-12-31 15:59:59.8 NULL 1969-12-08 10:43:03.25 1969-12-31 15:59:24 1969-12-31 15:56:40 NULL 1969-12-31 16:00:00 1969-12-31 15:59:45.748 NULL NULL -36 -200 NULL -2006216750 -36.0 -200.0 NULL 0 1969-12-31 15:59:45.748 NULL NULL NULL -36.0 NULL 0.9917788534431158 NULL
    +true true NULL true true true NULL false true NULL -36 -200 NULL 1599879000 -36 -200 NULL -6 NULL NULL -36 -36 -36 -36.0 -200.0 NULL 1.599879E9 -36.0 -200.0 NULL -6.183 NULL NULL NULL -200.0 1969-12-31 15:59:59.964 1969-12-31 15:59:59.8 NULL 1970-01-19 04:24:39 1969-12-31 15:59:24 1969-12-31 15:56:40 NULL 1969-12-31 16:00:00 1969-12-31 15:59:53.817 NULL NULL -36 -200 NULL 1599879000 -36.0 -200.0 NULL 0 1969-12-31 15:59:53.817 NULL NULL NULL -36.0 NULL 0.9917788534431158 NULL
    +true true NULL true true true NULL false true NULL -38 15601 NULL -1858689000 -38 15601 NULL -1 NULL NULL -38 -38 -38 -38.0 15601.0 NULL -1.858689E9 -38.0 15601.0 NULL -1.386 NULL NULL NULL 15601.0 1969-12-31 15:59:59.962 1969-12-31 16:00:15.601 NULL 1969-12-10 03:41:51 1969-12-31 15:59:22 1969-12-31 20:20:01 NULL 1969-12-31 16:00:00 1969-12-31 15:59:58.614 NULL NULL -38 15601 NULL -1858689000 -38.0 15601.0 NULL 0 1969-12-31 15:59:58.614 NULL NULL NULL -38.0 NULL -0.2963685787093853 NULL
    +true true NULL true true true NULL false true NULL -5 15601 NULL 612416000 -5 15601 NULL 4 NULL NULL -5 -5 -5 -5.0 15601.0 NULL 6.12416E8 -5.0 15601.0 NULL 4.679 NULL NULL NULL 15601.0 1969-12-31 15:59:59.995 1969-12-31 16:00:15.601 NULL 1970-01-07 18:06:56 1969-12-31 15:59:55 1969-12-31 20:20:01 NULL 1969-12-31 16:00:00 1969-12-31 16:00:04.679 NULL NULL -5 15601 NULL 612416000 -5.0 15601.0 NULL 0 1969-12-31 16:00:04.679 NULL NULL NULL -5.0 NULL 0.9589242746631385 NULL
    +true true NULL true true true NULL false true NULL -50 -7196 NULL -1031187250 -50 -7196 NULL -5 NULL NULL -50 -50 -50 -50.0 -7196.0 NULL -1.03118725E9 -50.0 -7196.0 NULL -5.267 NULL NULL NULL -7196.0 1969-12-31 15:59:59.95 1969-12-31 15:59:52.804 NULL 1969-12-19 17:33:32.75 1969-12-31 15:59:10 1969-12-31 14:00:04 NULL 1969-12-31 16:00:00 1969-12-31 15:59:54.733 NULL NULL -50 -7196 NULL -1031187250 -50.0 -7196.0 NULL 0 1969-12-31 15:59:54.733 NULL NULL NULL -50.0 NULL 0.26237485370392877 NULL
    +true true NULL true true true NULL false true NULL -59 -7196 NULL -1604890000 -59 -7196 NULL 13 NULL NULL -59 -59 -59 -59.0 -7196.0 NULL -1.60489E9 -59.0 -7196.0 NULL 13.15 NULL NULL NULL -7196.0 1969-12-31 15:59:59.941 1969-12-31 15:59:52.804 NULL 1969-12-13 02:11:50 1969-12-31 15:59:01 1969-12-31 14:00:04 NULL 1969-12-31 16:00:00 1969-12-31 16:00:13.15 NULL NULL -59 -7196 NULL -1604890000 -59.0 -7196.0 NULL 0 1969-12-31 16:00:13.15 NULL NULL NULL -59.0 NULL -0.6367380071391379 NULL
    +true true NULL true true true NULL false true NULL -60 -7196 NULL 1516314750 -60 -7196 NULL -7 NULL NULL -60 -60 -60 -60.0 -7196.0 NULL 1.51631475E9 -60.0 -7196.0 NULL -7.592 NULL NULL NULL -7196.0 1969-12-31 15:59:59.94 1969-12-31 15:59:52.804 NULL 1970-01-18 05:11:54.75 1969-12-31 15:59:00 1969-12-31 14:00:04 NULL 1969-12-31 16:00:00 1969-12-31 15:59:52.408 NULL NULL -60 -7196 NULL 1516314750 -60.0 -7196.0 NULL 0 1969-12-31 15:59:52.408 NULL NULL NULL -60.0 NULL 0.3048106211022167 NULL
    +true true NULL true true true NULL false true NULL -8 -7196 NULL -1849991500 -8 -7196 NULL 3 NULL NULL -8 -8 -8 -8.0 -7196.0 NULL -1.8499915E9 -8.0 -7196.0 NULL 3.136 NULL NULL NULL -7196.0 1969-12-31 15:59:59.992 1969-12-31 15:59:52.804 NULL 1969-12-10 06:06:48.5 1969-12-31 15:59:52 1969-12-31 14:00:04 NULL 1969-12-31 16:00:00 1969-12-31 16:00:03.136 NULL NULL -8 -7196 NULL -1849991500 -8.0 -7196.0 NULL 0 1969-12-31 16:00:03.136 NULL NULL NULL -8.0 NULL -0.9893582466233818 NULL
    +true true NULL true true true NULL false true NULL 20 15601 NULL -362433250 20 15601 NULL -14 NULL NULL 20 20 20 20.0 15601.0 NULL -3.6243325E8 20.0 15601.0 NULL -14.871 NULL NULL NULL 15601.0 1969-12-31 16:00:00.02 1969-12-31 16:00:15.601 NULL 1969-12-27 11:19:26.75 1969-12-31 16:00:20 1969-12-31 20:20:01 NULL 1969-12-31 16:00:00 1969-12-31 15:59:45.129 NULL NULL 20 15601 NULL -362433250 20.0 15601.0 NULL 0 1969-12-31 15:59:45.129 NULL NULL NULL 20.0 NULL 0.9129452507276277 NULL
    +true true NULL true true true NULL false true NULL 48 15601 NULL -795361000 48 15601 NULL -9 NULL NULL 48 48 48 48.0 15601.0 NULL -7.95361E8 48.0 15601.0 NULL -9.765 NULL NULL NULL 15601.0 1969-12-31 16:00:00.048 1969-12-31 16:00:15.601 NULL 1969-12-22 11:03:59 1969-12-31 16:00:48 1969-12-31 20:20:01 NULL 1969-12-31 16:00:00 1969-12-31 15:59:50.235 NULL NULL 48 15601 NULL -795361000 48.0 15601.0 NULL 0 1969-12-31 15:59:50.235 NULL NULL NULL 48.0 NULL -0.7682546613236668 NULL
    +true true NULL true true true NULL false true NULL 5 -7196 NULL -1015607500 5 -7196 NULL 10 NULL NULL 5 5 5 5.0 -7196.0 NULL -1.0156075E9 5.0 -7196.0 NULL 10.973 NULL NULL NULL -7196.0 1969-12-31 16:00:00.005 1969-12-31 15:59:52.804 NULL 1969-12-19 21:53:12.5 1969-12-31 16:00:05 1969-12-31 14:00:04 NULL 1969-12-31 16:00:00 1969-12-31 16:00:10.973 NULL NULL 5 -7196 NULL -1015607500 5.0 -7196.0 NULL 0 1969-12-31 16:00:10.973 NULL NULL NULL 5.0 NULL -0.9589242746631385 NULL
    +true true NULL true true true NULL false true NULL 59 -7196 NULL -1137754500 59 -7196 NULL 10 NULL NULL 59 59 59 59.0 -7196.0 NULL -1.1377545E9 59.0 -7196.0 NULL 10.956 NULL NULL NULL -7196.0 1969-12-31 16:00:00.059 1969-12-31 15:59:52.804 NULL 1969-12-18 11:57:25.5 1969-12-31 16:00:59 1969-12-31 14:00:04 NULL 1969-12-31 16:00:00 1969-12-31 16:00:10.956 NULL NULL 59 -7196 NULL -1137754500 59.0 -7196.0 NULL 0 1969-12-31 16:00:10.956 NULL NULL NULL 59.0 NULL 0.6367380071391379 NULL

    http://git-wip-us.apache.org/repos/asf/hive/blob/2de6fda0/ql/src/test/results/clientpositive/vector_char_mapjoin1.q.out
    ----------------------------------------------------------------------
    diff --git a/ql/src/test/results/clientpositive/vector_char_mapjoin1.q.out b/ql/src/test/results/clientpositive/vector_char_mapjoin1.q.out
    index 744bfb3..af5f0de 100644
    --- a/ql/src/test/results/clientpositive/vector_char_mapjoin1.q.out
    +++ b/ql/src/test/results/clientpositive/vector_char_mapjoin1.q.out
    @@ -371,6 +371,7 @@ STAGE PLANS:
                          value expressions: _col1 (type: char(10)), _col2 (type: int), _col3 (type: string)
            Local Work:
              Map Reduce Local Work
    + Execution mode: vectorized
            Reduce Operator Tree:
              Select Operator
                expressions: KEY.reducesinkkey0 (type: int), VALUE._col0 (type: char(10)), VALUE._col1 (type: int), VALUE._col2 (type: string)

    http://git-wip-us.apache.org/repos/asf/hive/blob/2de6fda0/ql/src/test/results/clientpositive/vector_varchar_mapjoin1.q.out
    ----------------------------------------------------------------------
    diff --git a/ql/src/test/results/clientpositive/vector_varchar_mapjoin1.q.out b/ql/src/test/results/clientpositive/vector_varchar_mapjoin1.q.out
    index 2994bff..936ef78 100644
    --- a/ql/src/test/results/clientpositive/vector_varchar_mapjoin1.q.out
    +++ b/ql/src/test/results/clientpositive/vector_varchar_mapjoin1.q.out
    @@ -359,6 +359,7 @@ STAGE PLANS:
                          value expressions: _col1 (type: varchar(10)), _col2 (type: int), _col3 (type: string)
            Local Work:
              Map Reduce Local Work
    + Execution mode: vectorized
            Reduce Operator Tree:
              Select Operator
                expressions: KEY.reducesinkkey0 (type: int), VALUE._col0 (type: varchar(10)), VALUE._col1 (type: int), VALUE._col2 (type: string)

    http://git-wip-us.apache.org/repos/asf/hive/blob/2de6fda0/ql/src/test/results/clientpositive/vectorized_casts.q.out
    ----------------------------------------------------------------------
    diff --git a/ql/src/test/results/clientpositive/vectorized_casts.q.out b/ql/src/test/results/clientpositive/vectorized_casts.q.out
    index 1113453..3c28064 100644
    --- a/ql/src/test/results/clientpositive/vectorized_casts.q.out
    +++ b/ql/src/test/results/clientpositive/vectorized_casts.q.out
    @@ -1,5 +1,6 @@
      PREHOOK: query: -- SORT_QUERY_RESULTS

    +-- Currently, vectorization is not supported in fetch task (hive.fetch.task.conversion=none)
      -- Test type casting in vectorized mode to verify end-to-end functionality.

      explain
    @@ -65,6 +66,8 @@ select
        ,cast (cbigint * 0 as string)
        ,cast (ctimestamp1 as string)
        ,cast (cstring1 as string)
    + ,cast (cast (cstring1 as char(10)) as string)
    + ,cast (cast (cstring1 as varchar(10)) as string)
      -- nested and expression arguments
        ,cast (cast (cfloat as int) as float)
        ,cast (cint * 2 as double)
    @@ -76,6 +79,7 @@ where cbigint % 250 = 0
      PREHOOK: type: QUERY
      POSTHOOK: query: -- SORT_QUERY_RESULTS

    +-- Currently, vectorization is not supported in fetch task (hive.fetch.task.conversion=none)
      -- Test type casting in vectorized mode to verify end-to-end functionality.

      explain
    @@ -141,6 +145,8 @@ select
        ,cast (cbigint * 0 as string)
        ,cast (ctimestamp1 as string)
        ,cast (cstring1 as string)
    + ,cast (cast (cstring1 as char(10)) as string)
    + ,cast (cast (cstring1 as varchar(10)) as string)
      -- nested and expression arguments
        ,cast (cast (cfloat as int) as float)
        ,cast (cint * 2 as double)
    @@ -165,8 +171,8 @@ STAGE PLANS:
                    predicate: ((cbigint % 250) = 0) (type: boolean)
                    Statistics: Num rows: 6144 Data size: 1320982 Basic stats: COMPLETE Column stats: NONE
                    Select Operator
    - expressions: UDFToBoolean(ctinyint) (type: boolean), UDFToBoolean(csmallint) (type: boolean), UDFToBoolean(cint) (type: boolean), UDFToBoolean(cbigint) (type: boolean), UDFToBoolean(cfloat) (type: boolean), UDFToBoolean(cdouble) (type: boolean), cboolean1 (type: boolean), UDFToBoolean((cbigint * 0)) (type: boolean), UDFToBoolean(ctimestamp1) (type: boolean), UDFToBoolean(cstring1) (type: boolean), UDFToInteger(ctinyint) (type: int), UDFToInteger(csmallint) (type: int), cint (type: int), UDFToInteger(cbigint) (type: int), UDFToInteger(cfloat) (type: int), UDFToInteger(cdouble) (type: int), UDFToInteger(cboolean1) (type: int), UDFToInteger(ctimestamp1) (type: int), UDFToInteger(cstring1) (type: int), UDFToInteger(substr(cstring1, 1, 1)) (type: int), UDFToByte(cfloat) (type: tinyint), UDFToShort(cfloat) (type: smallint), UDFToLong(cfloat) (type: bigint), UDFToDouble(ctinyint) (type: double), UDFToDouble(csmallint) (type: double), UDFToDouble(cint) (type: double), UDFToD
      ouble(cbigint) (type: double), UDFToDouble(cfloat) (type: double), cdouble (type: double), UDFToDouble(cboolean1) (type: double), UDFToDouble(ctimestamp1) (type: double), UDFToDouble(cstring1) (type: double), UDFToDouble(substr(cstring1, 1, 1)) (type: double), UDFToFloat(cint) (type: float), UDFToFloat(cdouble) (type: float), CAST( ctinyint AS TIMESTAMP) (type: timestamp), CAST( csmallint AS TIMESTAMP) (type: timestamp), CAST( cint AS TIMESTAMP) (type: timestamp), CAST( cbigint AS TIMESTAMP) (type: timestamp), CAST( cfloat AS TIMESTAMP) (type: timestamp), CAST( cdouble AS TIMESTAMP) (type: timestamp), CAST( cboolean1 AS TIMESTAMP) (type: timestamp), CAST( (cbigint * 0) AS TIMESTAMP) (type: timestamp), ctimestamp1 (type: timestamp), CAST( cstring1 AS TIMESTAMP) (type: timestamp), CAST( substr(cstring1, 1, 1) AS TIMESTAMP) (type: timestamp), UDFToString(ctinyint) (type: string), UDFToString(csmallint) (type: string), UDFToString(cint) (type: string), UDFToString(cbigint) (type: string
      ), UDFToString(cfloat) (type: string), UDFToString(cdouble) (type: string), UDFToString(cboolean1) (type: string), UDFToString((cbigint * 0)) (type: string), UDFToString(ctimestamp1) (type: string), cstring1 (type: string), UDFToFloat(UDFToInteger(cfloat)) (type: float), UDFToDouble((cint * 2)) (type: double), UDFToString(sin(cfloat)) (type: string), (UDFToDouble(UDFToFloat(cint)) + UDFToDouble(cboolean1)) (type: double)
    - outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22, _col23, _col24, _col25, _col26, _col27, _col28, _col29, _col30, _col31, _col32, _col33, _col34, _col35, _col36, _col37, _col38, _col39, _col40, _col41, _col42, _col43, _col44, _col45, _col46, _col47, _col48, _col49, _col50, _col51, _col52, _col53, _col54, _col55, _col56, _col57, _col58, _col59
    + expressions: UDFToBoolean(ctinyint) (type: boolean), UDFToBoolean(csmallint) (type: boolean), UDFToBoolean(cint) (type: boolean), UDFToBoolean(cbigint) (type: boolean), UDFToBoolean(cfloat) (type: boolean), UDFToBoolean(cdouble) (type: boolean), cboolean1 (type: boolean), UDFToBoolean((cbigint * 0)) (type: boolean), UDFToBoolean(ctimestamp1) (type: boolean), UDFToBoolean(cstring1) (type: boolean), UDFToInteger(ctinyint) (type: int), UDFToInteger(csmallint) (type: int), cint (type: int), UDFToInteger(cbigint) (type: int), UDFToInteger(cfloat) (type: int), UDFToInteger(cdouble) (type: int), UDFToInteger(cboolean1) (type: int), UDFToInteger(ctimestamp1) (type: int), UDFToInteger(cstring1) (type: int), UDFToInteger(substr(cstring1, 1, 1)) (type: int), UDFToByte(cfloat) (type: tinyint), UDFToShort(cfloat) (type: smallint), UDFToLong(cfloat) (type: bigint), UDFToDouble(ctinyint) (type: double), UDFToDouble(csmallint) (type: double), UDFToDouble(cint) (type: double), UDFToD
      ouble(cbigint) (type: double), UDFToDouble(cfloat) (type: double), cdouble (type: double), UDFToDouble(cboolean1) (type: double), UDFToDouble(ctimestamp1) (type: double), UDFToDouble(cstring1) (type: double), UDFToDouble(substr(cstring1, 1, 1)) (type: double), UDFToFloat(cint) (type: float), UDFToFloat(cdouble) (type: float), CAST( ctinyint AS TIMESTAMP) (type: timestamp), CAST( csmallint AS TIMESTAMP) (type: timestamp), CAST( cint AS TIMESTAMP) (type: timestamp), CAST( cbigint AS TIMESTAMP) (type: timestamp), CAST( cfloat AS TIMESTAMP) (type: timestamp), CAST( cdouble AS TIMESTAMP) (type: timestamp), CAST( cboolean1 AS TIMESTAMP) (type: timestamp), CAST( (cbigint * 0) AS TIMESTAMP) (type: timestamp), ctimestamp1 (type: timestamp), CAST( cstring1 AS TIMESTAMP) (type: timestamp), CAST( substr(cstring1, 1, 1) AS TIMESTAMP) (type: timestamp), UDFToString(ctinyint) (type: string), UDFToString(csmallint) (type: string), UDFToString(cint) (type: string), UDFToString(cbigint) (type: string
      ), UDFToString(cfloat) (type: string), UDFToString(cdouble) (type: string), UDFToString(cboolean1) (type: string), UDFToString((cbigint * 0)) (type: string), UDFToString(ctimestamp1) (type: string), cstring1 (type: string), UDFToString(CAST( cstring1 AS CHAR(10)) (type: string), UDFToString(CAST( cstring1 AS varchar(10))) (type: string), UDFToFloat(UDFToInteger(cfloat)) (type: float), UDFToDouble((cint * 2)) (type: double), UDFToString(sin(cfloat)) (type: string), (UDFToDouble(UDFToFloat(cint)) + UDFToDouble(cboolean1)) (type: double)
    + outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22, _col23, _col24, _col25, _col26, _col27, _col28, _col29, _col30, _col31, _col32, _col33, _col34, _col35, _col36, _col37, _col38, _col39, _col40, _col41, _col42, _col43, _col44, _col45, _col46, _col47, _col48, _col49, _col50, _col51, _col52, _col53, _col54, _col55, _col56, _col57, _col58, _col59, _col60, _col61
                      Statistics: Num rows: 6144 Data size: 1320982 Basic stats: COMPLETE Column stats: NONE
                      File Output Operator
                        compressed: false
    @@ -245,6 +251,8 @@ PREHOOK: query: select
        ,cast (cbigint * 0 as string)
        ,cast (ctimestamp1 as string)
        ,cast (cstring1 as string)
    + ,cast (cast (cstring1 as char(10)) as string)
    + ,cast (cast (cstring1 as varchar(10)) as string)
      -- nested and expression arguments
        ,cast (cast (cfloat as int) as float)
        ,cast (cint * 2 as double)
    @@ -318,6 +326,8 @@ POSTHOOK: query: select
        ,cast (cbigint * 0 as string)
        ,cast (ctimestamp1 as string)
        ,cast (cstring1 as string)
    + ,cast (cast (cstring1 as char(10)) as string)
    + ,cast (cast (cstring1 as varchar(10)) as string)
      -- nested and expression arguments
        ,cast (cast (cfloat as int) as float)
        ,cast (cint * 2 as double)
    @@ -329,29 +339,29 @@ where cbigint % 250 = 0
      POSTHOOK: type: QUERY
      POSTHOOK: Input: default@alltypesorc
      #### A masked pattern was here ####
    -true NULL true true true NULL false false true true -51 NULL 773600971 1053923250 -51 NULL 0 8 NULL 2 -51 -51 -51 -51.0 NULL 7.73600971E8 1.05392325E9 -51.0 NULL 0.0 8.451 NULL 2.0 7.7360096E8 NULL 1969-12-31 15:59:59.949 NULL 1970-01-09 14:53:20.971 1970-01-12 20:45:23.25 1969-12-31 15:59:09 NULL 1969-12-31 16:00:00 1969-12-31 16:00:00 1969-12-31 16:00:08.451 NULL NULL -51 NULL 773600971 1053923250 -51.0 NULL FALSE 0 1969-12-31 16:00:08.451 2yK4Bx76O -51.0 1.547201942E9 -0.6702291758433747 7.73600971E8
    -true NULL true true true NULL false false true true 8 NULL -102936434 -1312782750 8 NULL 0 15 NULL NULL 8 8 8 8.0 NULL -1.02936434E8 -1.31278275E9 8.0 NULL 0.0 15.892 NULL NULL -1.02936432E8 NULL 1969-12-31 16:00:00.008 NULL 1969-12-30 11:24:23.566 1969-12-16 11:20:17.25 1969-12-31 16:00:08 NULL 1969-12-31 16:00:00 1969-12-31 16:00:00 1969-12-31 16:00:15.892 NULL NULL 8 NULL -102936434 -1312782750 8.0 NULL FALSE 0 1969-12-31 16:00:15.892 eJROSNhugc3kQR7Pb 8.0 -2.05872868E8 0.9893582466233818 -1.02936434E8
    -true NULL true true true NULL false false true true 8 NULL -661621138 -931392750 8 NULL 0 15 NULL NULL 8 8 8 8.0 NULL -6.61621138E8 -9.3139275E8 8.0 NULL 0.0 15.892 NULL NULL -6.6162112E8 NULL 1969-12-31 16:00:00.008 NULL 1969-12-24 00:12:58.862 1969-12-20 21:16:47.25 1969-12-31 16:00:08 NULL 1969-12-31 16:00:00 1969-12-31 16:00:00 1969-12-31 16:00:15.892 NULL NULL 8 NULL -661621138 -931392750 8.0 NULL FALSE 0 1969-12-31 16:00:15.892 L15l8i5k558tBcDV20 8.0 -1.323242276E9 0.9893582466233818 -6.61621138E8
    -true NULL true true true NULL false false true true 8 NULL -669632311 1588591250 8 NULL 0 15 NULL 3 8 8 8 8.0 NULL -6.69632311E8 1.58859125E9 8.0 NULL 0.0 15.892 NULL 3.0 -6.6963232E8 NULL 1969-12-31 16:00:00.008 NULL 1969-12-23 21:59:27.689 1970-01-19 01:16:31.25 1969-12-31 16:00:08 NULL 1969-12-31 16:00:00 1969-12-31 16:00:00 1969-12-31 16:00:15.892 NULL NULL 8 NULL -669632311 1588591250 8.0 NULL FALSE 0 1969-12-31 16:00:15.892 3r3sDvfUkG0yTP3LnX5mNQRr 8.0 -1.339264622E9 0.9893582466233818 -6.69632311E8
    -true NULL true true true NULL false false true true 8 NULL 805179664 868161500 8 NULL 0 15 NULL NULL 8 8 8 8.0 NULL 8.05179664E8 8.681615E8 8.0 NULL 0.0 15.892 NULL NULL 8.0517965E8 NULL 1969-12-31 16:00:00.008 NULL 1970-01-09 23:39:39.664 1970-01-10 17:09:21.5 1969-12-31 16:00:08 NULL 1969-12-31 16:00:00 1969-12-31 16:00:00 1969-12-31 16:00:15.892 NULL NULL 8 NULL 805179664 868161500 8.0 NULL FALSE 0 1969-12-31 16:00:15.892 e005B5q 8.0 1.610359328E9 0.9893582466233818 8.05179664E8
    -true NULL true true true NULL true false true true -51 NULL 747553882 -1930467250 -51 NULL 1 8 NULL NULL -51 -51 -51 -51.0 NULL 7.47553882E8 -1.93046725E9 -51.0 NULL 1.0 8.451 NULL NULL 7.4755386E8 NULL 1969-12-31 15:59:59.949 NULL 1970-01-09 07:39:13.882 1969-12-09 07:45:32.75 1969-12-31 15:59:09 NULL 1969-12-31 16:00:00.001 1969-12-31 16:00:00 1969-12-31 16:00:08.451 NULL NULL -51 NULL 747553882 -1930467250 -51.0 NULL TRUE 0 1969-12-31 16:00:08.451 q8M86Fx0r -51.0 1.495107764E9 -0.6702291758433747 7.47553883E8
    -true NULL true true true NULL true false true true 11 NULL -335450417 1233327000 11 NULL 1 2 NULL NULL 11 11 11 11.0 NULL -3.35450417E8 1.233327E9 11.0 NULL 1.0 2.351 NULL NULL -3.35450432E8 NULL 1969-12-31 16:00:00.011 NULL 1969-12-27 18:49:09.583 1970-01-14 22:35:27 1969-12-31 16:00:11 NULL 1969-12-31 16:00:00.001 1969-12-31 16:00:00 1969-12-31 16:00:02.351 NULL NULL 11 NULL -335450417 1233327000 11.0 NULL TRUE 0 1969-12-31 16:00:02.351 dOYnqgaXoJ1P3ERwxe5N7 11.0 -6.70900834E8 -0.9999902065507035 -3.35450416E8
    -true NULL true true true NULL true false true true 11 NULL -64615982 1803053750 11 NULL 1 2 NULL 8 11 11 11 11.0 NULL -6.4615982E7 1.80305375E9 11.0 NULL 1.0 2.351 NULL 8.0 -6.4615984E7 NULL 1969-12-31 16:00:00.011 NULL 1969-12-30 22:03:04.018 1970-01-21 12:50:53.75 1969-12-31 16:00:11 NULL 1969-12-31 16:00:00.001 1969-12-31 16:00:00 1969-12-31 16:00:02.351 NULL NULL 11 NULL -64615982 1803053750 11.0 NULL TRUE 0 1969-12-31 16:00:02.351 8J5OB7K26PEV7kdbeHr3 11.0 -1.29231964E8 -0.9999902065507035 -6.4615981E7
    -true NULL true true true NULL true false true true 8 NULL 890988972 -1862301000 8 NULL 1 15 NULL NULL 8 8 8 8.0 NULL 8.90988972E8 -1.862301E9 8.0 NULL 1.0 15.892 NULL NULL 8.9098899E8 NULL 1969-12-31 16:00:00.008 NULL 1970-01-10 23:29:48.972 1969-12-10 02:41:39 1969-12-31 16:00:08 NULL 1969-12-31 16:00:00.001 1969-12-31 16:00:00 1969-12-31 16:00:15.892 NULL NULL 8 NULL 890988972 -1862301000 8.0 NULL TRUE 0 1969-12-31 16:00:15.892 XylAH4 8.0 1.781977944E9 0.9893582466233818 8.90988973E8
    -true NULL true true true NULL true false true true 8 NULL 930867246 1205399250 8 NULL 1 15 NULL NULL 8 8 8 8.0 NULL 9.30867246E8 1.20539925E9 8.0 NULL 1.0 15.892 NULL NULL 9.3086726E8 NULL 1969-12-31 16:00:00.008 NULL 1970-01-11 10:34:27.246 1970-01-14 14:49:59.25 1969-12-31 16:00:08 NULL 1969-12-31 16:00:00.001 1969-12-31 16:00:00 1969-12-31 16:00:15.892 NULL NULL 8 NULL 930867246 1205399250 8.0 NULL TRUE 0 1969-12-31 16:00:15.892 c1V8o1A 8.0 1.861734492E9 0.9893582466233818 9.30867247E8
    -true true NULL true true true NULL false true NULL -14 -7196 NULL -1552199500 -14 -7196 NULL 11 NULL NULL -14 -14 -14 -14.0 -7196.0 NULL -1.5521995E9 -14.0 -7196.0 NULL 11.065 NULL NULL NULL -7196.0 1969-12-31 15:59:59.986 1969-12-31 15:59:52.804 NULL 1969-12-13 16:50:00.5 1969-12-31 15:59:46 1969-12-31 14:00:04 NULL 1969-12-31 16:00:00 1969-12-31 16:00:11.065 NULL NULL -14 -7196 NULL -1552199500 -14.0 -7196.0 NULL 0 1969-12-31 16:00:11.065 NULL -14.0 NULL -0.9906073556948704 NULL
    -true true NULL true true true NULL false true NULL -21 -7196 NULL 1542429000 -21 -7196 NULL -4 NULL NULL -21 -21 -21 -21.0 -7196.0 NULL 1.542429E9 -21.0 -7196.0 NULL -4.1 NULL NULL NULL -7196.0 1969-12-31 15:59:59.979 1969-12-31 15:59:52.804 NULL 1970-01-18 12:27:09 1969-12-31 15:59:39 1969-12-31 14:00:04 NULL 1969-12-31 16:00:00 1969-12-31 15:59:55.9 NULL NULL -21 -7196 NULL 1542429000 -21.0 -7196.0 NULL 0 1969-12-31 15:59:55.9 NULL -21.0 NULL -0.8366556385360561 NULL
    -true true NULL true true true NULL false true NULL -24 -7196 NULL 829111000 -24 -7196 NULL -6 NULL NULL -24 -24 -24 -24.0 -7196.0 NULL 8.29111E8 -24.0 -7196.0 NULL -6.855 NULL NULL NULL -7196.0 1969-12-31 15:59:59.976 1969-12-31 15:59:52.804 NULL 1970-01-10 06:18:31 1969-12-31 15:59:36 1969-12-31 14:00:04 NULL 1969-12-31 16:00:00 1969-12-31 15:59:53.145 NULL NULL -24 -7196 NULL 829111000 -24.0 -7196.0 NULL 0 1969-12-31 15:59:53.145 NULL -24.0 NULL 0.9055783620066238 NULL
    -true true NULL true true true NULL false true NULL -30 -200 NULL 1429852250 -30 -200 NULL 12 NULL NULL -30 -30 -30 -30.0 -200.0 NULL 1.42985225E9 -30.0 -200.0 NULL 12.935 NULL NULL NULL -200.0 1969-12-31 15:59:59.97 1969-12-31 15:59:59.8 NULL 1970-01-17 05:10:52.25 1969-12-31 15:59:30 1969-12-31 15:56:40 NULL 1969-12-31 16:00:00 1969-12-31 16:00:12.935 NULL NULL -30 -200 NULL 1429852250 -30.0 -200.0 NULL 0 1969-12-31 16:00:12.935 NULL -30.0 NULL 0.9880316240928618 NULL
    -true true NULL true true true NULL false true NULL -36 -200 NULL -2006216750 -36 -200 NULL -14 NULL NULL -36 -36 -36 -36.0 -200.0 NULL -2.00621675E9 -36.0 -200.0 NULL -14.252 NULL NULL NULL -200.0 1969-12-31 15:59:59.964 1969-12-31 15:59:59.8 NULL 1969-12-08 10:43:03.25 1969-12-31 15:59:24 1969-12-31 15:56:40 NULL 1969-12-31 16:00:00 1969-12-31 15:59:45.748 NULL NULL -36 -200 NULL -2006216750 -36.0 -200.0 NULL 0 1969-12-31 15:59:45.748 NULL -36.0 NULL 0.9917788534431158 NULL
    -true true NULL true true true NULL false true NULL -36 -200 NULL 1599879000 -36 -200 NULL -6 NULL NULL -36 -36 -36 -36.0 -200.0 NULL 1.599879E9 -36.0 -200.0 NULL -6.183 NULL NULL NULL -200.0 1969-12-31 15:59:59.964 1969-12-31 15:59:59.8 NULL 1970-01-19 04:24:39 1969-12-31 15:59:24 1969-12-31 15:56:40 NULL 1969-12-31 16:00:00 1969-12-31 15:59:53.817 NULL NULL -36 -200 NULL 1599879000 -36.0 -200.0 NULL 0 1969-12-31 15:59:53.817 NULL -36.0 NULL 0.9917788534431158 NULL
    -true true NULL true true true NULL false true NULL -38 15601 NULL -1858689000 -38 15601 NULL -1 NULL NULL -38 -38 -38 -38.0 15601.0 NULL -1.858689E9 -38.0 15601.0 NULL -1.386 NULL NULL NULL 15601.0 1969-12-31 15:59:59.962 1969-12-31 16:00:15.601 NULL 1969-12-10 03:41:51 1969-12-31 15:59:22 1969-12-31 20:20:01 NULL 1969-12-31 16:00:00 1969-12-31 15:59:58.614 NULL NULL -38 15601 NULL -1858689000 -38.0 15601.0 NULL 0 1969-12-31 15:59:58.614 NULL -38.0 NULL -0.2963685787093853 NULL
    -true true NULL true true true NULL false true NULL -5 15601 NULL 612416000 -5 15601 NULL 4 NULL NULL -5 -5 -5 -5.0 15601.0 NULL 6.12416E8 -5.0 15601.0 NULL 4.679 NULL NULL NULL 15601.0 1969-12-31 15:59:59.995 1969-12-31 16:00:15.601 NULL 1970-01-07 18:06:56 1969-12-31 15:59:55 1969-12-31 20:20:01 NULL 1969-12-31 16:00:00 1969-12-31 16:00:04.679 NULL NULL -5 15601 NULL 612416000 -5.0 15601.0 NULL 0 1969-12-31 16:00:04.679 NULL -5.0 NULL 0.9589242746631385 NULL
    -true true NULL true true true NULL false true NULL -50 -7196 NULL -1031187250 -50 -7196 NULL -5 NULL NULL -50 -50 -50 -50.0 -7196.0 NULL -1.03118725E9 -50.0 -7196.0 NULL -5.267 NULL NULL NULL -7196.0 1969-12-31 15:59:59.95 1969-12-31 15:59:52.804 NULL 1969-12-19 17:33:32.75 1969-12-31 15:59:10 1969-12-31 14:00:04 NULL 1969-12-31 16:00:00 1969-12-31 15:59:54.733 NULL NULL -50 -7196 NULL -1031187250 -50.0 -7196.0 NULL 0 1969-12-31 15:59:54.733 NULL -50.0 NULL 0.26237485370392877 NULL
    -true true NULL true true true NULL false true NULL -59 -7196 NULL -1604890000 -59 -7196 NULL 13 NULL NULL -59 -59 -59 -59.0 -7196.0 NULL -1.60489E9 -59.0 -7196.0 NULL 13.15 NULL NULL NULL -7196.0 1969-12-31 15:59:59.941 1969-12-31 15:59:52.804 NULL 1969-12-13 02:11:50 1969-12-31 15:59:01 1969-12-31 14:00:04 NULL 1969-12-31 16:00:00 1969-12-31 16:00:13.15 NULL NULL -59 -7196 NULL -1604890000 -59.0 -7196.0 NULL 0 1969-12-31 16:00:13.15 NULL -59.0 NULL -0.6367380071391379 NULL
    -true true NULL true true true NULL false true NULL -60 -7196 NULL 1516314750 -60 -7196 NULL -7 NULL NULL -60 -60 -60 -60.0 -7196.0 NULL 1.51631475E9 -60.0 -7196.0 NULL -7.592 NULL NULL NULL -7196.0 1969-12-31 15:59:59.94 1969-12-31 15:59:52.804 NULL 1970-01-18 05:11:54.75 1969-12-31 15:59:00 1969-12-31 14:00:04 NULL 1969-12-31 16:00:00 1969-12-31 15:59:52.408 NULL NULL -60 -7196 NULL 1516314750 -60.0 -7196.0 NULL 0 1969-12-31 15:59:52.408 NULL -60.0 NULL 0.3048106211022167 NULL
    -true true NULL true true true NULL false true NULL -8 -7196 NULL -1849991500 -8 -7196 NULL 3 NULL NULL -8 -8 -8 -8.0 -7196.0 NULL -1.8499915E9 -8.0 -7196.0 NULL 3.136 NULL NULL NULL -7196.0 1969-12-31 15:59:59.992 1969-12-31 15:59:52.804 NULL 1969-12-10 06:06:48.5 1969-12-31 15:59:52 1969-12-31 14:00:04 NULL 1969-12-31 16:00:00 1969-12-31 16:00:03.136 NULL NULL -8 -7196 NULL -1849991500 -8.0 -7196.0 NULL 0 1969-12-31 16:00:03.136 NULL -8.0 NULL -0.9893582466233818 NULL
    -true true NULL true true true NULL false true NULL 20 15601 NULL -362433250 20 15601 NULL -14 NULL NULL 20 20 20 20.0 15601.0 NULL -3.6243325E8 20.0 15601.0 NULL -14.871 NULL NULL NULL 15601.0 1969-12-31 16:00:00.02 1969-12-31 16:00:15.601 NULL 1969-12-27 11:19:26.75 1969-12-31 16:00:20 1969-12-31 20:20:01 NULL 1969-12-31 16:00:00 1969-12-31 15:59:45.129 NULL NULL 20 15601 NULL -362433250 20.0 15601.0 NULL 0 1969-12-31 15:59:45.129 NULL 20.0 NULL 0.9129452507276277 NULL
    -true true NULL true true true NULL false true NULL 48 15601 NULL -795361000 48 15601 NULL -9 NULL NULL 48 48 48 48.0 15601.0 NULL -7.95361E8 48.0 15601.0 NULL -9.765 NULL NULL NULL 15601.0 1969-12-31 16:00:00.048 1969-12-31 16:00:15.601 NULL 1969-12-22 11:03:59 1969-12-31 16:00:48 1969-12-31 20:20:01 NULL 1969-12-31 16:00:00 1969-12-31 15:59:50.235 NULL NULL 48 15601 NULL -795361000 48.0 15601.0 NULL 0 1969-12-31 15:59:50.235 NULL 48.0 NULL -0.7682546613236668 NULL
    -true true NULL true true true NULL false true NULL 5 -7196 NULL -1015607500 5 -7196 NULL 10 NULL NULL 5 5 5 5.0 -7196.0 NULL -1.0156075E9 5.0 -7196.0 NULL 10.973 NULL NULL NULL -7196.0 1969-12-31 16:00:00.005 1969-12-31 15:59:52.804 NULL 1969-12-19 21:53:12.5 1969-12-31 16:00:05 1969-12-31 14:00:04 NULL 1969-12-31 16:00:00 1969-12-31 16:00:10.973 NULL NULL 5 -7196 NULL -1015607500 5.0 -7196.0 NULL 0 1969-12-31 16:00:10.973 NULL 5.0 NULL -0.9589242746631385 NULL
    -true true NULL true true true NULL false true NULL 59 -7196 NULL -1137754500 59 -7196 NULL 10 NULL NULL 59 59 59 59.0 -7196.0 NULL -1.1377545E9 59.0 -7196.0 NULL 10.956 NULL NULL NULL -7196.0 1969-12-31 16:00:00.059 1969-12-31 15:59:52.804 NULL 1969-12-18 11:57:25.5 1969-12-31 16:00:59 1969-12-31 14:00:04 NULL 1969-12-31 16:00:00 1969-12-31 16:00:10.956 NULL NULL 59 -7196 NULL -1137754500 59.0 -7196.0 NULL 0 1969-12-31 16:00:10.956 NULL 59.0 NULL 0.6367380071391379 NULL
    +true NULL true true true NULL false false true true -51 NULL 773600971 1053923250 -51 NULL 0 8 NULL 2 -51 -51 -51 -51.0 NULL 7.73600971E8 1.05392325E9 -51.0 NULL 0.0 8.451 NULL 2.0 7.7360096E8 NULL 1969-12-31 15:59:59.949 NULL 1970-01-09 14:53:20.971 1970-01-12 20:45:23.25 1969-12-31 15:59:09 NULL 1969-12-31 16:00:00 1969-12-31 16:00:00 1969-12-31 16:00:08.451 NULL NULL -51 NULL 773600971 1053923250 -51.0 NULL FALSE 0 1969-12-31 16:00:08.451 2yK4Bx76O 2yK4Bx76O 2yK4Bx76O -51.0 1.547201942E9 -0.6702291758433747 7.73600971E8
    +true NULL true true true NULL false false true true 8 NULL -102936434 -1312782750 8 NULL 0 15 NULL NULL 8 8 8 8.0 NULL -1.02936434E8 -1.31278275E9 8.0 NULL 0.0 15.892 NULL NULL -1.02936432E8 NULL 1969-12-31 16:00:00.008 NULL 1969-12-30 11:24:23.566 1969-12-16 11:20:17.25 1969-12-31 16:00:08 NULL 1969-12-31 16:00:00 1969-12-31 16:00:00 1969-12-31 16:00:15.892 NULL NULL 8 NULL -102936434 -1312782750 8.0 NULL FALSE 0 1969-12-31 16:00:15.892 eJROSNhugc3kQR7Pb eJROSNhugc eJROSNhugc 8.0 -2.05872868E8 0.9893582466233818 -1.02936434E8
    +true NULL true true true NULL false false true true 8 NULL -661621138 -931392750 8 NULL 0 15 NULL NULL 8 8 8 8.0 NULL -6.61621138E8 -9.3139275E8 8.0 NULL 0.0 15.892 NULL NULL -6.6162112E8 NULL 1969-12-31 16:00:00.008 NULL 1969-12-24 00:12:58.862 1969-12-20 21:16:47.25 1969-12-31 16:00:08 NULL 1969-12-31 16:00:00 1969-12-31 16:00:00 1969-12-31 16:00:15.892 NULL NULL 8 NULL -661621138 -931392750 8.0 NULL FALSE 0 1969-12-31 16:00:15.892 L15l8i5k558tBcDV20 L15l8i5k55 L15l8i5k55 8.0 -1.323242276E9 0.9893582466233818 -6.61621138E8
    +true NULL true true true NULL false false true true 8 NULL -669632311 1588591250 8 NULL 0 15 NULL 3 8 8 8 8.0 NULL -6.69632311E8 1.58859125E9 8.0 NULL 0.0 15.892 NULL 3.0 -6.6963232E8 NULL 1969-12-31 16:00:00.008 NULL 1969-12-23 21:59:27.689 1970-01-19 01:16:31.25 1969-12-31 16:00:08 NULL 1969-12-31 16:00:00 1969-12-31 16:00:00 1969-12-31 16:00:15.892 NULL NULL 8 NULL -669632311 1588591250 8.0 NULL FALSE 0 1969-12-31 16:00:15.892 3r3sDvfUkG0yTP3LnX5mNQRr 3r3sDvfUkG 3r3sDvfUkG 8.0 -1.339264622E9 0.9893582466233818 -6.69632311E8
    +true NULL true true true NULL false false true true 8 NULL 805179664 868161500 8 NULL 0 15 NULL NULL 8 8 8 8.0 NULL 8.05179664E8 8.681615E8 8.0 NULL 0.0 15.892 NULL NULL 8.0517965E8 NULL 1969-12-31 16:00:00.008 NULL 1970-01-09 23:39:39.664 1970-01-10 17:09:21.5 1969-12-31 16:00:08 NULL 1969-12-31 16:00:00 1969-12-31 16:00:00 1969-12-31 16:00:15.892 NULL NULL 8 NULL 805179664 868161500 8.0 NULL FALSE 0 1969-12-31 16:00:15.892 e005B5q e005B5q e005B5q 8.0 1.610359328E9 0.9893582466233818 8.05179664E8
    +true NULL true true true NULL true false true true -51 NULL 747553882 -1930467250 -51 NULL 1 8 NULL NULL -51 -51 -51 -51.0 NULL 7.47553882E8 -1.93046725E9 -51.0 NULL 1.0 8.451 NULL NULL 7.4755386E8 NULL 1969-12-31 15:59:59.949 NULL 1970-01-09 07:39:13.882 1969-12-09 07:45:32.75 1969-12-31 15:59:09 NULL 1969-12-31 16:00:00.001 1969-12-31 16:00:00 1969-12-31 16:00:08.451 NULL NULL -51 NULL 747553882 -1930467250 -51.0 NULL TRUE 0 1969-12-31 16:00:08.451 q8M86Fx0r q8M86Fx0r q8M86Fx0r -51.0 1.495107764E9 -0.6702291758433747 7.47553883E8
    +true NULL true true true NULL true false true true 11 NULL -335450417 1233327000 11 NULL 1 2 NULL NULL 11 11 11 11.0 NULL -3.35450417E8 1.233327E9 11.0 NULL 1.0 2.351 NULL NULL -3.35450432E8 NULL 1969-12-31 16:00:00.011 NULL 1969-12-27 18:49:09.583 1970-01-14 22:35:27 1969-12-31 16:00:11 NULL 1969-12-31 16:00:00.001 1969-12-31 16:00:00 1969-12-31 16:00:02.351 NULL NULL 11 NULL -335450417 1233327000 11.0 NULL TRUE 0 1969-12-31 16:00:02.351 dOYnqgaXoJ1P3ERwxe5N7 dOYnqgaXoJ dOYnqgaXoJ 11.0 -6.70900834E8 -0.9999902065507035 -3.35450416E8
    +true NULL true true true NULL true false true true 11 NULL -64615982 1803053750 11 NULL 1 2 NULL 8 11 11 11 11.0 NULL -6.4615982E7 1.80305375E9 11.0 NULL 1.0 2.351 NULL 8.0 -6.4615984E7 NULL 1969-12-31 16:00:00.011 NULL 1969-12-30 22:03:04.018 1970-01-21 12:50:53.75 1969-12-31 16:00:11 NULL 1969-12-31 16:00:00.001 1969-12-31 16:00:00 1969-12-31 16:00:02.351 NULL NULL 11 NULL -64615982 1803053750 11.0 NULL TRUE 0 1969-12-31 16:00:02.351 8J5OB7K26PEV7kdbeHr3 8J5OB7K26P 8J5OB7K26P 11.0 -1.29231964E8 -0.9999902065507035 -6.4615981E7
    +true NULL true true true NULL true false true true 8 NULL 890988972 -1862301000 8 NULL 1 15 NULL NULL 8 8 8 8.0 NULL 8.90988972E8 -1.862301E9 8.0 NULL 1.0 15.892 NULL NULL 8.9098899E8 NULL 1969-12-31 16:00:00.008 NULL 1970-01-10 23:29:48.972 1969-12-10 02:41:39 1969-12-31 16:00:08 NULL 1969-12-31 16:00:00.001 1969-12-31 16:00:00 1969-12-31 16:00:15.892 NULL NULL 8 NULL 890988972 -1862301000 8.0 NULL TRUE 0 1969-12-31 16:00:15.892 XylAH4 XylAH4 XylAH4 8.0 1.781977944E9 0.9893582466233818 8.90988973E8
    +true NULL true true true NULL true false true true 8 NULL 930867246 1205399250 8 NULL 1 15 NULL NULL 8 8 8 8.0 NULL 9.30867246E8 1.20539925E9 8.0 NULL 1.0 15.892 NULL NULL 9.3086726E8 NULL 1969-12-31 16:00:00.008 NULL 1970-01-11 10:34:27.246 1970-01-14 14:49:59.25 1969-12-31 16:00:08 NULL 1969-12-31 16:00:00.001 1969-12-31 16:00:00 1969-12-31 16:00:15.892 NULL NULL 8 NULL 930867246 1205399250 8.0 NULL TRUE 0 1969-12-31 16:00:15.892 c1V8o1A c1V8o1A c1V8o1A 8.0 1.861734492E9 0.9893582466233818 9.30867247E8
    +true true NULL true true true NULL false true NULL -14 -7196 NULL -1552199500 -14 -7196 NULL 11 NULL NULL -14 -14 -14 -14.0 -7196.0 NULL -1.5521995E9 -14.0 -7196.0 NULL 11.065 NULL NULL NULL -7196.0 1969-12-31 15:59:59.986 1969-12-31 15:59:52.804 NULL 1969-12-13 16:50:00.5 1969-12-31 15:59:46 1969-12-31 14:00:04 NULL 1969-12-31 16:00:00 1969-12-31 16:00:11.065 NULL NULL -14 -7196 NULL -1552199500 -14.0 -7196.0 NULL 0 1969-12-31 16:00:11.065 NULL NULL NULL -14.0 NULL -0.9906073556948704 NULL
    +true true NULL true true true NULL false true NULL -21 -7196 NULL 1542429000 -21 -7196 NULL -4 NULL NULL -21 -21 -21 -21.0 -7196.0 NULL 1.542429E9 -21.0 -7196.0 NULL -4.1 NULL NULL NULL -7196.0 1969-12-31 15:59:59.979 1969-12-31 15:59:52.804 NULL 1970-01-18 12:27:09 1969-12-31 15:59:39 1969-12-31 14:00:04 NULL 1969-12-31 16:00:00 1969-12-31 15:59:55.9 NULL NULL -21 -7196 NULL 1542429000 -21.0 -7196.0 NULL 0 1969-12-31 15:59:55.9 NULL NULL NULL -21.0 NULL -0.8366556385360561 NULL
    +true true NULL true true true NULL false true NULL -24 -7196 NULL 829111000 -24 -7196 NULL -6 NULL NULL -24 -24 -24 -24.0 -7196.0 NULL 8.29111E8 -24.0 -7196.0 NULL -6.855 NULL NULL NULL -7196.0 1969-12-31 15:59:59.976 1969-12-31 15:59:52.804 NULL 1970-01-10 06:18:31 1969-12-31 15:59:36 1969-12-31 14:00:04 NULL 1969-12-31 16:00:00 1969-12-31 15:59:53.145 NULL NULL -24 -7196 NULL 829111000 -24.0 -7196.0 NULL 0 1969-12-31 15:59:53.145 NULL NULL NULL -24.0 NULL 0.9055783620066238 NULL
    +true true NULL true true true NULL false true NULL -30 -200 NULL 1429852250 -30 -200 NULL 12 NULL NULL -30 -30 -30 -30.0 -200.0 NULL 1.42985225E9 -30.0 -200.0 NULL 12.935 NULL NULL NULL -200.0 1969-12-31 15:59:59.97 1969-12-31 15:59:59.8 NULL 1970-01-17 05:10:52.25 1969-12-31 15:59:30 1969-12-31 15:56:40 NULL 1969-12-31 16:00:00 1969-12-31 16:00:12.935 NULL NULL -30 -200 NULL 1429852250 -30.0 -200.0 NULL 0 1969-12-31 16:00:12.935 NULL NULL NULL -30.0 NULL 0.9880316240928618 NULL
    +true true NULL true true true NULL false true NULL -36 -200 NULL -2006216750 -36 -200 NULL -14 NULL NULL -36 -36 -36 -36.0 -200.0 NULL -2.00621675E9 -36.0 -200.0 NULL -14.252 NULL NULL NULL -200.0 1969-12-31 15:59:59.964 1969-12-31 15:59:59.8 NULL 1969-12-08 10:43:03.25 1969-12-31 15:59:24 1969-12-31 15:56:40 NULL 1969-12-31 16:00:00 1969-12-31 15:59:45.748 NULL NULL -36 -200 NULL -2006216750 -36.0 -200.0 NULL 0 1969-12-31 15:59:45.748 NULL NULL NULL -36.0 NULL 0.9917788534431158 NULL
    +true true NULL true true true NULL false true NULL -36 -200 NULL 1599879000 -36 -200 NULL -6 NULL NULL -36 -36 -36 -36.0 -200.0 NULL 1.599879E9 -36.0 -200.0 NULL -6.183 NULL NULL NULL -200.0 1969-12-31 15:59:59.964 1969-12-31 15:59:59.8 NULL 1970-01-19 04:24:39 1969-12-31 15:59:24 1969-12-31 15:56:40 NULL 1969-12-31 16:00:00 1969-12-31 15:59:53.817 NULL NULL -36 -200 NULL 1599879000 -36.0 -200.0 NULL 0 1969-12-31 15:59:53.817 NULL NULL NULL -36.0 NULL 0.9917788534431158 NULL
    +true true NULL true true true NULL false true NULL -38 15601 NULL -1858689000 -38 15601 NULL -1 NULL NULL -38 -38 -38 -38.0 15601.0 NULL -1.858689E9 -38.0 15601.0 NULL -1.386 NULL NULL NULL 15601.0 1969-12-31 15:59:59.962 1969-12-31 16:00:15.601 NULL 1969-12-10 03:41:51 1969-12-31 15:59:22 1969-12-31 20:20:01 NULL 1969-12-31 16:00:00 1969-12-31 15:59:58.614 NULL NULL -38 15601 NULL -1858689000 -38.0 15601.0 NULL 0 1969-12-31 15:59:58.614 NULL NULL NULL -38.0 NULL -0.2963685787093853 NULL
    +true true NULL true true true NULL false true NULL -5 15601 NULL 612416000 -5 15601 NULL 4 NULL NULL -5 -5 -5 -5.0 15601.0 NULL 6.12416E8 -5.0 15601.0 NULL 4.679 NULL NULL NULL 15601.0 1969-12-31 15:59:59.995 1969-12-31 16:00:15.601 NULL 1970-01-07 18:06:56 1969-12-31 15:59:55 1969-12-31 20:20:01 NULL 1969-12-31 16:00:00 1969-12-31 16:00:04.679 NULL NULL -5 15601 NULL 612416000 -5.0 15601.0 NULL 0 1969-12-31 16:00:04.679 NULL NULL NULL -5.0 NULL 0.9589242746631385 NULL
    +true true NULL true true true NULL false true NULL -50 -7196 NULL -1031187250 -50 -7196 NULL -5 NULL NULL -50 -50 -50 -50.0 -7196.0 NULL -1.03118725E9 -50.0 -7196.0 NULL -5.267 NULL NULL NULL -7196.0 1969-12-31 15:59:59.95 1969-12-31 15:59:52.804 NULL 1969-12-19 17:33:32.75 1969-12-31 15:59:10 1969-12-31 14:00:04 NULL 1969-12-31 16:00:00 1969-12-31 15:59:54.733 NULL NULL -50 -7196 NULL -1031187250 -50.0 -7196.0 NULL 0 1969-12-31 15:59:54.733 NULL NULL NULL -50.0 NULL 0.26237485370392877 NULL
    +true true NULL true true true NULL false true NULL -59 -7196 NULL -1604890000 -59 -7196 NULL 13 NULL NULL -59 -59 -59 -59.0 -7196.0 NULL -1.60489E9 -59.0 -7196.0 NULL 13.15 NULL NULL NULL -7196.0 1969-12-31 15:59:59.941 1969-12-31 15:59:52.804 NULL 1969-12-13 02:11:50 1969-12-31 15:59:01 1969-12-31 14:00:04 NULL 1969-12-31 16:00:00 1969-12-31 16:00:13.15 NULL NULL -59 -7196 NULL -1604890000 -59.0 -7196.0 NULL 0 1969-12-31 16:00:13.15 NULL NULL NULL -59.0 NULL -0.6367380071391379 NULL
    +true true NULL true true true NULL false true NULL -60 -7196 NULL 1516314750 -60 -7196 NULL -7 NULL NULL -60 -60 -60 -60.0 -7196.0 NULL 1.51631475E9 -60.0 -7196.0 NULL -7.592 NULL NULL NULL -7196.0 1969-12-31 15:59:59.94 1969-12-31 15:59:52.804 NULL 1970-01-18 05:11:54.75 1969-12-31 15:59:00 1969-12-31 14:00:04 NULL 1969-12-31 16:00:00 1969-12-31 15:59:52.408 NULL NULL -60 -7196 NULL 1516314750 -60.0 -7196.0 NULL 0 1969-12-31 15:59:52.408 NULL NULL NULL -60.0 NULL 0.3048106211022167 NULL
    +true true NULL true true true NULL false true NULL -8 -7196 NULL -1849991500 -8 -7196 NULL 3 NULL NULL -8 -8 -8 -8.0 -7196.0 NULL -1.8499915E9 -8.0 -7196.0 NULL 3.136 NULL NULL NULL -7196.0 1969-12-31 15:59:59.992 1969-12-31 15:59:52.804 NULL 1969-12-10 06:06:48.5 1969-12-31 15:59:52 1969-12-31 14:00:04 NULL 1969-12-31 16:00:00 1969-12-31 16:00:03.136 NULL NULL -8 -7196 NULL -1849991500 -8.0 -7196.0 NULL 0 1969-12-31 16:00:03.136 NULL NULL NULL -8.0 NULL -0.9893582466233818 NULL
    +true true NULL true true true NULL false true NULL 20 15601 NULL -362433250 20 15601 NULL -14 NULL NULL 20 20 20 20.0 15601.0 NULL -3.6243325E8 20.0 15601.0 NULL -14.871 NULL NULL NULL 15601.0 1969-12-31 16:00:00.02 1969-12-31 16:00:15.601 NULL 1969-12-27 11:19:26.75 1969-12-31 16:00:20 1969-12-31 20:20:01 NULL 1969-12-31 16:00:00 1969-12-31 15:59:45.129 NULL NULL 20 15601 NULL -362433250 20.0 15601.0 NULL 0 1969-12-31 15:59:45.129 NULL NULL NULL 20.0 NULL 0.9129452507276277 NULL
    +true true NULL true true true NULL false true NULL 48 15601 NULL -795361000 48 15601 NULL -9 NULL NULL 48 48 48 48.0 15601.0 NULL -7.95361E8 48.0 15601.0 NULL -9.765 NULL NULL NULL 15601.0 1969-12-31 16:00:00.048 1969-12-31 16:00:15.601 NULL 1969-12-22 11:03:59 1969-12-31 16:00:48 1969-12-31 20:20:01 NULL 1969-12-31 16:00:00 1969-12-31 15:59:50.235 NULL NULL 48 15601 NULL -795361000 48.0 15601.0 NULL 0 1969-12-31 15:59:50.235 NULL NULL NULL 48.0 NULL -0.7682546613236668 NULL
    +true true NULL true true true NULL false true NULL 5 -7196 NULL -1015607500 5 -7196 NULL 10 NULL NULL 5 5 5 5.0 -7196.0 NULL -1.0156075E9 5.0 -7196.0 NULL 10.973 NULL NULL NULL -7196.0 1969-12-31 16:00:00.005 1969-12-31 15:59:52.804 NULL 1969-12-19 21:53:12.5 1969-12-31 16:00:05 1969-12-31 14:00:04 NULL 1969-12-31 16:00:00 1969-12-31 16:00:10.973 NULL NULL 5 -7196 NULL -1015607500 5.0 -7196.0 NULL 0 1969-12-31 16:00:10.973 NULL NULL NULL 5.0 NULL -0.9589242746631385 NULL
    +true true NULL true true true NULL false true NULL 59 -7196 NULL -1137754500 59 -7196 NULL 10 NULL NULL 59 59 59 59.0 -7196.0 NULL -1.1377545E9 59.0 -7196.0 NULL 10.956 NULL NULL NULL -7196.0 1969-12-31 16:00:00.059 1969-12-31 15:59:52.804 NULL 1969-12-18 11:57:25.5 1969-12-31 16:00:59 1969-12-31 14:00:04 NULL 1969-12-31 16:00:00 1969-12-31 16:00:10.956 NULL NULL 59 -7196 NULL -1137754500 59.0 -7196.0 NULL 0 1969-12-31 16:00:10.956 NULL NULL NULL 59.0 NULL 0.6367380071391379 NULL
  • Sershe at Aug 25, 2015 at 9:28 pm
    HIVE-11581: HiveServer2 should store connection params in ZK when using dynamic service discovery for simpler client connection string (Vaibhav Gumashta reviewed by Thejas Nair)


    Project: http://git-wip-us.apache.org/repos/asf/hive/repo
    Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/0e54991d
    Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/0e54991d
    Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/0e54991d

    Branch: refs/heads/llap
    Commit: 0e54991d897c9acc26b015b6df82b44c0c90c6fb
    Parents: dd2bdfc
    Author: Vaibhav Gumashta <vgumashta@apache.org>
    Authored: Mon Aug 24 17:14:27 2015 -0700
    Committer: Vaibhav Gumashta <vgumashta@apache.org>
    Committed: Mon Aug 24 17:15:22 2015 -0700

    ----------------------------------------------------------------------
      .../org/apache/hadoop/hive/conf/HiveConf.java | 72 ++++++------
      .../org/apache/hive/jdbc/HiveConnection.java | 4 +-
      jdbc/src/java/org/apache/hive/jdbc/Utils.java | 117 +++++++------------
      .../hive/jdbc/ZooKeeperHiveClientHelper.java | 104 ++++++++++++++---
      .../apache/hive/service/server/HiveServer2.java | 74 +++++++++++-
      5 files changed, 239 insertions(+), 132 deletions(-)
    ----------------------------------------------------------------------


    http://git-wip-us.apache.org/repos/asf/hive/blob/0e54991d/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
    ----------------------------------------------------------------------
    diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
    index da171b1..8706a2d 100644
    --- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
    +++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
    @@ -1683,22 +1683,6 @@ public class HiveConf extends Configuration {
              "to construct a list exception handlers to handle exceptions thrown\n" +
              "by record readers"),

    - // operation log configuration
    - HIVE_SERVER2_LOGGING_OPERATION_ENABLED("hive.server2.logging.operation.enabled", true,
    - "When true, HS2 will save operation logs and make them available for clients"),
    - HIVE_SERVER2_LOGGING_OPERATION_LOG_LOCATION("hive.server2.logging.operation.log.location",
    - "${system:java.io.tmpdir}" + File.separator + "${system:user.name}" + File.separator +
    - "operation_logs",
    - "Top level directory where operation logs are stored if logging functionality is enabled"),
    - HIVE_SERVER2_LOGGING_OPERATION_LEVEL("hive.server2.logging.operation.level", "EXECUTION",
    - new StringSet("NONE", "EXECUTION", "PERFORMANCE", "VERBOSE"),
    - "HS2 operation logging mode available to clients to be set at session level.\n" +
    - "For this to work, hive.server2.logging.operation.enabled should be set to true.\n" +
    - " NONE: Ignore any logging\n" +
    - " EXECUTION: Log completion of tasks\n" +
    - " PERFORMANCE: Execution + Performance logs \n" +
    - " VERBOSE: All logs" ),
    - HIVE_SERVER2_METRICS_ENABLED("hive.server2.metrics.enabled", false, "Enable metrics on the HiveServer2."),
          // logging configuration
          HIVE_LOG4J_FILE("hive.log4j.file", "",
              "Hive log4j configuration file.\n" +
    @@ -1790,6 +1774,7 @@ public class HiveConf extends Configuration {
              "hive.zookeeper.quorum in their connection string."),
          HIVE_SERVER2_ZOOKEEPER_NAMESPACE("hive.server2.zookeeper.namespace", "hiveserver2",
              "The parent node in ZooKeeper used by HiveServer2 when supporting dynamic service discovery."),
    +
          // HiveServer2 global init file location
          HIVE_SERVER2_GLOBAL_INIT_FILE_LOCATION("hive.server2.global.init.file.location", "${env:HIVE_CONF_DIR}",
              "Either the location of a HS2 global init file or a directory containing a .hiverc file. If the \n" +
    @@ -1801,6 +1786,39 @@ public class HiveConf extends Configuration {
          HIVE_SERVER2_PARALLEL_COMPILATION("hive.driver.parallel.compilation", false, "Whether to\n" +
              "enable parallel compilation between sessions on HiveServer2. The default is false."),

    + // Tez session settings
    + HIVE_SERVER2_TEZ_DEFAULT_QUEUES("hive.server2.tez.default.queues", "",
    + "A list of comma separated values corresponding to YARN queues of the same name.\n" +
    + "When HiveServer2 is launched in Tez mode, this configuration needs to be set\n" +
    + "for multiple Tez sessions to run in parallel on the cluster."),
    + HIVE_SERVER2_TEZ_SESSIONS_PER_DEFAULT_QUEUE("hive.server2.tez.sessions.per.default.queue", 1,
    + "A positive integer that determines the number of Tez sessions that should be\n" +
    + "launched on each of the queues specified by \"hive.server2.tez.default.queues\".\n" +
    + "Determines the parallelism on each queue."),
    + HIVE_SERVER2_TEZ_INITIALIZE_DEFAULT_SESSIONS("hive.server2.tez.initialize.default.sessions", false,
    + "This flag is used in HiveServer2 to enable a user to use HiveServer2 without\n" +
    + "turning on Tez for HiveServer2. The user could potentially want to run queries\n" +
    + "over Tez without the pool of sessions."),
    +
    + // Operation log configuration
    + HIVE_SERVER2_LOGGING_OPERATION_ENABLED("hive.server2.logging.operation.enabled", true,
    + "When true, HS2 will save operation logs and make them available for clients"),
    + HIVE_SERVER2_LOGGING_OPERATION_LOG_LOCATION("hive.server2.logging.operation.log.location",
    + "${system:java.io.tmpdir}" + File.separator + "${system:user.name}" + File.separator +
    + "operation_logs",
    + "Top level directory where operation logs are stored if logging functionality is enabled"),
    + HIVE_SERVER2_LOGGING_OPERATION_LEVEL("hive.server2.logging.operation.level", "EXECUTION",
    + new StringSet("NONE", "EXECUTION", "PERFORMANCE", "VERBOSE"),
    + "HS2 operation logging mode available to clients to be set at session level.\n" +
    + "For this to work, hive.server2.logging.operation.enabled should be set to true.\n" +
    + " NONE: Ignore any logging\n" +
    + " EXECUTION: Log completion of tasks\n" +
    + " PERFORMANCE: Execution + Performance logs \n" +
    + " VERBOSE: All logs" ),
    +
    + // Enable metric collection for HiveServer2
    + HIVE_SERVER2_METRICS_ENABLED("hive.server2.metrics.enabled", false, "Enable metrics on the HiveServer2."),
    +
          // http (over thrift) transport settings
          HIVE_SERVER2_THRIFT_HTTP_PORT("hive.server2.thrift.http.port", 10001,
              "Port number of HiveServer2 Thrift interface when hive.server2.transport.mode is 'http'."),
    @@ -1816,7 +1834,7 @@ public class HiveConf extends Configuration {
              "Keepalive time for an idle http worker thread. When the number of workers exceeds min workers, " +
              "excessive threads are killed after this time interval."),

    - // Cookie based authentication
    + // Cookie based authentication when using HTTP Transport
          HIVE_SERVER2_THRIFT_HTTP_COOKIE_AUTH_ENABLED("hive.server2.thrift.http.cookie.auth.enabled", true,
              "When true, HiveServer2 in HTTP transport mode, will use cookie based authentication mechanism."),
          HIVE_SERVER2_THRIFT_HTTP_COOKIE_MAX_AGE("hive.server2.thrift.http.cookie.max.age", "86400s",
    @@ -1963,6 +1981,8 @@ public class HiveConf extends Configuration {
              " HIVE : Exposes Hive's native table types like MANAGED_TABLE, EXTERNAL_TABLE, VIRTUAL_VIEW\n" +
              " CLASSIC : More generic types like TABLE and VIEW"),
          HIVE_SERVER2_SESSION_HOOK("hive.server2.session.hook", "", ""),
    +
    + // SSL settings
          HIVE_SERVER2_USE_SSL("hive.server2.use.SSL", false,
              "Set this to true for using SSL encryption in HiveServer2."),
          HIVE_SERVER2_SSL_KEYSTORE_PATH("hive.server2.keystore.path", "",
    @@ -1983,9 +2003,6 @@ public class HiveConf extends Configuration {
               "Comma separated list of udfs names. These udfs will not be allowed in queries." +
               " The udf black list takes precedence over udf white list"),

    - HIVE_SECURITY_COMMAND_WHITELIST("hive.security.command.whitelist", "set,reset,dfs,add,list,delete,reload,compile",
    - "Comma separated list of non-SQL Hive commands users are authorized to execute"),
    -
          HIVE_SERVER2_SESSION_CHECK_INTERVAL("hive.server2.session.check.interval", "6h",
              new TimeValidator(TimeUnit.MILLISECONDS, 3000l, true, null, false),
              "The check interval for session/operation timeout, which can be disabled by setting to zero or negative value."),
    @@ -2002,6 +2019,8 @@ public class HiveConf extends Configuration {
              " This setting takes effect only if session idle timeout (hive.server2.idle.session.timeout) and checking\n" +
              "(hive.server2.session.check.interval) are enabled."),

    + HIVE_SECURITY_COMMAND_WHITELIST("hive.security.command.whitelist", "set,reset,dfs,add,list,delete,reload,compile",
    + "Comma separated list of non-SQL Hive commands users are authorized to execute"),
          HIVE_CONF_RESTRICTED_LIST("hive.conf.restricted.list",
              "hive.security.authenticator.manager,hive.security.authorization.manager,hive.users.in.admin.role",
              "Comma separated list of configuration options which are immutable at runtime"),
    @@ -2127,19 +2146,6 @@ public class HiveConf extends Configuration {
          HIVECOUNTERGROUP("hive.counters.group.name", "HIVE",
              "The name of counter group for internal Hive variables (CREATED_FILE, FATAL_ERROR, etc.)"),

    - HIVE_SERVER2_TEZ_DEFAULT_QUEUES("hive.server2.tez.default.queues", "",
    - "A list of comma separated values corresponding to YARN queues of the same name.\n" +
    - "When HiveServer2 is launched in Tez mode, this configuration needs to be set\n" +
    - "for multiple Tez sessions to run in parallel on the cluster."),
    - HIVE_SERVER2_TEZ_SESSIONS_PER_DEFAULT_QUEUE("hive.server2.tez.sessions.per.default.queue", 1,
    - "A positive integer that determines the number of Tez sessions that should be\n" +
    - "launched on each of the queues specified by \"hive.server2.tez.default.queues\".\n" +
    - "Determines the parallelism on each queue."),
    - HIVE_SERVER2_TEZ_INITIALIZE_DEFAULT_SESSIONS("hive.server2.tez.initialize.default.sessions", false,
    - "This flag is used in HiveServer2 to enable a user to use HiveServer2 without\n" +
    - "turning on Tez for HiveServer2. The user could potentially want to run queries\n" +
    - "over Tez without the pool of sessions."),
    -
          HIVE_QUOTEDID_SUPPORT("hive.support.quoted.identifiers", "column",
              new StringSet("none", "column"),
              "Whether to use quoted identifier. 'none' or 'column' can be used. \n" +

    http://git-wip-us.apache.org/repos/asf/hive/blob/0e54991d/jdbc/src/java/org/apache/hive/jdbc/HiveConnection.java
    ----------------------------------------------------------------------
    diff --git a/jdbc/src/java/org/apache/hive/jdbc/HiveConnection.java b/jdbc/src/java/org/apache/hive/jdbc/HiveConnection.java
    index a9dac03..ba971fd 100644
    --- a/jdbc/src/java/org/apache/hive/jdbc/HiveConnection.java
    +++ b/jdbc/src/java/org/apache/hive/jdbc/HiveConnection.java
    @@ -211,13 +211,13 @@ public class HiveConnection implements java.sql.Connection {
              break;
            } catch (TTransportException e) {
              LOG.info("Could not open client transport with JDBC Uri: " + jdbcUriString);
    - // We'll retry till we exhaust all HiveServer2 uris from ZooKeeper
    + // We'll retry till we exhaust all HiveServer2 nodes from ZooKeeper
              if ((sessConfMap.get(JdbcConnectionParams.SERVICE_DISCOVERY_MODE) != null)
                  && (JdbcConnectionParams.SERVICE_DISCOVERY_MODE_ZOOKEEPER.equalsIgnoreCase(sessConfMap
                      .get(JdbcConnectionParams.SERVICE_DISCOVERY_MODE)))) {
                try {
                  // Update jdbcUriString, host & port variables in connParams
    - // Throw an exception if all HiveServer2 uris have been exhausted,
    + // Throw an exception if all HiveServer2 nodes have been exhausted,
                  // or if we're unable to connect to ZooKeeper.
                  Utils.updateConnParamsFromZooKeeper(connParams);
                } catch (ZooKeeperHiveClientException ze) {

    http://git-wip-us.apache.org/repos/asf/hive/blob/0e54991d/jdbc/src/java/org/apache/hive/jdbc/Utils.java
    ----------------------------------------------------------------------
    diff --git a/jdbc/src/java/org/apache/hive/jdbc/Utils.java b/jdbc/src/java/org/apache/hive/jdbc/Utils.java
    index 0e4693b..d8368a4 100644
    --- a/jdbc/src/java/org/apache/hive/jdbc/Utils.java
    +++ b/jdbc/src/java/org/apache/hive/jdbc/Utils.java
    @@ -19,7 +19,6 @@
      package org.apache.hive.jdbc;

      import java.net.URI;
    -import java.net.URISyntaxException;
      import java.sql.SQLException;
      import java.util.ArrayList;
      import java.util.Arrays;
    @@ -37,22 +36,22 @@ import org.apache.hive.service.cli.thrift.TStatusCode;
      import org.apache.http.client.CookieStore;
      import org.apache.http.cookie.Cookie;

    -public class Utils {
    - public static final Log LOG = LogFactory.getLog(Utils.class.getName());
    +class Utils {
    + static final Log LOG = LogFactory.getLog(Utils.class.getName());
        /**
          * The required prefix for the connection URL.
          */
    - public static final String URL_PREFIX = "jdbc:hive2://";
    + static final String URL_PREFIX = "jdbc:hive2://";

        /**
          * If host is provided, without a port.
          */
    - public static final String DEFAULT_PORT = "10000";
    + static final String DEFAULT_PORT = "10000";

        /**
         * Hive's default database name
         */
    - public static final String DEFAULT_DATABASE = "default";
    + static final String DEFAULT_DATABASE = "default";

        private static final String URI_JDBC_PREFIX = "jdbc:";

    @@ -63,7 +62,7 @@ public class Utils {
        static final String HIVE_SERVER2_RETRY_TRUE = "true";
        static final String HIVE_SERVER2_RETRY_FALSE = "false";

    - public static class JdbcConnectionParams {
    + static class JdbcConnectionParams {
          // Note on client side parameter naming convention:
          // Prefer using a shorter camelCase param name instead of using the same name as the
          // corresponding
    @@ -129,7 +128,7 @@ public class Utils {
          static final String SSL_TRUST_STORE_TYPE = "JKS";

          private String host = null;
    - private int port;
    + private int port = 0;
          private String jdbcUriString;
          private String dbName = DEFAULT_DATABASE;
          private Map<String,String> hiveConfs = new LinkedHashMap<String,String>();
    @@ -238,17 +237,17 @@ public class Utils {
        }

        // Verify success or success_with_info status, else throw SQLException
    - public static void verifySuccessWithInfo(TStatus status) throws SQLException {
    + static void verifySuccessWithInfo(TStatus status) throws SQLException {
          verifySuccess(status, true);
        }

        // Verify success status, else throw SQLException
    - public static void verifySuccess(TStatus status) throws SQLException {
    + static void verifySuccess(TStatus status) throws SQLException {
          verifySuccess(status, false);
        }

        // Verify success and optionally with_info status, else throw SQLException
    - public static void verifySuccess(TStatus status, boolean withInfo) throws SQLException {
    + static void verifySuccess(TStatus status, boolean withInfo) throws SQLException {
          if (status.getStatusCode() == TStatusCode.SUCCESS_STATUS ||
              (withInfo && status.getStatusCode() == TStatusCode.SUCCESS_WITH_INFO_STATUS)) {
            return;
    @@ -279,7 +278,7 @@ public class Utils {
         * @return
         * @throws SQLException
         */
    - public static JdbcConnectionParams parseURL(String uri) throws JdbcUriParseException,
    + static JdbcConnectionParams parseURL(String uri) throws JdbcUriParseException,
            SQLException, ZooKeeperHiveClientException {
          JdbcConnectionParams connParams = new JdbcConnectionParams();

    @@ -383,7 +382,6 @@ public class Utils {
          newUsage = usageUrlBase + JdbcConnectionParams.HTTP_PATH + "=<http_path_value>";
          handleParamDeprecation(connParams.getHiveConfs(), connParams.getSessionVars(),
              JdbcConnectionParams.HTTP_PATH_DEPRECATED, JdbcConnectionParams.HTTP_PATH, newUsage);
    -
          // Extract host, port
          if (connParams.isEmbeddedMode()) {
            // In case of embedded mode we were supplied with an empty authority.
    @@ -391,23 +389,15 @@ public class Utils {
            connParams.setHost(jdbcURI.getHost());
            connParams.setPort(jdbcURI.getPort());
          } else {
    - // Else substitute the dummy authority with a resolved one.
    - // In case of dynamic service discovery using ZooKeeper, it picks a server uri from ZooKeeper
    - String resolvedAuthorityString = resolveAuthority(connParams);
    - LOG.info("Resolved authority: " + resolvedAuthorityString);
    - uri = uri.replace(dummyAuthorityString, resolvedAuthorityString);
    + // Configure host, port and params from ZooKeeper if used,
    + // and substitute the dummy authority with a resolved one
    + configureConnParams(connParams);
    + // We check for invalid host, port while configuring connParams with configureConnParams()
    + String authorityStr = connParams.getHost() + ":" + connParams.getPort();
    + LOG.info("Resolved authority: " + authorityStr);
    + uri = uri.replace(dummyAuthorityString, authorityStr);
            connParams.setJdbcUriString(uri);
    - // Create a Java URI from the resolved URI for extracting the host/port
    - URI resolvedAuthorityURI = null;
    - try {
    - resolvedAuthorityURI = new URI(null, resolvedAuthorityString, null, null, null);
    - } catch (URISyntaxException e) {
    - throw new JdbcUriParseException("Bad URL format: ", e);
    - }
    - connParams.setHost(resolvedAuthorityURI.getHost());
    - connParams.setPort(resolvedAuthorityURI.getPort());
          }
    -
          return connParams;
        }

    @@ -471,22 +461,17 @@ public class Utils {
          return authorities;
        }

    - /**
    - * Get a string representing a specific host:port
    - * @param connParams
    - * @return
    - * @throws JdbcUriParseException
    - * @throws ZooKeeperHiveClientException
    - */
    - private static String resolveAuthority(JdbcConnectionParams connParams)
    + private static void configureConnParams(JdbcConnectionParams connParams)
            throws JdbcUriParseException, ZooKeeperHiveClientException {
          String serviceDiscoveryMode =
              connParams.getSessionVars().get(JdbcConnectionParams.SERVICE_DISCOVERY_MODE);
          if ((serviceDiscoveryMode != null)
              && (JdbcConnectionParams.SERVICE_DISCOVERY_MODE_ZOOKEEPER
                  .equalsIgnoreCase(serviceDiscoveryMode))) {
    - // Resolve using ZooKeeper
    - return resolveAuthorityUsingZooKeeper(connParams);
    + // Set ZooKeeper ensemble in connParams for later use
    + connParams.setZooKeeperEnsemble(joinStringArray(connParams.getAuthorityList(), ","));
    + // Configure using ZooKeeper
    + ZooKeeperHiveClientHelper.configureConnParams(connParams);
          } else {
            String authority = connParams.getAuthorityList()[0];
            URI jdbcURI = URI.create(URI_HIVE_PREFIX + "//" + authority);
    @@ -494,32 +479,28 @@ public class Utils {
            // to separate the 'path' portion of URI can result in this.
            // The missing "/" common typo while using secure mode, eg of such url -
            // jdbc:hive2://localhost:10000;principal=hive/HiveServer2Host@YOUR-REALM.COM
    - if ((jdbcURI.getAuthority() != null) && (jdbcURI.getHost() == null)) {
    - throw new JdbcUriParseException("Bad URL format. Hostname not found "
    - + " in authority part of the url: " + jdbcURI.getAuthority()
    - + ". Are you missing a '/' after the hostname ?");
    + if (jdbcURI.getAuthority() != null) {
    + String host = jdbcURI.getHost();
    + int port = jdbcURI.getPort();
    + if (host == null) {
    + throw new JdbcUriParseException("Bad URL format. Hostname not found "
    + + " in authority part of the url: " + jdbcURI.getAuthority()
    + + ". Are you missing a '/' after the hostname ?");
    + }
    + // Set the port to default value; we do support jdbc url like:
    + // jdbc:hive2://localhost/db
    + if (port <= 0) {
    + port = Integer.parseInt(Utils.DEFAULT_PORT);
    + }
    + connParams.setHost(jdbcURI.getHost());
    + connParams.setPort(jdbcURI.getPort());
            }
    - // Return the 1st element of the array
    - return jdbcURI.getAuthority();
          }
        }

        /**
    - * Read a specific host:port from ZooKeeper
    - * @param connParams
    - * @return
    - * @throws ZooKeeperHiveClientException
    - */
    - private static String resolveAuthorityUsingZooKeeper(JdbcConnectionParams connParams)
    - throws ZooKeeperHiveClientException {
    - // Set ZooKeeper ensemble in connParams for later use
    - connParams.setZooKeeperEnsemble(joinStringArray(connParams.getAuthorityList(), ","));
    - return ZooKeeperHiveClientHelper.getNextServerUriFromZooKeeper(connParams);
    - }
    -
    - /**
         * Read the next server coordinates (host:port combo) from ZooKeeper. Ignore the znodes already
    - * explored. Also update the host, port, jdbcUriString fields of connParams.
    + * explored. Also update the host, port, jdbcUriString and other configs published by the server.
         *
         * @param connParams
         * @throws ZooKeeperHiveClientException
    @@ -528,25 +509,13 @@ public class Utils {
            throws ZooKeeperHiveClientException {
          // Add current host to the rejected list
          connParams.getRejectedHostZnodePaths().add(connParams.getCurrentHostZnodePath());
    - // Get another HiveServer2 uri from ZooKeeper
    - String serverUriString = ZooKeeperHiveClientHelper.getNextServerUriFromZooKeeper(connParams);
    - // Parse serverUri to a java URI and extract host, port
    - URI serverUri = null;
    - try {
    - // Note URL_PREFIX is not a valid scheme format, therefore leaving it null in the constructor
    - // to construct a valid URI
    - serverUri = new URI(null, serverUriString, null, null, null);
    - } catch (URISyntaxException e) {
    - throw new ZooKeeperHiveClientException(e);
    - }
          String oldServerHost = connParams.getHost();
          int oldServerPort = connParams.getPort();
    - String newServerHost = serverUri.getHost();
    - int newServerPort = serverUri.getPort();
    - connParams.setHost(newServerHost);
    - connParams.setPort(newServerPort);
    + // Update connection params (including host, port) from ZooKeeper
    + ZooKeeperHiveClientHelper.configureConnParams(connParams);
          connParams.setJdbcUriString(connParams.getJdbcUriString().replace(
    - oldServerHost + ":" + oldServerPort, newServerHost + ":" + newServerPort));
    + oldServerHost + ":" + oldServerPort, connParams.getHost() + ":" + connParams.getPort()));
    + LOG.info("Selected HiveServer2 instance with uri: " + connParams.getJdbcUriString());
        }

        private static String joinStringArray(String[] stringArray, String seperator) {

    http://git-wip-us.apache.org/repos/asf/hive/blob/0e54991d/jdbc/src/java/org/apache/hive/jdbc/ZooKeeperHiveClientHelper.java
    ----------------------------------------------------------------------
    diff --git a/jdbc/src/java/org/apache/hive/jdbc/ZooKeeperHiveClientHelper.java b/jdbc/src/java/org/apache/hive/jdbc/ZooKeeperHiveClientHelper.java
    index e24b3dc..eeb3cf9 100644
    --- a/jdbc/src/java/org/apache/hive/jdbc/ZooKeeperHiveClientHelper.java
    +++ b/jdbc/src/java/org/apache/hive/jdbc/ZooKeeperHiveClientHelper.java
    @@ -19,9 +19,10 @@
      package org.apache.hive.jdbc;

      import java.nio.charset.Charset;
    -import java.sql.SQLException;
      import java.util.List;
      import java.util.Random;
    +import java.util.regex.Matcher;
    +import java.util.regex.Pattern;

      import org.apache.commons.logging.Log;
      import org.apache.commons.logging.LogFactory;
    @@ -31,26 +32,19 @@ import org.apache.curator.retry.ExponentialBackoffRetry;
      import org.apache.hive.jdbc.Utils.JdbcConnectionParams;
      import org.apache.zookeeper.Watcher;

    -public class ZooKeeperHiveClientHelper {
    - public static final Log LOG = LogFactory.getLog(ZooKeeperHiveClientHelper.class.getName());
    -
    +class ZooKeeperHiveClientHelper {
    + static final Log LOG = LogFactory.getLog(ZooKeeperHiveClientHelper.class.getName());
    + // Pattern for key1=value1;key2=value2
    + private static final Pattern kvPattern = Pattern.compile("([^=;]*)=([^;]*)[;]?");
        /**
         * A no-op watcher class
         */
    - public static class DummyWatcher implements Watcher {
    + static class DummyWatcher implements Watcher {
          public void process(org.apache.zookeeper.WatchedEvent event) {
          }
        }

    - /**
    - * Resolve to a host:port by connecting to ZooKeeper and picking a host randomly.
    - *
    - * @param uri
    - * @param connParams
    - * @return
    - * @throws SQLException
    - */
    - static String getNextServerUriFromZooKeeper(JdbcConnectionParams connParams)
    + static void configureConnParams(JdbcConnectionParams connParams)
            throws ZooKeeperHiveClientException {
          String zooKeeperEnsemble = connParams.getZooKeeperEnsemble();
          String zooKeeperNamespace =
    @@ -73,17 +67,17 @@ public class ZooKeeperHiveClientHelper {
              throw new ZooKeeperHiveClientException(
                  "Tried all existing HiveServer2 uris from ZooKeeper.");
            }
    - // Now pick a host randomly
    + // Now pick a server node randomly
            serverNode = serverHosts.get(randomizer.nextInt(serverHosts.size()));
            connParams.setCurrentHostZnodePath(serverNode);
    - String serverUri =
    + // Read config string from the znode for this server node
    + String serverConfStr =
                new String(
                    zooKeeperClient.getData().forPath("/" + zooKeeperNamespace + "/" + serverNode),
                    Charset.forName("UTF-8"));
    - LOG.info("Selected HiveServer2 instance with uri: " + serverUri);
    - return serverUri;
    + applyConfs(serverConfStr, connParams);
          } catch (Exception e) {
    - throw new ZooKeeperHiveClientException("Unable to read HiveServer2 uri from ZooKeeper", e);
    + throw new ZooKeeperHiveClientException("Unable to read HiveServer2 configs from ZooKeeper", e);
          } finally {
            // Close the client connection with ZooKeeper
            if (zooKeeperClient != null) {
    @@ -91,4 +85,76 @@ public class ZooKeeperHiveClientHelper {
            }
          }
        }
    +
    + /**
    + * Apply configs published by the server. Configs specified from client's JDBC URI override
    + * configs published by the server.
    + *
    + * @param serverConfStr
    + * @param connParams
    + * @throws Exception
    + */
    + private static void applyConfs(String serverConfStr, JdbcConnectionParams connParams)
    + throws Exception {
    + Matcher matcher = kvPattern.matcher(serverConfStr);
    + while (matcher.find()) {
    + // Have to use this if-else since switch-case on String is supported Java 7 onwards
    + if ((matcher.group(1) != null)) {
    + if ((matcher.group(2) == null)) {
    + throw new Exception("Null config value for: " + matcher.group(1)
    + + " published by the server.");
    + }
    + // Set host
    + if ((matcher.group(1).equals("hive.server2.thrift.bind.host"))
    + && (connParams.getHost() == null)) {
    + connParams.setHost(matcher.group(2));
    + }
    + // Set transportMode
    + if ((matcher.group(1).equals("hive.server2.transport.mode"))
    + && !(connParams.getSessionVars().containsKey(JdbcConnectionParams.TRANSPORT_MODE))) {
    + connParams.getSessionVars().put(JdbcConnectionParams.TRANSPORT_MODE, matcher.group(2));
    + }
    + // Set port
    + if ((matcher.group(1).equals("hive.server2.thrift.port")) && !(connParams.getPort() > 0)) {
    + connParams.setPort(Integer.parseInt(matcher.group(2)));
    + }
    + if ((matcher.group(1).equals("hive.server2.thrift.http.port"))
    + && !(connParams.getPort() > 0)) {
    + connParams.setPort(Integer.parseInt(matcher.group(2)));
    + }
    + // Set sasl qop
    + if ((matcher.group(1).equals("hive.server2.thrift.sasl.qop"))
    + && !(connParams.getSessionVars().containsKey(JdbcConnectionParams.AUTH_QOP))) {
    + connParams.getSessionVars().put(JdbcConnectionParams.AUTH_QOP, matcher.group(2));
    + }
    + // Set http path
    + if ((matcher.group(1).equals("hive.server2.thrift.http.path"))
    + && !(connParams.getSessionVars().containsKey(JdbcConnectionParams.HTTP_PATH))) {
    + connParams.getSessionVars().put(JdbcConnectionParams.HTTP_PATH, "/" + matcher.group(2));
    + }
    + // Set SSL
    + if ((matcher.group(1) != null) && (matcher.group(1).equals("hive.server2.use.SSL"))
    + && !(connParams.getSessionVars().containsKey(JdbcConnectionParams.USE_SSL))) {
    + connParams.getSessionVars().put(JdbcConnectionParams.USE_SSL, matcher.group(2));
    + }
    + // Set authentication configs
    + // Note that in JDBC driver, we have 3 auth modes: NOSASL, Kerberos and password based
    + // The use of "JdbcConnectionParams.AUTH_TYPE=JdbcConnectionParams.AUTH_SIMPLE" picks NOSASL
    + // The presence of "JdbcConnectionParams.AUTH_PRINCIPAL=<principal>" picks Kerberos
    + // Otherwise password based (which includes NONE, PAM, LDAP, CUSTOM)
    + if ((matcher.group(1).equals("hive.server2.authentication"))
    + && !(connParams.getSessionVars().containsKey(JdbcConnectionParams.AUTH_TYPE))) {
    + if (matcher.group(2).equalsIgnoreCase("NOSASL")) {
    + connParams.getSessionVars().put(JdbcConnectionParams.AUTH_TYPE,
    + JdbcConnectionParams.AUTH_SIMPLE);
    + }
    + }
    + // Set server's kerberos principal
    + if ((matcher.group(1).equals("hive.server2.authentication.kerberos.principal"))
    + && !(connParams.getSessionVars().containsKey(JdbcConnectionParams.AUTH_PRINCIPAL))) {
    + connParams.getSessionVars().put(JdbcConnectionParams.AUTH_PRINCIPAL, matcher.group(2));
    + }
    + }
    + }
    + }
      }

    http://git-wip-us.apache.org/repos/asf/hive/blob/0e54991d/service/src/java/org/apache/hive/service/server/HiveServer2.java
    ----------------------------------------------------------------------
    diff --git a/service/src/java/org/apache/hive/service/server/HiveServer2.java b/service/src/java/org/apache/hive/service/server/HiveServer2.java
    index 4a4be97..d7ba964 100644
    --- a/service/src/java/org/apache/hive/service/server/HiveServer2.java
    +++ b/service/src/java/org/apache/hive/service/server/HiveServer2.java
    @@ -21,7 +21,9 @@ package org.apache.hive.service.server;
      import java.io.IOException;
      import java.nio.charset.Charset;
      import java.util.ArrayList;
    +import java.util.HashMap;
      import java.util.List;
    +import java.util.Map;
      import java.util.Properties;
      import java.util.concurrent.CountDownLatch;
      import java.util.concurrent.TimeUnit;
    @@ -69,6 +71,8 @@ import org.apache.zookeeper.ZooDefs.Ids;
      import org.apache.zookeeper.ZooDefs.Perms;
      import org.apache.zookeeper.data.ACL;

    +import com.google.common.base.Joiner;
    +
      /**
       * HiveServer2.
       *
    @@ -100,7 +104,12 @@ public class HiveServer2 extends CompositeService {
          }
          addService(thriftCLIService);
          super.init(hiveConf);
    -
    + // Set host name in hiveConf
    + try {
    + hiveConf.set(HiveConf.ConfVars.HIVE_SERVER2_THRIFT_BIND_HOST.varname, getServerHost());
    + } catch (Throwable t) {
    + throw new Error("Unable to intitialize HiveServer2", t);
    + }
          // Add a shutdown hook for catching SIGTERM & SIGINT
          final HiveServer2 hiveServer2 = this;
          Runtime.getRuntime().addShutdownHook(new Thread() {
    @@ -122,6 +131,14 @@ public class HiveServer2 extends CompositeService {
          return false;
        }

    + public static boolean isKerberosAuthMode(HiveConf hiveConf) {
    + String authMode = hiveConf.getVar(HiveConf.ConfVars.HIVE_SERVER2_AUTHENTICATION);
    + if (authMode != null && (authMode.equalsIgnoreCase("KERBEROS"))) {
    + return true;
    + }
    + return false;
    + }
    +
        /**
         * ACLProvider for providing appropriate ACLs to CuratorFrameworkFactory
         */
    @@ -158,9 +175,12 @@ public class HiveServer2 extends CompositeService {
        private void addServerInstanceToZooKeeper(HiveConf hiveConf) throws Exception {
          String zooKeeperEnsemble = ZooKeeperHiveHelper.getQuorumServers(hiveConf);
          String rootNamespace = hiveConf.getVar(HiveConf.ConfVars.HIVE_SERVER2_ZOOKEEPER_NAMESPACE);
    - String instanceURI = getServerInstanceURI(hiveConf);
    - byte[] znodeDataUTF8 = instanceURI.getBytes(Charset.forName("UTF-8"));
    + String instanceURI = getServerInstanceURI();
          setUpZooKeeperAuth(hiveConf);
    + // HiveServer2 configs that this instance will publish to ZooKeeper,
    + // so that the clients can read these and configure themselves properly.
    + Map<String, String> confsToPublish = new HashMap<String, String>();
    + addConfsToPublish(hiveConf, confsToPublish);
          int sessionTimeout =
              (int) hiveConf.getTimeVar(HiveConf.ConfVars.HIVE_ZOOKEEPER_SESSION_TIMEOUT,
                  TimeUnit.MILLISECONDS);
    @@ -193,6 +213,10 @@ public class HiveServer2 extends CompositeService {
                ZooKeeperHiveHelper.ZOOKEEPER_PATH_SEPARATOR + rootNamespace
                    + ZooKeeperHiveHelper.ZOOKEEPER_PATH_SEPARATOR + "serverUri=" + instanceURI + ";"
                    + "version=" + HiveVersionInfo.getVersion() + ";" + "sequence=";
    + String znodeData = "";
    + // Publish configs for this instance as the data on the node
    + znodeData = Joiner.on(';').withKeyValueSeparator("=").join(confsToPublish);
    + byte[] znodeDataUTF8 = znodeData.getBytes(Charset.forName("UTF-8"));
            znode =
                new PersistentEphemeralNode(zooKeeperClient,
                    PersistentEphemeralNode.Mode.EPHEMERAL_SEQUENTIAL, pathPrefix, znodeDataUTF8);
    @@ -220,6 +244,41 @@ public class HiveServer2 extends CompositeService {
        }

        /**
    + * Add conf keys, values that HiveServer2 will publish to ZooKeeper.
    + * @param hiveConf
    + */
    + private void addConfsToPublish(HiveConf hiveConf, Map<String, String> confsToPublish) {
    + // Hostname
    + confsToPublish.put(ConfVars.HIVE_SERVER2_THRIFT_BIND_HOST.varname,
    + hiveConf.getVar(ConfVars.HIVE_SERVER2_THRIFT_BIND_HOST));
    + // Transport mode
    + confsToPublish.put(ConfVars.HIVE_SERVER2_TRANSPORT_MODE.varname,
    + hiveConf.getVar(ConfVars.HIVE_SERVER2_TRANSPORT_MODE));
    + // Transport specific confs
    + if (isHTTPTransportMode(hiveConf)) {
    + confsToPublish.put(ConfVars.HIVE_SERVER2_THRIFT_HTTP_PORT.varname,
    + hiveConf.getVar(ConfVars.HIVE_SERVER2_THRIFT_HTTP_PORT));
    + confsToPublish.put(ConfVars.HIVE_SERVER2_THRIFT_HTTP_PATH.varname,
    + hiveConf.getVar(ConfVars.HIVE_SERVER2_THRIFT_HTTP_PATH));
    + } else {
    + confsToPublish.put(ConfVars.HIVE_SERVER2_THRIFT_PORT.varname,
    + hiveConf.getVar(ConfVars.HIVE_SERVER2_THRIFT_PORT));
    + confsToPublish.put(ConfVars.HIVE_SERVER2_THRIFT_SASL_QOP.varname,
    + hiveConf.getVar(ConfVars.HIVE_SERVER2_THRIFT_SASL_QOP));
    + }
    + // Auth specific confs
    + confsToPublish.put(ConfVars.HIVE_SERVER2_AUTHENTICATION.varname,
    + hiveConf.getVar(ConfVars.HIVE_SERVER2_AUTHENTICATION));
    + if (isKerberosAuthMode(hiveConf)) {
    + confsToPublish.put(ConfVars.HIVE_SERVER2_KERBEROS_PRINCIPAL.varname,
    + hiveConf.getVar(ConfVars.HIVE_SERVER2_KERBEROS_PRINCIPAL));
    + }
    + // SSL conf
    + confsToPublish.put(ConfVars.HIVE_SERVER2_USE_SSL.varname,
    + hiveConf.getVar(ConfVars.HIVE_SERVER2_USE_SSL));
    + }
    +
    + /**
         * For a kerberized cluster, we dynamically set up the client's JAAS conf.
         *
         * @param hiveConf
    @@ -289,7 +348,7 @@ public class HiveServer2 extends CompositeService {
          this.registeredWithZooKeeper = registeredWithZooKeeper;
        }

    - private String getServerInstanceURI(HiveConf hiveConf) throws Exception {
    + private String getServerInstanceURI() throws Exception {
          if ((thriftCLIService == null) || (thriftCLIService.getServerIPAddress() == null)) {
            throw new Exception("Unable to get the server address; it hasn't been initialized yet.");
          }
    @@ -297,6 +356,13 @@ public class HiveServer2 extends CompositeService {
              + thriftCLIService.getPortNumber();
        }

    + private String getServerHost() throws Exception {
    + if ((thriftCLIService == null) || (thriftCLIService.getServerIPAddress() == null)) {
    + throw new Exception("Unable to get the server address; it hasn't been initialized yet.");
    + }
    + return thriftCLIService.getServerIPAddress().getHostName();
    + }
    +
        @Override
        public synchronized void start() {
          super.start();
  • Sershe at Aug 25, 2015 at 9:28 pm
    HIVE-11501 : Make Connections.setReadOnly method compliant (Michal Wegrzyn via Sergey Shelukhin)


    Project: http://git-wip-us.apache.org/repos/asf/hive/repo
    Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/f7ebff00
    Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/f7ebff00
    Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/f7ebff00

    Branch: refs/heads/llap
    Commit: f7ebff0057c8b16f44d03feb96f8a58e2ee848b0
    Parents: 2de6fda
    Author: Michał Węgrzyn <michal.wegrzyn@gmail.com>
    Authored: Sat Aug 8 10:27:35 2015 +0200
    Committer: Ashutosh Chauhan <hashutosh@apache.org>
    Committed: Sun Aug 23 22:13:52 2015 -0700

    ----------------------------------------------------------------------
      jdbc/src/java/org/apache/hive/jdbc/HiveConnection.java | 12 ++++++++++--
      1 file changed, 10 insertions(+), 2 deletions(-)
    ----------------------------------------------------------------------


    http://git-wip-us.apache.org/repos/asf/hive/blob/f7ebff00/jdbc/src/java/org/apache/hive/jdbc/HiveConnection.java
    ----------------------------------------------------------------------
    diff --git a/jdbc/src/java/org/apache/hive/jdbc/HiveConnection.java b/jdbc/src/java/org/apache/hive/jdbc/HiveConnection.java
    index bb2b695..3b7e004 100644
    --- a/jdbc/src/java/org/apache/hive/jdbc/HiveConnection.java
    +++ b/jdbc/src/java/org/apache/hive/jdbc/HiveConnection.java
    @@ -1279,8 +1279,16 @@ public class HiveConnection implements java.sql.Connection {

        @Override
        public void setReadOnly(boolean readOnly) throws SQLException {
    - // TODO Auto-generated method stub
    - throw new SQLException("Method not supported");
    + // Per JDBC spec, if the connection is closed a SQLException should be thrown.
    + if (isClosed) {
    + throw new SQLException("Connection is closed");
    + }
    + // Per JDBC spec, the request defines a hint to the driver to enable database optimizations.
    + // The read-only mode for this connection is disabled and cannot be enabled (isReadOnly always returns false).
    + // The most correct behavior is to throw only if the request tries to enable the read-only mode.
    + if(readOnly) {
    + throw new SQLException("Enabling read-only mode not supported");
    + }
        }

        /*
  • Sershe at Aug 25, 2015 at 9:28 pm
    HIVE-10697 : ObjectInspectorConvertors#UnionConvertor does a faulty conversion (Swarnim Kulkarni, reviewed by Hari Subramaniyan)


    Project: http://git-wip-us.apache.org/repos/asf/hive/repo
    Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/2688b680
    Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/2688b680
    Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/2688b680

    Branch: refs/heads/llap
    Commit: 2688b6800f203ff5fd4b283e462594e9b4279975
    Parents: c3c2bda
    Author: Hari Subramaniyan <harisankar@apache.org>
    Authored: Wed Aug 19 10:25:14 2015 -0700
    Committer: Hari Subramaniyan <harisankar@apache.org>
    Committed: Wed Aug 19 10:25:14 2015 -0700

    ----------------------------------------------------------------------
      .../ObjectInspectorConverters.java | 31 +++----
      .../SettableUnionObjectInspector.java | 4 +-
      .../StandardUnionObjectInspector.java | 4 +-
      .../TestObjectInspectorConverters.java | 89 +++++++++++++++++++-
      4 files changed, 108 insertions(+), 20 deletions(-)
    ----------------------------------------------------------------------


    http://git-wip-us.apache.org/repos/asf/hive/blob/2688b680/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorConverters.java
    ----------------------------------------------------------------------
    diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorConverters.java b/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorConverters.java
    index 8ef8ce1..24b3d4e 100644
    --- a/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorConverters.java
    +++ b/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorConverters.java
    @@ -424,8 +424,9 @@ public final class ObjectInspectorConverters {
          UnionObjectInspector inputOI;
          SettableUnionObjectInspector outputOI;

    - List<? extends ObjectInspector> inputFields;
    - List<? extends ObjectInspector> outputFields;
    + // Object inspectors for the tags for the input and output unionss
    + List<? extends ObjectInspector> inputTagsOIs;
    + List<? extends ObjectInspector> outputTagsOIs;

          ArrayList<Converter> fieldConverters;

    @@ -436,14 +437,14 @@ public final class ObjectInspectorConverters {
            if (inputOI instanceof UnionObjectInspector) {
              this.inputOI = (UnionObjectInspector)inputOI;
              this.outputOI = outputOI;
    - inputFields = this.inputOI.getObjectInspectors();
    - outputFields = outputOI.getObjectInspectors();
    + inputTagsOIs = this.inputOI.getObjectInspectors();
    + outputTagsOIs = outputOI.getObjectInspectors();

              // If the output has some extra fields, set them to NULL in convert().
    - int minFields = Math.min(inputFields.size(), outputFields.size());
    + int minFields = Math.min(inputTagsOIs.size(), outputTagsOIs.size());
              fieldConverters = new ArrayList<Converter>(minFields);
              for (int f = 0; f < minFields; f++) {
    - fieldConverters.add(getConverter(inputFields.get(f), outputFields.get(f)));
    + fieldConverters.add(getConverter(inputTagsOIs.get(f), outputTagsOIs.get(f)));
              }

              // Create an empty output object which will be populated when convert() is invoked.
    @@ -461,18 +462,18 @@ public final class ObjectInspectorConverters {
              return null;
            }

    - int minFields = Math.min(inputFields.size(), outputFields.size());
    - // Convert the fields
    - for (int f = 0; f < minFields; f++) {
    - Object outputFieldValue = fieldConverters.get(f).convert(inputOI);
    - outputOI.addField(output, (ObjectInspector)outputFieldValue);
    - }
    + Object inputFieldValue = inputOI.getField(input);
    + Object inputFieldTag = inputOI.getTag(input);
    + Object outputFieldValue = null;

    - // set the extra fields to null
    - for (int f = minFields; f < outputFields.size(); f++) {
    - outputOI.addField(output, null);
    + int inputFieldTagIndex = ((Byte)inputFieldTag).intValue();
    +
    + if (inputFieldTagIndex >= 0 && inputFieldTagIndex < fieldConverters.size()) {
    + outputFieldValue = fieldConverters.get(inputFieldTagIndex).convert(inputFieldValue);
            }

    + outputOI.addField(output, outputFieldValue);
    +
            return output;
          }
        }

    http://git-wip-us.apache.org/repos/asf/hive/blob/2688b680/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/SettableUnionObjectInspector.java
    ----------------------------------------------------------------------
    diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/SettableUnionObjectInspector.java b/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/SettableUnionObjectInspector.java
    index a64aee0..564d8d6 100644
    --- a/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/SettableUnionObjectInspector.java
    +++ b/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/SettableUnionObjectInspector.java
    @@ -29,6 +29,6 @@ public abstract class SettableUnionObjectInspector implements
        /* Create an empty object */
        public abstract Object create();

    - /* Add fields to the object */
    - public abstract Object addField(Object union, ObjectInspector oi);
    + /* Add field to the object */
    + public abstract Object addField(Object union, Object field);
      }

    http://git-wip-us.apache.org/repos/asf/hive/blob/2688b680/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/StandardUnionObjectInspector.java
    ----------------------------------------------------------------------
    diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/StandardUnionObjectInspector.java b/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/StandardUnionObjectInspector.java
    index d1b11e8..f26c9ec 100644
    --- a/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/StandardUnionObjectInspector.java
    +++ b/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/StandardUnionObjectInspector.java
    @@ -124,9 +124,9 @@ public class StandardUnionObjectInspector extends SettableUnionObjectInspector {
        }

        @Override
    - public Object addField(Object union, ObjectInspector oi) {
    + public Object addField(Object union, Object field) {
          ArrayList<Object> a = (ArrayList<Object>) union;
    - a.add(oi);
    + a.add(field);
          return a;
        }


    http://git-wip-us.apache.org/repos/asf/hive/blob/2688b680/serde/src/test/org/apache/hadoop/hive/serde2/objectinspector/TestObjectInspectorConverters.java
    ----------------------------------------------------------------------
    diff --git a/serde/src/test/org/apache/hadoop/hive/serde2/objectinspector/TestObjectInspectorConverters.java b/serde/src/test/org/apache/hadoop/hive/serde2/objectinspector/TestObjectInspectorConverters.java
    index 1185283..dd18517 100644
    --- a/serde/src/test/org/apache/hadoop/hive/serde2/objectinspector/TestObjectInspectorConverters.java
    +++ b/serde/src/test/org/apache/hadoop/hive/serde2/objectinspector/TestObjectInspectorConverters.java
    @@ -17,6 +17,9 @@
       */
      package org.apache.hadoop.hive.serde2.objectinspector;

    +import java.util.ArrayList;
    +import java.util.List;
    +
      import junit.framework.TestCase;

      import org.apache.hadoop.hive.common.type.HiveDecimal;
    @@ -168,6 +171,90 @@ public class TestObjectInspectorConverters extends TestCase {
                {(byte)'h', (byte)'i',(byte)'v',(byte)'e'}),
                baConverter.convert(new Text("hive")));
            assertEquals("BAConverter", null, baConverter.convert(null));
    +
    + // Union
    + ArrayList<String> fieldNames = new ArrayList<String>();
    + fieldNames.add("firstInteger");
    + fieldNames.add("secondString");
    + fieldNames.add("thirdBoolean");
    + ArrayList<ObjectInspector> fieldObjectInspectors = new ArrayList<ObjectInspector>();
    + fieldObjectInspectors
    + .add(PrimitiveObjectInspectorFactory.javaIntObjectInspector);
    + fieldObjectInspectors
    + .add(PrimitiveObjectInspectorFactory.javaStringObjectInspector);
    + fieldObjectInspectors
    + .add(PrimitiveObjectInspectorFactory.javaBooleanObjectInspector);
    +
    + ArrayList<String> fieldNames2 = new ArrayList<String>();
    + fieldNames2.add("firstString");
    + fieldNames2.add("secondInteger");
    + fieldNames2.add("thirdBoolean");
    + ArrayList<ObjectInspector> fieldObjectInspectors2 = new ArrayList<ObjectInspector>();
    + fieldObjectInspectors2
    + .add(PrimitiveObjectInspectorFactory.javaStringObjectInspector);
    + fieldObjectInspectors2
    + .add(PrimitiveObjectInspectorFactory.javaIntObjectInspector);
    + fieldObjectInspectors2
    + .add(PrimitiveObjectInspectorFactory.javaBooleanObjectInspector);
    +
    + Converter unionConverter0 = ObjectInspectorConverters.getConverter(ObjectInspectorFactory.getStandardUnionObjectInspector(fieldObjectInspectors),
    + ObjectInspectorFactory.getStandardUnionObjectInspector(fieldObjectInspectors2));
    +
    + Object convertedObject0 = unionConverter0.convert(new StandardUnionObjectInspector.StandardUnion((byte)0, 1));
    + List<String> expectedObject0 = new ArrayList<String>();
    + expectedObject0.add("1");
    +
    + assertEquals(expectedObject0, convertedObject0);
    +
    + Converter unionConverter1 = ObjectInspectorConverters.getConverter(ObjectInspectorFactory.getStandardUnionObjectInspector(fieldObjectInspectors),
    + ObjectInspectorFactory.getStandardUnionObjectInspector(fieldObjectInspectors2));
    +
    + Object convertedObject1 = unionConverter1.convert(new StandardUnionObjectInspector.StandardUnion((byte)1, "1"));
    + List<Integer> expectedObject1 = new ArrayList<Integer>();
    + expectedObject1.add(1);
    +
    + assertEquals(expectedObject1, convertedObject1);
    +
    + Converter unionConverter2 = ObjectInspectorConverters.getConverter(ObjectInspectorFactory.getStandardUnionObjectInspector(fieldObjectInspectors),
    + ObjectInspectorFactory.getStandardUnionObjectInspector(fieldObjectInspectors2));
    +
    + Object convertedObject2 = unionConverter2.convert(new StandardUnionObjectInspector.StandardUnion((byte)2, true));
    + List<Boolean> expectedObject2 = new ArrayList<Boolean>();
    + expectedObject2.add(true);
    +
    + assertEquals(expectedObject2, convertedObject2);
    +
    + // Union (extra fields)
    + ArrayList<String> fieldNamesExtra = new ArrayList<String>();
    + fieldNamesExtra.add("firstInteger");
    + fieldNamesExtra.add("secondString");
    + fieldNamesExtra.add("thirdBoolean");
    + ArrayList<ObjectInspector> fieldObjectInspectorsExtra = new ArrayList<ObjectInspector>();
    + fieldObjectInspectorsExtra
    + .add(PrimitiveObjectInspectorFactory.javaIntObjectInspector);
    + fieldObjectInspectorsExtra
    + .add(PrimitiveObjectInspectorFactory.javaStringObjectInspector);
    + fieldObjectInspectorsExtra
    + .add(PrimitiveObjectInspectorFactory.javaBooleanObjectInspector);
    +
    + ArrayList<String> fieldNamesExtra2 = new ArrayList<String>();
    + fieldNamesExtra2.add("firstString");
    + fieldNamesExtra2.add("secondInteger");
    + ArrayList<ObjectInspector> fieldObjectInspectorsExtra2 = new ArrayList<ObjectInspector>();
    + fieldObjectInspectorsExtra2
    + .add(PrimitiveObjectInspectorFactory.javaStringObjectInspector);
    + fieldObjectInspectorsExtra2
    + .add(PrimitiveObjectInspectorFactory.javaIntObjectInspector);
    +
    + Converter unionConverterExtra = ObjectInspectorConverters.getConverter(ObjectInspectorFactory.getStandardUnionObjectInspector(fieldObjectInspectorsExtra),
    + ObjectInspectorFactory.getStandardUnionObjectInspector(fieldObjectInspectorsExtra2));
    +
    + Object convertedObjectExtra = unionConverterExtra.convert(new StandardUnionObjectInspector.StandardUnion((byte)2, true));
    + List<Object> expectedObjectExtra = new ArrayList<Object>();
    + expectedObjectExtra.add(null);
    +
    + assertEquals(expectedObjectExtra, convertedObjectExtra); // we should get back null
    +
          } catch (Throwable e) {
            e.printStackTrace();
            throw e;
    @@ -192,4 +279,4 @@ public class TestObjectInspectorConverters extends TestCase {
          VarcharTypeInfo vcParams = (VarcharTypeInfo) poi.getTypeInfo();
          assertEquals("varchar length doesn't match", 5, vcParams.getLength());
        }
    -}
    +}
    \ No newline at end of file
  • Sershe at Aug 25, 2015 at 9:28 pm
    HIVE-11502: Map side aggregation is extremely slow (Yongzhi Chen, reviewed by Chao Sun)


    Project: http://git-wip-us.apache.org/repos/asf/hive/repo
    Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/ab03dc97
    Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/ab03dc97
    Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/ab03dc97

    Branch: refs/heads/llap
    Commit: ab03dc97b53ac6138de1be68fcb2f8b15aef400f
    Parents: f8b0261
    Author: Yongzhi Chen <yongzhi_chen@hotmail.com>
    Authored: Wed Aug 19 18:21:54 2015 -0700
    Committer: Chao Sun <sunchao@apache.org>
    Committed: Wed Aug 19 18:21:54 2015 -0700

    ----------------------------------------------------------------------
      .../hadoop/hive/ql/exec/KeyWrapperFactory.java | 18 +++++++++++++++++-
      1 file changed, 17 insertions(+), 1 deletion(-)
    ----------------------------------------------------------------------


    http://git-wip-us.apache.org/repos/asf/hive/blob/ab03dc97/ql/src/java/org/apache/hadoop/hive/ql/exec/KeyWrapperFactory.java
    ----------------------------------------------------------------------
    diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/KeyWrapperFactory.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/KeyWrapperFactory.java
    index bf4ba7f..22bd951 100644
    --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/KeyWrapperFactory.java
    +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/KeyWrapperFactory.java
    @@ -21,6 +21,7 @@ package org.apache.hadoop.hive.ql.exec;
      import java.util.Arrays;

      import org.apache.hadoop.hive.ql.metadata.HiveException;
    +import org.apache.hadoop.hive.serde2.lazy.LazyDouble;
      import org.apache.hadoop.hive.serde2.objectinspector.ListObjectsEqualComparer;
      import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
      import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils;
    @@ -103,7 +104,22 @@ public class KeyWrapperFactory {

          @Override
          public void setHashKey() {
    - hashcode = Arrays.hashCode(keys);
    + if (keys == null) {
    + hashcode = 0;
    + } else {
    + hashcode = 1;
    + for (Object element : keys) {
    + hashcode = 31 * hashcode;
    + if(element != null) {
    + if(element instanceof LazyDouble) {
    + long v = Double.doubleToLongBits(((LazyDouble)element).getWritableObject().get());
    + hashcode = hashcode + (int) (v ^ (v >>> 32));
    + } else {
    + hashcode = hashcode + element.hashCode();
    + }
    + }
    + }
    + }
          }

          @Override
  • Sershe at Aug 25, 2015 at 9:28 pm
    HIVE-11607 : Export tables broken for data > 32 MB (Ashutosh Chauhan via Sushanth Sowmyan, Sergio Pena)


    Project: http://git-wip-us.apache.org/repos/asf/hive/repo
    Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/e3664e28
    Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/e3664e28
    Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/e3664e28

    Branch: refs/heads/llap
    Commit: e3664e2899456ed926b028ceba8167c6775eb85d
    Parents: d78bb7c
    Author: Ashutosh Chauhan <hashutosh@apache.org>
    Authored: Fri Aug 21 10:10:52 2015 -0700
    Committer: Ashutosh Chauhan <hashutosh@apache.org>
    Committed: Fri Aug 21 10:12:01 2015 -0700

    ----------------------------------------------------------------------
      shims/0.20S/pom.xml | 8 +++-
      .../hadoop/hive/shims/Hadoop20SShims.java | 35 ++++++++---------
      shims/0.23/pom.xml | 21 ++++------
      .../apache/hadoop/hive/shims/Hadoop23Shims.java | 41 ++++++++------------
      4 files changed, 46 insertions(+), 59 deletions(-)
    ----------------------------------------------------------------------


    http://git-wip-us.apache.org/repos/asf/hive/blob/e3664e28/shims/0.20S/pom.xml
    ----------------------------------------------------------------------
    diff --git a/shims/0.20S/pom.xml b/shims/0.20S/pom.xml
    index 040136b..565dd5e 100644
    --- a/shims/0.20S/pom.xml
    +++ b/shims/0.20S/pom.xml
    @@ -53,5 +53,11 @@
            <version>${hadoop-20S.version}</version>
            <optional>true</optional>
          </dependency>
    - </dependencies>
    + <dependency>
    + <groupId>org.apache.hadoop</groupId>
    + <artifactId>hadoop-tools</artifactId>
    + <version>${hadoop-20S.version}</version>
    + <scope>provided</scope>
    + </dependency>
    + </dependencies>
      </project>

    http://git-wip-us.apache.org/repos/asf/hive/blob/e3664e28/shims/0.20S/src/main/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java
    ----------------------------------------------------------------------
    diff --git a/shims/0.20S/src/main/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java b/shims/0.20S/src/main/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java
    index a56309f..6c6ccbc 100644
    --- a/shims/0.20S/src/main/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java
    +++ b/shims/0.20S/src/main/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java
    @@ -19,7 +19,6 @@ package org.apache.hadoop.hive.shims;

      import java.io.IOException;
      import java.lang.Override;
    -import java.lang.reflect.Constructor;
      import java.net.InetSocketAddress;
      import java.net.MalformedURLException;
      import java.net.URI;
    @@ -27,6 +26,7 @@ import java.net.URL;
      import java.nio.ByteBuffer;
      import java.util.ArrayList;
      import java.util.Arrays;
    +import java.util.Collections;
      import java.util.Comparator;
      import java.util.HashMap;
      import java.util.List;
    @@ -71,6 +71,8 @@ import org.apache.hadoop.security.Credentials;
      import org.apache.hadoop.security.KerberosName;
      import org.apache.hadoop.security.token.Token;
      import org.apache.hadoop.security.UserGroupInformation;
    +import org.apache.hadoop.tools.distcp2.DistCp;
    +import org.apache.hadoop.tools.distcp2.DistCpOptions;
      import org.apache.hadoop.util.Progressable;
      import org.apache.hadoop.util.Tool;
      import org.apache.hadoop.util.VersionInfo;
    @@ -503,7 +505,7 @@ public class Hadoop20SShims extends HadoopShimsSecure {
        }

        public class Hadoop20SFileStatus implements HdfsFileStatus {
    - private FileStatus fileStatus;
    + private final FileStatus fileStatus;
          public Hadoop20SFileStatus(FileStatus fileStatus) {
            this.fileStatus = fileStatus;
          }
    @@ -629,28 +631,33 @@ public class Hadoop20SShims extends HadoopShimsSecure {
         */
        public class KerberosNameShim implements HadoopShimsSecure.KerberosNameShim {

    - private KerberosName kerberosName;
    + private final KerberosName kerberosName;

          public KerberosNameShim(String name) {
            kerberosName = new KerberosName(name);
          }

    + @Override
          public String getDefaultRealm() {
            return kerberosName.getDefaultRealm();
          }

    + @Override
          public String getServiceName() {
            return kerberosName.getServiceName();
          }

    + @Override
          public String getHostName() {
            return kerberosName.getHostName();
          }

    + @Override
          public String getRealm() {
            return kerberosName.getRealm();
          }

    + @Override
          public String getShortName() throws IOException {
            return kerberosName.getShortName();
          }
    @@ -663,27 +670,17 @@ public class Hadoop20SShims extends HadoopShimsSecure {

        @Override
        public boolean runDistCp(Path src, Path dst, Configuration conf) throws IOException {
    - int rc;
    -
    - // Creates the command-line parameters for distcp
    - String[] params = {"-update", "-skipcrccheck", src.toString(), dst.toString()};

    + DistCpOptions options = new DistCpOptions(Collections.singletonList(src), dst);
    + options.setSkipCRC(true);
    + options.setSyncFolder(true);
          try {
    - Class clazzDistCp = Class.forName("org.apache.hadoop.tools.distcp2");
    - Constructor c = clazzDistCp.getConstructor();
    - c.setAccessible(true);
    - Tool distcp = (Tool)c.newInstance();
    - distcp.setConf(conf);
    - rc = distcp.run(params);
    - } catch (ClassNotFoundException e) {
    - throw new IOException("Cannot find DistCp class package: " + e.getMessage());
    - } catch (NoSuchMethodException e) {
    - throw new IOException("Cannot get DistCp constructor: " + e.getMessage());
    + DistCp distcp = new DistCp(conf, options);
    + distcp.execute();
    + return true;
          } catch (Exception e) {
            throw new IOException("Cannot execute DistCp process: " + e, e);
          }
    -
    - return (0 == rc) ? true : false;
        }

        @Override

    http://git-wip-us.apache.org/repos/asf/hive/blob/e3664e28/shims/0.23/pom.xml
    ----------------------------------------------------------------------
    diff --git a/shims/0.23/pom.xml b/shims/0.23/pom.xml
    index 060dd17..2e16956 100644
    --- a/shims/0.23/pom.xml
    +++ b/shims/0.23/pom.xml
    @@ -144,18 +144,11 @@
            <optional>true</optional>
           <type>test-jar</type>
         </dependency>
    - </dependencies>
    -
    - <profiles>
    - <profile>
    - <id>hadoop-2</id>
    - <dependencies>
    - <dependency>
    - <groupId>org.apache.hadoop</groupId>
    - <artifactId>hadoop-distcp</artifactId>
    - <version>${hadoop-23.version}</version>
    - </dependency>
    - </dependencies>
    - </profile>
    - </profiles>
    + <dependency>
    + <groupId>org.apache.hadoop</groupId>
    + <artifactId>hadoop-distcp</artifactId>
    + <version>${hadoop-23.version}</version>
    + <scope>provided</scope>
    + </dependency>
    + </dependencies>
      </project>

    http://git-wip-us.apache.org/repos/asf/hive/blob/e3664e28/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java
    ----------------------------------------------------------------------
    diff --git a/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java b/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java
    index e5be8d6..29d0f13 100644
    --- a/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java
    +++ b/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java
    @@ -19,7 +19,6 @@ package org.apache.hadoop.hive.shims;

      import java.io.FileNotFoundException;
      import java.io.IOException;
    -import java.lang.reflect.Constructor;
      import java.lang.reflect.Method;
      import java.net.InetSocketAddress;
      import java.net.MalformedURLException;
    @@ -28,12 +27,12 @@ import java.nio.ByteBuffer;
      import java.security.AccessControlException;
      import java.security.NoSuchAlgorithmException;
      import java.util.ArrayList;
    +import java.util.Collections;
      import java.util.Comparator;
      import java.util.HashMap;
      import java.util.Iterator;
      import java.util.List;
      import java.util.Map;
    -import java.util.Set;
      import java.util.TreeMap;

      import org.apache.commons.lang.StringUtils;
    @@ -64,7 +63,6 @@ import org.apache.hadoop.fs.permission.FsPermission;
      import org.apache.hadoop.hdfs.DFSClient;
      import org.apache.hadoop.hdfs.DistributedFileSystem;
      import org.apache.hadoop.hdfs.MiniDFSCluster;
    -import org.apache.hadoop.hdfs.protocol.BlockStoragePolicy;
      import org.apache.hadoop.hdfs.protocol.DirectoryListing;
      import org.apache.hadoop.hdfs.protocol.HdfsConstants;
      import org.apache.hadoop.hdfs.protocol.HdfsLocatedFileStatus;
    @@ -93,8 +91,9 @@ import org.apache.hadoop.net.NetUtils;
      import org.apache.hadoop.security.authentication.util.KerberosName;
      import org.apache.hadoop.security.Credentials;
      import org.apache.hadoop.security.UserGroupInformation;
    +import org.apache.hadoop.tools.DistCp;
    +import org.apache.hadoop.tools.DistCpOptions;
      import org.apache.hadoop.util.Progressable;
    -import org.apache.hadoop.util.Tool;
      import org.apache.hadoop.yarn.conf.YarnConfiguration;
      import org.apache.tez.test.MiniTezCluster;

    @@ -121,7 +120,7 @@ public class Hadoop23Shims extends HadoopShimsSecure {
            zcr = true;
          } catch (ClassNotFoundException ce) {
          }
    -
    +
          if (zcr) {
            // in-memory HDFS is only available after zcr
            try {
    @@ -576,11 +575,11 @@ public class Hadoop23Shims extends HadoopShimsSecure {
                                                                                      org.apache.hadoop.mapred.TaskAttemptID taskId, Progressable progressable) {
            org.apache.hadoop.mapred.TaskAttemptContext newContext = null;
            try {
    - java.lang.reflect.Constructor construct = org.apache.hadoop.mapred.TaskAttemptContextImpl.class.getDeclaredConstructor(
    + java.lang.reflect.Constructor<org.apache.hadoop.mapred.TaskAttemptContextImpl> construct = org.apache.hadoop.mapred.TaskAttemptContextImpl.class.getDeclaredConstructor(
                      org.apache.hadoop.mapred.JobConf.class, org.apache.hadoop.mapred.TaskAttemptID.class,
                      Reporter.class);
              construct.setAccessible(true);
    - newContext = (org.apache.hadoop.mapred.TaskAttemptContext) construct.newInstance(
    + newContext = construct.newInstance(
                      new JobConf(conf), taskId, progressable);
            } catch (Exception e) {
              throw new RuntimeException(e);
    @@ -913,6 +912,7 @@ public class Hadoop23Shims extends HadoopShimsSecure {
           * Cannot add Override annotation since FileSystem.access() may not exist in
           * the version of hadoop used to build Hive.
           */
    + @Override
          public void access(Path path, FsAction action) throws AccessControlException,
              FileNotFoundException, IOException {
            Path underlyingFsPath = swizzleParamPath(path);
    @@ -1179,7 +1179,7 @@ public class Hadoop23Shims extends HadoopShimsSecure {
            }
          }
        }
    -
    +

        @Override
        public HadoopShims.StoragePolicyShim getStoragePolicyShim(FileSystem fs) {
    @@ -1195,27 +1195,17 @@ public class Hadoop23Shims extends HadoopShimsSecure {

        @Override
        public boolean runDistCp(Path src, Path dst, Configuration conf) throws IOException {
    - int rc;
    -
    - // Creates the command-line parameters for distcp
    - String[] params = {"-update", "-skipcrccheck", src.toString(), dst.toString()};

    + DistCpOptions options = new DistCpOptions(Collections.singletonList(src), dst);
    + options.setSkipCRC(true);
    + options.setSyncFolder(true);
          try {
    - Class clazzDistCp = Class.forName("org.apache.hadoop.tools.DistCp");
    - Constructor c = clazzDistCp.getConstructor();
    - c.setAccessible(true);
    - Tool distcp = (Tool)c.newInstance();
    - distcp.setConf(conf);
    - rc = distcp.run(params);
    - } catch (ClassNotFoundException e) {
    - throw new IOException("Cannot find DistCp class package: " + e.getMessage());
    - } catch (NoSuchMethodException e) {
    - throw new IOException("Cannot get DistCp constructor: " + e.getMessage());
    + DistCp distcp = new DistCp(conf, options);
    + distcp.execute();
    + return true;
          } catch (Exception e) {
            throw new IOException("Cannot execute DistCp process: " + e, e);
          }
    -
    - return (0 == rc);
        }

        private static Boolean hdfsEncryptionSupport;
    @@ -1250,7 +1240,7 @@ public class Hadoop23Shims extends HadoopShimsSecure {
           */
          private KeyProvider keyProvider = null;

    - private Configuration conf;
    + private final Configuration conf;

          public HdfsEncryptionShim(URI uri, Configuration conf) throws IOException {
            DistributedFileSystem dfs = (DistributedFileSystem)FileSystem.get(uri, conf);
    @@ -1409,6 +1399,7 @@ public class Hadoop23Shims extends HadoopShimsSecure {
          }
          return result;
        }
    + @Override
        public void addDelegationTokens(FileSystem fs, Credentials cred, String uname) throws IOException {
          // Use method addDelegationTokens instead of getDelegationToken to get all the tokens including KMS.
          fs.addDelegationTokens(uname, cred);
  • Sershe at Aug 25, 2015 at 9:28 pm
    HIVE-11592: ORC metadata section can sometimes exceed protobuf message size limit (Prasanth Jayachandran reviewed by Sergey Shelukhin)


    Project: http://git-wip-us.apache.org/repos/asf/hive/repo
    Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/f8b02610
    Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/f8b02610
    Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/f8b02610

    Branch: refs/heads/llap
    Commit: f8b02610d745e63e3d596d7532e84e49eedbd62e
    Parents: 2688b68
    Author: Prasanth Jayachandran <j.prasanth.j@gmail.com>
    Authored: Wed Aug 19 11:40:52 2015 -0700
    Committer: Prasanth Jayachandran <j.prasanth.j@gmail.com>
    Committed: Wed Aug 19 11:40:52 2015 -0700

    ----------------------------------------------------------------------
      .../hadoop/hive/ql/io/orc/ReaderImpl.java | 34 +++++++++++++++++++-
      1 file changed, 33 insertions(+), 1 deletion(-)
    ----------------------------------------------------------------------


    http://git-wip-us.apache.org/repos/asf/hive/blob/f8b02610/ql/src/java/org/apache/hadoop/hive/ql/io/orc/ReaderImpl.java
    ----------------------------------------------------------------------
    diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/ReaderImpl.java b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/ReaderImpl.java
    index a6448b6..c990d85 100644
    --- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/ReaderImpl.java
    +++ b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/ReaderImpl.java
    @@ -46,12 +46,15 @@ import org.apache.hadoop.io.Text;
      import com.google.common.collect.Lists;
      import com.google.common.collect.Sets;
      import com.google.protobuf.CodedInputStream;
    +import com.google.protobuf.InvalidProtocolBufferException;

      public class ReaderImpl implements Reader {

        private static final Log LOG = LogFactory.getLog(ReaderImpl.class);

        private static final int DIRECTORY_SIZE_GUESS = 16 * 1024;
    + private static final int DEFAULT_PROTOBUF_MESSAGE_LIMIT = 64 << 20; // 64MB
    + private static final int PROTOBUF_MESSAGE_MAX_LIMIT = 1024 << 20; // 1GB

        protected final FileSystem fileSystem;
        protected final Path path;
    @@ -468,7 +471,36 @@ public class ReaderImpl implements Reader {

            InputStream instream = InStream.create("metadata", Lists.<DiskRange>newArrayList(
                new BufferChunk(footerBuffer, 0)), metadataSize, codec, bufferSize);
    - this.metadata = OrcProto.Metadata.parseFrom(instream);
    + CodedInputStream in = CodedInputStream.newInstance(instream);
    + int msgLimit = DEFAULT_PROTOBUF_MESSAGE_LIMIT;
    + OrcProto.Metadata meta = null;
    + do {
    + try {
    + in.setSizeLimit(msgLimit);
    + meta = OrcProto.Metadata.parseFrom(in);
    + } catch (InvalidProtocolBufferException e) {
    + if (e.getMessage().contains("Protocol message was too large")) {
    + LOG.warn("Metadata section is larger than " + msgLimit + " bytes. Increasing the max" +
    + " size of the coded input stream." );
    +
    + msgLimit = msgLimit << 1;
    + if (msgLimit > PROTOBUF_MESSAGE_MAX_LIMIT) {
    + LOG.error("Metadata section exceeds max protobuf message size of " +
    + PROTOBUF_MESSAGE_MAX_LIMIT + " bytes.");
    + throw e;
    + }
    +
    + // we must have failed in the middle of reading instream and instream doesn't support
    + // resetting the stream
    + instream = InStream.create("metadata", Lists.<DiskRange>newArrayList(
    + new BufferChunk(footerBuffer, 0)), metadataSize, codec, bufferSize);
    + in = CodedInputStream.newInstance(instream);
    + } else {
    + throw e;
    + }
    + }
    + } while (meta == null);
    + this.metadata = meta;

            footerBuffer.position(position + metadataSize);
            footerBuffer.limit(position + metadataSize + footerBufferSize);
  • Sershe at Aug 25, 2015 at 9:28 pm
    HIVE-8583 Cleanup & Test for hive.script.operator.env.blacklist (Lars Franke via gates)


    Project: http://git-wip-us.apache.org/repos/asf/hive/repo
    Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/d78bb7c4
    Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/d78bb7c4
    Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/d78bb7c4

    Branch: refs/heads/llap
    Commit: d78bb7c49d88a8acf3fd317a06f9563003571f4c
    Parents: 87930e2
    Author: Alan Gates <gates@hortonworks.com>
    Authored: Fri Aug 21 10:09:22 2015 -0700
    Committer: Alan Gates <gates@hortonworks.com>
    Committed: Fri Aug 21 10:09:22 2015 -0700

    ----------------------------------------------------------------------
      .../hadoop/hive/ql/exec/ScriptOperator.java | 85 ++++++++++----------
      .../hadoop/hive/ql/exec/TestOperators.java | 16 ++++
      2 files changed, 60 insertions(+), 41 deletions(-)
    ----------------------------------------------------------------------


    http://git-wip-us.apache.org/repos/asf/hive/blob/d78bb7c4/ql/src/java/org/apache/hadoop/hive/ql/exec/ScriptOperator.java
    ----------------------------------------------------------------------
    diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/ScriptOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/ScriptOperator.java
    index f2eed44..82ec668 100644
    --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/ScriptOperator.java
    +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/ScriptOperator.java
    @@ -18,25 +18,6 @@

      package org.apache.hadoop.hive.ql.exec;

    -import java.io.BufferedInputStream;
    -import java.io.BufferedOutputStream;
    -import java.io.DataInputStream;
    -import java.io.DataOutputStream;
    -import java.io.File;
    -import java.io.IOException;
    -import java.io.Serializable;
    -import java.util.ArrayList;
    -import java.util.Arrays;
    -import java.util.Collection;
    -import java.util.HashSet;
    -import java.util.Iterator;
    -import java.util.Map;
    -import java.util.Set;
    -import java.util.Timer;
    -import java.util.TimerTask;
    -import java.util.concurrent.Future;
    -import java.util.concurrent.TimeUnit;
    -
      import org.apache.hadoop.conf.Configuration;
      import org.apache.hadoop.hive.conf.HiveConf;
      import org.apache.hadoop.hive.ql.ErrorMsg;
    @@ -57,6 +38,26 @@ import org.apache.hadoop.util.Shell;
      import org.apache.hadoop.util.StringUtils;
      import org.apache.spark.SparkFiles;

    +import java.io.BufferedInputStream;
    +import java.io.BufferedOutputStream;
    +import java.io.DataInputStream;
    +import java.io.DataOutputStream;
    +import java.io.File;
    +import java.io.IOException;
    +import java.io.Serializable;
    +import java.util.ArrayList;
    +import java.util.Arrays;
    +import java.util.Collection;
    +import java.util.Collections;
    +import java.util.HashSet;
    +import java.util.Iterator;
    +import java.util.Map;
    +import java.util.Set;
    +import java.util.Timer;
    +import java.util.TimerTask;
    +import java.util.concurrent.Future;
    +import java.util.concurrent.TimeUnit;
    +
      /**
       * ScriptOperator.
       *
    @@ -146,16 +147,19 @@ public class ScriptOperator extends Operator<ScriptDesc> implements
          return value;
        }

    - boolean blackListed(String name) {
    + /**
    + * Checks whether a given configuration name is blacklisted and should not be converted
    + * to an environment variable.
    + */
    + boolean blackListed(Configuration conf, String name) {
          if (blackListedConfEntries == null) {
            blackListedConfEntries = new HashSet<String>();
    - if (hconf != null) {
    - String bl = hconf.get(HiveConf.ConfVars.HIVESCRIPT_ENV_BLACKLIST.toString());
    - if (bl != null && bl.length() > 0) {
    + if (conf != null) {
    + String bl = conf.get(HiveConf.ConfVars.HIVESCRIPT_ENV_BLACKLIST.toString(),
    + HiveConf.ConfVars.HIVESCRIPT_ENV_BLACKLIST.getDefaultValue());
    + if (bl != null && !bl.isEmpty()) {
                String[] bls = bl.split(",");
    - for (String b : bls) {
    - blackListedConfEntries.add(b);
    - }
    + Collections.addAll(blackListedConfEntries, bls);
              }
            }
          }
    @@ -171,7 +175,7 @@ public class ScriptOperator extends Operator<ScriptDesc> implements
          while (it.hasNext()) {
            Map.Entry<String, String> en = it.next();
            String name = en.getKey();
    - if (!blackListed(name)) {
    + if (!blackListed(conf, name)) {
              // String value = (String)en.getValue(); // does not apply variable
              // expansion
              String value = conf.get(name); // does variable expansion
    @@ -306,8 +310,7 @@ public class ScriptOperator extends Operator<ScriptDesc> implements
        void displayBrokenPipeInfo() {
          if (isLogInfoEnabled) {
            LOG.info("The script did not consume all input data. This is considered as an error.");
    - LOG.info("set " + HiveConf.ConfVars.ALLOWPARTIALCONSUMP.toString()
    - + "=true; to ignore it.");
    + LOG.info("set " + HiveConf.ConfVars.ALLOWPARTIALCONSUMP.toString() + "=true; to ignore it.");
          }
          return;
        }
    @@ -349,12 +352,12 @@ public class ScriptOperator extends Operator<ScriptDesc> implements
              }

              String[] wrappedCmdArgs = addWrapper(cmdArgs);
    - if (isLogInfoEnabled) {
    - LOG.info("Executing " + Arrays.asList(wrappedCmdArgs));
    - LOG.info("tablename=" + tableName);
    - LOG.info("partname=" + partitionName);
    - LOG.info("alias=" + alias);
    - }
    + if (isLogInfoEnabled) {
    + LOG.info("Executing " + Arrays.asList(wrappedCmdArgs));
    + LOG.info("tablename=" + tableName);
    + LOG.info("partname=" + partitionName);
    + LOG.info("alias=" + alias);
    + }

              ProcessBuilder pb = new ProcessBuilder(wrappedCmdArgs);
              Map<String, String> env = pb.environment();
    @@ -672,9 +675,9 @@ public class ScriptOperator extends Operator<ScriptDesc> implements
            long now = System.currentTimeMillis();
            // reporter is a member variable of the Operator class.
            if (now - lastReportTime > 60 * 1000 && reporter != null) {
    - if (isLogInfoEnabled) {
    - LOG.info("ErrorStreamProcessor calling reporter.progress()");
    - }
    + if (isLogInfoEnabled) {
    + LOG.info("ErrorStreamProcessor calling reporter.progress()");
    + }
              lastReportTime = now;
              reporter.progress();
            }
    @@ -730,9 +733,9 @@ public class ScriptOperator extends Operator<ScriptDesc> implements
                }
                proc.processLine(row);
              }
    - if (isLogInfoEnabled) {
    - LOG.info("StreamThread " + name + " done");
    - }
    + if (isLogInfoEnabled) {
    + LOG.info("StreamThread " + name + " done");
    + }

            } catch (Throwable th) {
              scriptError = th;

    http://git-wip-us.apache.org/repos/asf/hive/blob/d78bb7c4/ql/src/test/org/apache/hadoop/hive/ql/exec/TestOperators.java
    ----------------------------------------------------------------------
    diff --git a/ql/src/test/org/apache/hadoop/hive/ql/exec/TestOperators.java b/ql/src/test/org/apache/hadoop/hive/ql/exec/TestOperators.java
    index c3a36c0..21dcf86 100644
    --- a/ql/src/test/org/apache/hadoop/hive/ql/exec/TestOperators.java
    +++ b/ql/src/test/org/apache/hadoop/hive/ql/exec/TestOperators.java
    @@ -58,6 +58,7 @@ import org.apache.hadoop.io.Text;
      import org.apache.hadoop.mapred.InputSplit;
      import org.apache.hadoop.mapred.JobConf;
      import org.apache.hadoop.mapred.TextInputFormat;
    +import org.junit.Assert;
      import org.junit.Test;

      /**
    @@ -192,6 +193,21 @@ public class TestOperators extends TestCase {
          }
        }

    + public void testScriptOperatorBlacklistedEnvVarsProcessing() {
    + ScriptOperator scriptOperator = new ScriptOperator();
    +
    + Configuration hconf = new JobConf(ScriptOperator.class);
    +
    + Map<String, String> env = new HashMap<String, String>();
    +
    + HiveConf.setVar(hconf, HiveConf.ConfVars.HIVESCRIPT_ENV_BLACKLIST, "foobar");
    + hconf.set("foobar", "foobar");
    + hconf.set("barfoo", "barfoo");
    + scriptOperator.addJobConfToEnvironment(hconf, env);
    + Assert.assertFalse(env.containsKey("foobar"));
    + Assert.assertTrue(env.containsKey("barfoo"));
    + }
    +
        public void testScriptOperator() throws Throwable {
          try {
            System.out.println("Testing Script Operator");
  • Sershe at Aug 25, 2015 at 9:28 pm
    HIVE-10631 : create_table_core method has invalid update for Fast Stats (Aaron Tokhy via Ashutosh Chauhan)


    Project: http://git-wip-us.apache.org/repos/asf/hive/repo
    Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/9d3d3d01
    Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/9d3d3d01
    Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/9d3d3d01

    Branch: refs/heads/llap
    Commit: 9d3d3d0134fed704684fdefa9faa6ed949615cc0
    Parents: a16bbd4
    Author: Aaron Tokhy <atokhy@gmail.com>
    Authored: Thu Aug 13 10:45:00 2015 -0800
    Committer: Ashutosh Chauhan <hashutosh@apache.org>
    Committed: Mon Aug 24 10:12:18 2015 -0700

    ----------------------------------------------------------------------
      .../hadoop/hive/metastore/HiveAlterHandler.java | 2 +-
      .../hadoop/hive/metastore/HiveMetaStore.java | 6 +-----
      .../hadoop/hive/metastore/MetaStoreUtils.java | 17 ++++++++++-------
      .../ql/metadata/SessionHiveMetaStoreClient.java | 2 +-
      4 files changed, 13 insertions(+), 14 deletions(-)
    ----------------------------------------------------------------------


    http://git-wip-us.apache.org/repos/asf/hive/blob/9d3d3d01/metastore/src/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java
    ----------------------------------------------------------------------
    diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java
    index 99d6680..ee20430 100644
    --- a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java
    +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java
    @@ -236,7 +236,7 @@ public class HiveAlterHandler implements AlterHandler {
                Database db = msdb.getDatabase(newt.getDbName());
                // Update table stats. For partitioned table, we update stats in
                // alterPartition()
    - MetaStoreUtils.updateUnpartitionedTableStatsFast(db, newt, wh, false, true);
    + MetaStoreUtils.updateTableStatsFast(db, newt, wh, false, true);
            }
            updateTableColumnStatsForAlterTable(msdb, oldt, newt);
            // now finally call alter table

    http://git-wip-us.apache.org/repos/asf/hive/blob/9d3d3d01/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
    ----------------------------------------------------------------------
    diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
    index ae500bf..1840e76 100644
    --- a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
    +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
    @@ -1408,11 +1408,7 @@ public class HiveMetaStore extends ThriftHiveMetastore {
              }
              if (HiveConf.getBoolVar(hiveConf, HiveConf.ConfVars.HIVESTATSAUTOGATHER) &&
                  !MetaStoreUtils.isView(tbl)) {
    - if (tbl.getPartitionKeysSize() == 0) { // Unpartitioned table
    - MetaStoreUtils.updateUnpartitionedTableStatsFast(db, tbl, wh, madeDir);
    - } else { // Partitioned table with no partitions.
    - MetaStoreUtils.updateUnpartitionedTableStatsFast(db, tbl, wh, true);
    - }
    + MetaStoreUtils.updateTableStatsFast(db, tbl, wh, madeDir);
              }

              // set create time

    http://git-wip-us.apache.org/repos/asf/hive/blob/9d3d3d01/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java
    ----------------------------------------------------------------------
    diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java b/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java
    index 907cbbf..f88f4dd 100644
    --- a/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java
    +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java
    @@ -169,19 +169,22 @@ public class MetaStoreUtils {
          return true;
        }

    - public static boolean updateUnpartitionedTableStatsFast(Database db, Table tbl, Warehouse wh,
    + public static boolean updateTableStatsFast(Database db, Table tbl, Warehouse wh,
            boolean madeDir) throws MetaException {
    - return updateUnpartitionedTableStatsFast(db, tbl, wh, madeDir, false);
    + return updateTableStatsFast(db, tbl, wh, madeDir, false);
        }

    - public static boolean updateUnpartitionedTableStatsFast(Database db, Table tbl, Warehouse wh,
    + public static boolean updateTableStatsFast(Database db, Table tbl, Warehouse wh,
            boolean madeDir, boolean forceRecompute) throws MetaException {
    - return updateUnpartitionedTableStatsFast(tbl,
    - wh.getFileStatusesForUnpartitionedTable(db, tbl), madeDir, forceRecompute);
    + FileStatus[] fileStatuses = {};
    + if (tbl.getPartitionKeysSize() == 0) { // Update stats only when unpartitioned
    + fileStatuses = wh.getFileStatusesForUnpartitionedTable(db, tbl);
    + }
    + return updateTableStatsFast(tbl, fileStatuses, madeDir, forceRecompute);
        }

        /**
    - * Updates the numFiles and totalSize parameters for the passed unpartitioned Table by querying
    + * Updates the numFiles and totalSize parameters for the passed Table by querying
         * the warehouse if the passed Table does not already have values for these parameters.
         * @param tbl
         * @param fileStatus
    @@ -190,7 +193,7 @@ public class MetaStoreUtils {
         * these parameters set
         * @return true if the stats were updated, false otherwise
         */
    - public static boolean updateUnpartitionedTableStatsFast(Table tbl,
    + public static boolean updateTableStatsFast(Table tbl,
            FileStatus[] fileStatus, boolean newDir, boolean forceRecompute) throws MetaException {

          Map<String,String> params = tbl.getParameters();

    http://git-wip-us.apache.org/repos/asf/hive/blob/9d3d3d01/ql/src/java/org/apache/hadoop/hive/ql/metadata/SessionHiveMetaStoreClient.java
    ----------------------------------------------------------------------
    diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/SessionHiveMetaStoreClient.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/SessionHiveMetaStoreClient.java
    index d20f196..51ff262 100644
    --- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/SessionHiveMetaStoreClient.java
    +++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/SessionHiveMetaStoreClient.java
    @@ -386,7 +386,7 @@ public class SessionHiveMetaStoreClient extends HiveMetaStoreClient implements I
          }

          org.apache.hadoop.hive.metastore.api.Table newtCopy = deepCopyAndLowerCaseTable(newt);
    - MetaStoreUtils.updateUnpartitionedTableStatsFast(newtCopy,
    + MetaStoreUtils.updateTableStatsFast(newtCopy,
              getWh().getFileStatusesForSD(newtCopy.getSd()), false, true);
          Table newTable = new Table(newtCopy);
          String newDbName = newTable.getDbName();
  • Sershe at Aug 25, 2015 at 9:28 pm
    HIVE-5277: HBase handler skips rows with null valued first cells when only row key is selected (Swarnim Kulkarni, reviewed by Xuefu Zhang)


    Project: http://git-wip-us.apache.org/repos/asf/hive/repo
    Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/1030ae71
    Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/1030ae71
    Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/1030ae71

    Branch: refs/heads/llap
    Commit: 1030ae719e2bbea47d8f2835244589621ab126fb
    Parents: b4a6ac9
    Author: Swarnim Kulkarni <kulkarni.swarnim@gmail.com>
    Authored: Fri Aug 21 11:21:08 2015 -0700
    Committer: Chao Sun <sunchao@apache.org>
    Committed: Fri Aug 21 11:21:08 2015 -0700

    ----------------------------------------------------------------------
      .../hive/hbase/HiveHBaseInputFormatUtil.java | 50 +++++----
      .../queries/positive/hbase_null_first_col.q | 22 ++++
      .../results/positive/hbase_null_first_col.q.out | 109 +++++++++++++++++++
      3 files changed, 158 insertions(+), 23 deletions(-)
    ----------------------------------------------------------------------


    http://git-wip-us.apache.org/repos/asf/hive/blob/1030ae71/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HiveHBaseInputFormatUtil.java
    ----------------------------------------------------------------------
    diff --git a/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HiveHBaseInputFormatUtil.java b/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HiveHBaseInputFormatUtil.java
    index 0524572..c002070 100644
    --- a/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HiveHBaseInputFormatUtil.java
    +++ b/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HiveHBaseInputFormatUtil.java
    @@ -18,9 +18,18 @@

      package org.apache.hadoop.hive.hbase;

    +import java.io.IOException;
    +import java.util.ArrayList;
    +import java.util.HashMap;
    +import java.util.List;
    +import java.util.Map;
    +
      import org.apache.hadoop.hbase.HBaseConfiguration;
      import org.apache.hadoop.hbase.client.HTable;
      import org.apache.hadoop.hbase.client.Scan;
    +import org.apache.hadoop.hbase.filter.FilterList;
    +import org.apache.hadoop.hbase.filter.FirstKeyOnlyFilter;
    +import org.apache.hadoop.hbase.filter.KeyOnlyFilter;
      import org.apache.hadoop.hbase.util.Bytes;
      import org.apache.hadoop.hive.hbase.ColumnMappings.ColumnMapping;
      import org.apache.hadoop.hive.ql.index.IndexSearchCondition;
    @@ -28,12 +37,6 @@ import org.apache.hadoop.hive.serde2.ColumnProjectionUtils;
      import org.apache.hadoop.hive.serde2.SerDeException;
      import org.apache.hadoop.mapred.JobConf;

    -import java.io.IOException;
    -import java.util.ArrayList;
    -import java.util.HashMap;
    -import java.util.List;
    -import java.util.Map;
    -
      /**
       * Util code common between HiveHBaseTableInputFormat and HiveHBaseTableSnapshotInputFormat.
       */
    @@ -95,26 +98,27 @@ class HiveHBaseInputFormatUtil {
            }
          }

    - // The HBase table's row key maps to a Hive table column. In the corner case when only the
    - // row key column is selected in Hive, the HBase Scan will be empty i.e. no column family/
    - // column qualifier will have been added to the scan. We arbitrarily add at least one column
    - // to the HBase scan so that we can retrieve all of the row keys and return them as the Hive
    - // tables column projection.
    + // If we have cases where we are running a query like count(key) or count(*),
    + // in such cases, the readColIDs is either empty(for count(*)) or has just the
    + // key column in it. In either case, nothing gets added to the scan. So if readAllColumns is
    + // true, we are going to add all columns. Else we are just going to add a key filter to run a
    + // count only on the keys
          if (empty) {
    - for (ColumnMapping colMap: columnMappings) {
    - if (colMap.hbaseRowKey || colMap.hbaseTimestamp) {
    - continue;
    - }
    -
    - if (colMap.qualifierName == null) {
    - scan.addFamily(colMap.familyNameBytes);
    - } else {
    - scan.addColumn(colMap.familyNameBytes, colMap.qualifierNameBytes);
    - }
    + if (readAllColumns) {
    + for (ColumnMapping colMap: columnMappings) {
    + if (colMap.hbaseRowKey || colMap.hbaseTimestamp) {
    + continue;
    + }

    - if (!readAllColumns) {
    - break;
    + if (colMap.qualifierName == null) {
    + scan.addFamily(colMap.familyNameBytes);
    + } else {
    + scan.addColumn(colMap.familyNameBytes, colMap.qualifierNameBytes);
    + }
              }
    + } else {
    + // Add a filter to just do a scan on the keys so that we pick up everything
    + scan.setFilter(new FilterList(new FirstKeyOnlyFilter(), new KeyOnlyFilter()));
            }
          }


    http://git-wip-us.apache.org/repos/asf/hive/blob/1030ae71/hbase-handler/src/test/queries/positive/hbase_null_first_col.q
    ----------------------------------------------------------------------
    diff --git a/hbase-handler/src/test/queries/positive/hbase_null_first_col.q b/hbase-handler/src/test/queries/positive/hbase_null_first_col.q
    new file mode 100644
    index 0000000..0d9ff56
    --- /dev/null
    +++ b/hbase-handler/src/test/queries/positive/hbase_null_first_col.q
    @@ -0,0 +1,22 @@
    +DROP TABLE src_null;
    +DROP TABLE hbase_null;
    +
    +CREATE TABLE src_null(a STRING, b STRING, c STRING, d STRING) STORED AS TEXTFILE;
    +LOAD DATA LOCAL INPATH '../../data/files/null.txt' INTO TABLE src_null;
    +
    +CREATE TABLE hbase_null(key string, col1 string, col2 string)
    +STORED BY 'org.apache.hadoop.hive.hbase.HBaseStorageHandler'
    +WITH SERDEPROPERTIES (
    +"hbase.columns.mapping" = ":key,cf1:c1,cf1:c2"
    +);
    +
    +SELECT d, a, c FROM src_null;
    +
    +INSERT INTO TABLE hbase_null SELECT d, a, c FROM src_null;
    +
    +SELECT COUNT(d) FROM src_null;
    +SELECT COUNT(key) FROM hbase_null;
    +SELECT COUNT(*) FROM hbase_null;
    +
    +DROP TABLE src_null;
    +DROP TABLE hbase_null;

    http://git-wip-us.apache.org/repos/asf/hive/blob/1030ae71/hbase-handler/src/test/results/positive/hbase_null_first_col.q.out
    ----------------------------------------------------------------------
    diff --git a/hbase-handler/src/test/results/positive/hbase_null_first_col.q.out b/hbase-handler/src/test/results/positive/hbase_null_first_col.q.out
    new file mode 100644
    index 0000000..bb4491b
    --- /dev/null
    +++ b/hbase-handler/src/test/results/positive/hbase_null_first_col.q.out
    @@ -0,0 +1,109 @@
    +PREHOOK: query: DROP TABLE src_null
    +PREHOOK: type: DROPTABLE
    +POSTHOOK: query: DROP TABLE src_null
    +POSTHOOK: type: DROPTABLE
    +PREHOOK: query: DROP TABLE hbase_null
    +PREHOOK: type: DROPTABLE
    +POSTHOOK: query: DROP TABLE hbase_null
    +POSTHOOK: type: DROPTABLE
    +PREHOOK: query: CREATE TABLE src_null(a STRING, b STRING, c STRING, d STRING) STORED AS TEXTFILE
    +PREHOOK: type: CREATETABLE
    +PREHOOK: Output: database:default
    +PREHOOK: Output: default@src_null
    +POSTHOOK: query: CREATE TABLE src_null(a STRING, b STRING, c STRING, d STRING) STORED AS TEXTFILE
    +POSTHOOK: type: CREATETABLE
    +POSTHOOK: Output: database:default
    +POSTHOOK: Output: default@src_null
    +PREHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/null.txt' INTO TABLE src_null
    +PREHOOK: type: LOAD
    +#### A masked pattern was here ####
    +PREHOOK: Output: default@src_null
    +POSTHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/null.txt' INTO TABLE src_null
    +POSTHOOK: type: LOAD
    +#### A masked pattern was here ####
    +POSTHOOK: Output: default@src_null
    +PREHOOK: query: CREATE TABLE hbase_null(key string, col1 string, col2 string)
    +STORED BY 'org.apache.hadoop.hive.hbase.HBaseStorageHandler'
    +WITH SERDEPROPERTIES (
    +"hbase.columns.mapping" = ":key,cf1:c1,cf1:c2"
    +)
    +PREHOOK: type: CREATETABLE
    +PREHOOK: Output: database:default
    +PREHOOK: Output: default@hbase_null
    +POSTHOOK: query: CREATE TABLE hbase_null(key string, col1 string, col2 string)
    +STORED BY 'org.apache.hadoop.hive.hbase.HBaseStorageHandler'
    +WITH SERDEPROPERTIES (
    +"hbase.columns.mapping" = ":key,cf1:c1,cf1:c2"
    +)
    +POSTHOOK: type: CREATETABLE
    +POSTHOOK: Output: database:default
    +POSTHOOK: Output: default@hbase_null
    +PREHOOK: query: SELECT d, a, c FROM src_null
    +PREHOOK: type: QUERY
    +PREHOOK: Input: default@src_null
    +#### A masked pattern was here ####
    +POSTHOOK: query: SELECT d, a, c FROM src_null
    +POSTHOOK: type: QUERY
    +POSTHOOK: Input: default@src_null
    +#### A masked pattern was here ####
    +0 1.0 same
    +1 1.0 same
    +2 1.0 same
    +3 1.0 same
    +4 1.0 same
    +5 NULL same
    +6 NULL same
    +7 1.0 same
    +8 1.0 same
    +9 1.0 same
    +PREHOOK: query: INSERT INTO TABLE hbase_null SELECT d, a, c FROM src_null
    +PREHOOK: type: QUERY
    +PREHOOK: Input: default@src_null
    +PREHOOK: Output: default@hbase_null
    +POSTHOOK: query: INSERT INTO TABLE hbase_null SELECT d, a, c FROM src_null
    +POSTHOOK: type: QUERY
    +POSTHOOK: Input: default@src_null
    +POSTHOOK: Output: default@hbase_null
    +PREHOOK: query: SELECT COUNT(d) FROM src_null
    +PREHOOK: type: QUERY
    +PREHOOK: Input: default@src_null
    +#### A masked pattern was here ####
    +POSTHOOK: query: SELECT COUNT(d) FROM src_null
    +POSTHOOK: type: QUERY
    +POSTHOOK: Input: default@src_null
    +#### A masked pattern was here ####
    +10
    +PREHOOK: query: SELECT COUNT(key) FROM hbase_null
    +PREHOOK: type: QUERY
    +PREHOOK: Input: default@hbase_null
    +#### A masked pattern was here ####
    +POSTHOOK: query: SELECT COUNT(key) FROM hbase_null
    +POSTHOOK: type: QUERY
    +POSTHOOK: Input: default@hbase_null
    +#### A masked pattern was here ####
    +10
    +PREHOOK: query: SELECT COUNT(*) FROM hbase_null
    +PREHOOK: type: QUERY
    +PREHOOK: Input: default@hbase_null
    +#### A masked pattern was here ####
    +POSTHOOK: query: SELECT COUNT(*) FROM hbase_null
    +POSTHOOK: type: QUERY
    +POSTHOOK: Input: default@hbase_null
    +#### A masked pattern was here ####
    +10
    +PREHOOK: query: DROP TABLE src_null
    +PREHOOK: type: DROPTABLE
    +PREHOOK: Input: default@src_null
    +PREHOOK: Output: default@src_null
    +POSTHOOK: query: DROP TABLE src_null
    +POSTHOOK: type: DROPTABLE
    +POSTHOOK: Input: default@src_null
    +POSTHOOK: Output: default@src_null
    +PREHOOK: query: DROP TABLE hbase_null
    +PREHOOK: type: DROPTABLE
    +PREHOOK: Input: default@hbase_null
    +PREHOOK: Output: default@hbase_null
    +POSTHOOK: query: DROP TABLE hbase_null
    +POSTHOOK: type: DROPTABLE
    +POSTHOOK: Input: default@hbase_null
    +POSTHOOK: Output: default@hbase_null
  • Sershe at Aug 25, 2015 at 9:28 pm
    HIVE-11375: Broken processing of queries containing NOT (x IS NOT NULL and x <> 0) (Aihua Xu, reviewed by Ashutosh Chauhan)


    Project: http://git-wip-us.apache.org/repos/asf/hive/repo
    Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/a1626749
    Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/a1626749
    Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/a1626749

    Branch: refs/heads/llap
    Commit: a16267491978a35f4dd0478405b88a74725050a5
    Parents: e3664e2
    Author: Aihua Xu <aihuaxu@gmail.com>
    Authored: Fri Aug 21 10:39:21 2015 -0700
    Committer: Chao Sun <sunchao@apache.org>
    Committed: Fri Aug 21 10:39:21 2015 -0700

    ----------------------------------------------------------------------
      .../optimizer/ConstantPropagateProcFactory.java | 100 ++++-
      .../hadoop/hive/ql/udf/generic/GenericUDF.java | 14 +-
      .../hive/ql/udf/generic/GenericUDFOPAnd.java | 4 +
      .../hive/ql/udf/generic/GenericUDFOPEqual.java | 4 +
      .../generic/GenericUDFOPEqualOrGreaterThan.java | 4 +
      .../generic/GenericUDFOPEqualOrLessThan.java | 4 +
      .../ql/udf/generic/GenericUDFOPGreaterThan.java | 4 +
      .../ql/udf/generic/GenericUDFOPLessThan.java | 4 +
      .../ql/udf/generic/GenericUDFOPNotEqual.java | 5 +
      .../ql/udf/generic/GenericUDFOPNotNull.java | 4 +
      .../hive/ql/udf/generic/GenericUDFOPNull.java | 4 +
      .../hive/ql/udf/generic/GenericUDFOPOr.java | 4 +
      .../queries/clientpositive/folder_predicate.q | 32 ++
      .../clientpositive/annotate_stats_filter.q.out | 10 +-
      .../results/clientpositive/decimal_udf.q.out | 18 +-
      .../clientpositive/filter_cond_pushdown.q.out | 32 +-
      .../clientpositive/filter_join_breaktask.q.out | 12 +-
      .../test/results/clientpositive/fold_when.q.out | 16 +-
      .../clientpositive/folder_predicate.q.out | 368 +++++++++++++++++++
      .../clientpositive/input_testxpath2.q.out | 2 +-
      .../list_bucket_query_oneskew_3.q.out | 6 +-
      .../clientpositive/rand_partitionpruner3.q.out | 12 +-
      .../clientpositive/select_unquote_not.q.out | 8 +-
      .../spark/filter_join_breaktask.q.out | 12 +-
      .../tez/filter_join_breaktask.q.out | 12 +-
      .../clientpositive/tez/vector_decimal_udf.q.out | 24 +-
      .../clientpositive/udf_isnull_isnotnull.q.out | 2 +-
      .../test/results/clientpositive/udf_size.q.out | 2 +-
      .../clientpositive/vector_decimal_udf.q.out | 24 +-
      29 files changed, 640 insertions(+), 107 deletions(-)
    ----------------------------------------------------------------------


    http://git-wip-us.apache.org/repos/asf/hive/blob/a1626749/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConstantPropagateProcFactory.java
    ----------------------------------------------------------------------
    diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConstantPropagateProcFactory.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConstantPropagateProcFactory.java
    index 55ad0ce..5c6a6df 100644
    --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConstantPropagateProcFactory.java
    +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConstantPropagateProcFactory.java
    @@ -18,6 +18,7 @@ package org.apache.hadoop.hive.ql.optimizer;
      import java.io.Serializable;
      import java.util.ArrayList;
      import java.util.BitSet;
    +import java.util.Arrays;
      import java.util.HashMap;
      import java.util.HashSet;
      import java.util.Iterator;
    @@ -70,6 +71,10 @@ import org.apache.hadoop.hive.ql.udf.generic.GenericUDFBridge;
      import org.apache.hadoop.hive.ql.udf.generic.GenericUDFCase;
      import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPAnd;
      import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPEqual;
    +import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPEqualOrGreaterThan;
    +import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPEqualOrLessThan;
    +import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPGreaterThan;
    +import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPLessThan;
      import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPNot;
      import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPNotEqual;
      import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPNotNull;
    @@ -94,6 +99,7 @@ import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
      import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
      import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils;

    +import com.google.common.collect.ImmutableMap;
      import com.google.common.collect.ImmutableSet;
      import com.google.common.collect.Lists;

    @@ -223,6 +229,65 @@ public final class ConstantPropagateProcFactory {
        }

        /**
    + * Combines the logical not() operator with the child operator if possible.
    + * @param desc the expression to be evaluated
    + * @return the new expression to be replaced
    + * @throws UDFArgumentException
    + */
    + private static ExprNodeDesc foldNegative(ExprNodeDesc desc) throws UDFArgumentException {
    + if (desc instanceof ExprNodeGenericFuncDesc) {
    + ExprNodeGenericFuncDesc funcDesc = (ExprNodeGenericFuncDesc) desc;
    +
    + GenericUDF udf = funcDesc.getGenericUDF();
    + if (udf instanceof GenericUDFOPNot) {
    + ExprNodeDesc child = funcDesc.getChildren().get(0);
    + if (child instanceof ExprNodeGenericFuncDesc) {
    + ExprNodeGenericFuncDesc childDesc = (ExprNodeGenericFuncDesc)child;
    + GenericUDF childUDF = childDesc.getGenericUDF();
    + List<ExprNodeDesc> grandChildren = child.getChildren();
    +
    + if (childUDF instanceof GenericUDFBaseCompare ||
    + childUDF instanceof GenericUDFOPNull ||
    + childUDF instanceof GenericUDFOPNotNull) {
    + List<ExprNodeDesc> newGrandChildren = new ArrayList<ExprNodeDesc>();
    + for(ExprNodeDesc grandChild : grandChildren) {
    + newGrandChildren.add(foldNegative(grandChild));
    + }
    +
    + return ExprNodeGenericFuncDesc.newInstance(
    + childUDF.negative(),
    + newGrandChildren);
    + } else if (childUDF instanceof GenericUDFOPAnd ||
    + childUDF instanceof GenericUDFOPOr) {
    + List<ExprNodeDesc> newGrandChildren = new ArrayList<ExprNodeDesc>();
    + for(ExprNodeDesc grandChild : grandChildren) {
    + newGrandChildren.add(foldNegative(
    + ExprNodeGenericFuncDesc.newInstance(new GenericUDFOPNot(),
    + Arrays.asList(grandChild))));
    + }
    +
    + return ExprNodeGenericFuncDesc.newInstance(
    + childUDF.negative(),
    + newGrandChildren);
    + }else if (childUDF instanceof GenericUDFOPNot) {
    + return foldNegative(child.getChildren().get(0));
    + } else {
    + // For operator like if() that cannot be handled, leave not() as it
    + // is and continue processing the children
    + List<ExprNodeDesc> newGrandChildren = new ArrayList<ExprNodeDesc>();
    + for(ExprNodeDesc grandChild : grandChildren) {
    + newGrandChildren.add(foldNegative(grandChild));
    + }
    + childDesc.setChildren(newGrandChildren);
    + return funcDesc;
    + }
    + }
    + }
    + }
    + return desc;
    + }
    +
    + /**
         * Fold input expression desc, only performing short-cutting.
         *
         * Unnecessary AND/OR operations involving a constant true/false value will be eliminated.
    @@ -238,6 +303,11 @@ public final class ConstantPropagateProcFactory {
        private static ExprNodeDesc foldExprShortcut(ExprNodeDesc desc, Map<ColumnInfo, ExprNodeDesc> constants,
            ConstantPropagateProcCtx cppCtx, Operator<? extends Serializable> op, int tag,
            boolean propagate) throws UDFArgumentException {
    + // Combine NOT operator with the child operator. Otherwise, the following optimization
    + // from bottom up could lead to incorrect result, such as not(x > 3 and x is not null),
    + // should not be optimized to not(x > 3), but (x <=3 or x is null).
    + desc = foldNegative(desc);
    +
          if (desc instanceof ExprNodeGenericFuncDesc) {
            ExprNodeGenericFuncDesc funcDesc = (ExprNodeGenericFuncDesc) desc;

    @@ -249,10 +319,10 @@ public final class ConstantPropagateProcFactory {
              newExprs.add(foldExpr(childExpr, constants, cppCtx, op, tag, propagateNext));
            }

    - // Don't evalulate nondeterministic function since the value can only calculate during runtime.
    + // Don't evaluate nondeterministic function since the value can only calculate during runtime.
            if (!isDeterministicUdf(udf)) {
              if (LOG.isDebugEnabled()) {
    - LOG.debug("Function " + udf.getClass() + " is undeterministic. Don't evalulating immediately.");
    + LOG.debug("Function " + udf.getClass() + " is undeterministic. Don't evalulate immediately.");
              }
              ((ExprNodeGenericFuncDesc) desc).setChildren(newExprs);
              return desc;
    @@ -276,11 +346,11 @@ public final class ConstantPropagateProcFactory {
         *
         * This function recursively checks if any subexpression of a specified expression
         * can be evaluated to be constant and replaces such subexpression with the constant.
    - * If the expression is a derterministic UDF and all the subexpressions are constants,
    + * If the expression is a deterministic UDF and all the subexpressions are constants,
         * the value will be calculated immediately (during compilation time vs. runtime).
         * e.g.:
         * concat(year, month) => 200112 for year=2001, month=12 since concat is deterministic UDF
    - * unix_timestamp(time) => unix_timestamp(123) for time=123 since unix_timestamp is nonderministic UDF
    + * unix_timestamp(time) => unix_timestamp(123) for time=123 since unix_timestamp is nondeterministic UDF
         * @param desc folding expression
         * @param constants current propagated constant map
         * @param cppCtx
    @@ -292,6 +362,11 @@ public final class ConstantPropagateProcFactory {
        private static ExprNodeDesc foldExprFull(ExprNodeDesc desc, Map<ColumnInfo, ExprNodeDesc> constants,
            ConstantPropagateProcCtx cppCtx, Operator<? extends Serializable> op, int tag,
            boolean propagate) throws UDFArgumentException {
    + // Combine NOT operator with the child operator. Otherwise, the following optimization
    + // from bottom up could lead to incorrect result, such as not(x > 3 and x is not null),
    + // should not be optimized to not(x > 3), but (x <=3 or x is null).
    + desc = foldNegative(desc);
    +
          if (desc instanceof ExprNodeGenericFuncDesc) {
            ExprNodeGenericFuncDesc funcDesc = (ExprNodeGenericFuncDesc) desc;

    @@ -303,10 +378,10 @@ public final class ConstantPropagateProcFactory {
              newExprs.add(foldExpr(childExpr, constants, cppCtx, op, tag, propagateNext));
            }

    - // Don't evalulate nondeterministic function since the value can only calculate during runtime.
    + // Don't evaluate nondeterministic function since the value can only calculate during runtime.
            if (!isDeterministicUdf(udf)) {
              if (LOG.isDebugEnabled()) {
    - LOG.debug("Function " + udf.getClass() + " is undeterministic. Don't evalulating immediately.");
    + LOG.debug("Function " + udf.getClass() + " is undeterministic. Don't evaluate immediately.");
              }
              ((ExprNodeGenericFuncDesc) desc).setChildren(newExprs);
              return desc;
    @@ -460,7 +535,6 @@ public final class ConstantPropagateProcFactory {

        private static ExprNodeDesc shortcutFunction(GenericUDF udf, List<ExprNodeDesc> newExprs,
          Operator<? extends Serializable> op) throws UDFArgumentException {
    -
          if (udf instanceof GenericUDFOPEqual) {
           assert newExprs.size() == 2;
           boolean foundUDFInFirst = false;
    @@ -511,6 +585,7 @@ public final class ConstantPropagateProcFactory {
             return null;
           }
          }
    +
          if (udf instanceof GenericUDFOPAnd) {
            final BitSet positionsToRemove = new BitSet();
            final List<ExprNodeDesc> notNullExprs = new ArrayList<ExprNodeDesc>();
    @@ -532,9 +607,13 @@ public final class ConstantPropagateProcFactory {
                      childExpr.getChildren().get(0) instanceof ExprNodeColumnDesc) {
                notNullExprs.add(childExpr.getChildren().get(0));
                notNullExprsPositions.add(i);
    - } else if (childExpr instanceof ExprNodeGenericFuncDesc
    - && ((ExprNodeGenericFuncDesc)childExpr).getGenericUDF() instanceof GenericUDFBaseCompare
    - && childExpr.getChildren().size() == 2) {
    + } else if (childExpr instanceof ExprNodeGenericFuncDesc &&
    + ((ExprNodeGenericFuncDesc)childExpr).getGenericUDF() instanceof GenericUDFBaseCompare &&
    + !(((ExprNodeGenericFuncDesc)childExpr).getGenericUDF() instanceof GenericUDFOPNotEqual) &&
    + childExpr.getChildren().size() == 2) {
    + // Try to fold (key <op> 86) and (key is not null) to (key <op> 86)
    + // where <op> can be "=", ">=", "<=", ">", "<".
    + // Note: (key <> 86) and (key is not null) cannot be folded
                ExprNodeColumnDesc colDesc = getColumnExpr(childExpr.getChildren().get(0));
                if (null == colDesc) {
                  colDesc = getColumnExpr(childExpr.getChildren().get(1));
    @@ -646,6 +725,7 @@ public final class ConstantPropagateProcFactory {
              }
            }
          }
    +
          if (udf instanceof GenericUDFCase) {
            // HIVE-9644 Attempt to fold expression like :
            // where (case ss_sold_date when '1998-01-01' then 1=1 else null=1 end);

    http://git-wip-us.apache.org/repos/asf/hive/blob/a1626749/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDF.java
    ----------------------------------------------------------------------
    diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDF.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDF.java
    index e8df5d3..259fde8 100644
    --- a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDF.java
    +++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDF.java
    @@ -213,12 +213,24 @@ public abstract class GenericUDF implements Closeable {
        }

        /**
    - * Some functions are affected by appearing order of arguments (comparisons, for example)
    + * Some functions like comparisons may be affected by appearing order of arguments.
    + * This is to convert a function, such as 3 > x to x < 3. The flip function of
    + * GenericUDFOPGreaterThan is GenericUDFOPLessThan.
         */
        public GenericUDF flip() {
          return this;
        }

    + /**
    + * Gets the negative function of the current one. E.g., GenericUDFOPNotEqual for
    + * GenericUDFOPEqual, or GenericUDFOPNull for GenericUDFOPNotNull.
    + * @return Negative function
    + * @throws UDFArgumentException
    + */
    + public GenericUDF negative() {
    + throw new UnsupportedOperationException("Negative function doesn't exist for " + getFuncName());
    + }
    +
        public String getUdfName() {
          return getClass().getName();
        }

    http://git-wip-us.apache.org/repos/asf/hive/blob/a1626749/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPAnd.java
    ----------------------------------------------------------------------
    diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPAnd.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPAnd.java
    index db7fbac..eb33d98 100644
    --- a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPAnd.java
    +++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPAnd.java
    @@ -97,4 +97,8 @@ public class GenericUDFOPAnd extends GenericUDF {
          return sb.toString();
        }

    + @Override
    + public GenericUDF negative() {
    + return new GenericUDFOPOr();
    + }
      }

    http://git-wip-us.apache.org/repos/asf/hive/blob/a1626749/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPEqual.java
    ----------------------------------------------------------------------
    diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPEqual.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPEqual.java
    index 428ced7..23f7f1f 100644
    --- a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPEqual.java
    +++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPEqual.java
    @@ -122,4 +122,8 @@ public class GenericUDFOPEqual extends GenericUDFBaseCompare {
          return result;
        }

    + @Override
    + public GenericUDF negative() {
    + return new GenericUDFOPNotEqual();
    + }
      }

    http://git-wip-us.apache.org/repos/asf/hive/blob/a1626749/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPEqualOrGreaterThan.java
    ----------------------------------------------------------------------
    diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPEqualOrGreaterThan.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPEqualOrGreaterThan.java
    index d9556cc..bd5b61a 100644
    --- a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPEqualOrGreaterThan.java
    +++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPEqualOrGreaterThan.java
    @@ -134,4 +134,8 @@ public class GenericUDFOPEqualOrGreaterThan extends GenericUDFBaseCompare {
          return new GenericUDFOPEqualOrLessThan();
        }

    + @Override
    + public GenericUDF negative() {
    + return new GenericUDFOPLessThan();
    + }
      }

    http://git-wip-us.apache.org/repos/asf/hive/blob/a1626749/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPEqualOrLessThan.java
    ----------------------------------------------------------------------
    diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPEqualOrLessThan.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPEqualOrLessThan.java
    index 1d9eaf6..ad971cd 100644
    --- a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPEqualOrLessThan.java
    +++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPEqualOrLessThan.java
    @@ -134,5 +134,9 @@ public class GenericUDFOPEqualOrLessThan extends GenericUDFBaseCompare {
          return new GenericUDFOPEqualOrGreaterThan();
        }

    + @Override
    + public GenericUDF negative() {
    + return new GenericUDFOPGreaterThan();
    + }
      }


    http://git-wip-us.apache.org/repos/asf/hive/blob/a1626749/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPGreaterThan.java
    ----------------------------------------------------------------------
    diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPGreaterThan.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPGreaterThan.java
    index 8e1f2b1..85f0601 100644
    --- a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPGreaterThan.java
    +++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPGreaterThan.java
    @@ -134,5 +134,9 @@ public class GenericUDFOPGreaterThan extends GenericUDFBaseCompare {
          return new GenericUDFOPLessThan();
        }

    + @Override
    + public GenericUDF negative() {
    + return new GenericUDFOPEqualOrLessThan();
    + }
      }


    http://git-wip-us.apache.org/repos/asf/hive/blob/a1626749/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPLessThan.java
    ----------------------------------------------------------------------
    diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPLessThan.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPLessThan.java
    index 101b348..b445ae6 100644
    --- a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPLessThan.java
    +++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPLessThan.java
    @@ -134,4 +134,8 @@ public class GenericUDFOPLessThan extends GenericUDFBaseCompare {
          return new GenericUDFOPGreaterThan();
        }

    + @Override
    + public GenericUDF negative() {
    + return new GenericUDFOPEqualOrGreaterThan();
    + }
      }

    http://git-wip-us.apache.org/repos/asf/hive/blob/a1626749/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPNotEqual.java
    ----------------------------------------------------------------------
    diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPNotEqual.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPNotEqual.java
    index b5da57a..ac8ec2f 100644
    --- a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPNotEqual.java
    +++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPNotEqual.java
    @@ -121,4 +121,9 @@ public class GenericUDFOPNotEqual extends GenericUDFBaseCompare {
          }
          return result;
        }
    +
    + @Override
    + public GenericUDF negative() {
    + return new GenericUDFOPEqual();
    + }
      }

    http://git-wip-us.apache.org/repos/asf/hive/blob/a1626749/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPNotNull.java
    ----------------------------------------------------------------------
    diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPNotNull.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPNotNull.java
    index d22b35d..2b67c38 100644
    --- a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPNotNull.java
    +++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPNotNull.java
    @@ -60,4 +60,8 @@ public class GenericUDFOPNotNull extends GenericUDF {
          return children[0] + " is not null";
        }

    + @Override
    + public GenericUDF negative() {
    + return new GenericUDFOPNull();
    + }
      }

    http://git-wip-us.apache.org/repos/asf/hive/blob/a1626749/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPNull.java
    ----------------------------------------------------------------------
    diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPNull.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPNull.java
    index fe20e9a..4eb92eb 100644
    --- a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPNull.java
    +++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPNull.java
    @@ -60,4 +60,8 @@ public class GenericUDFOPNull extends GenericUDF {
          return children[0] + " is null";
        }

    + @Override
    + public GenericUDF negative() {
    + return new GenericUDFOPNotNull();
    + }
      }

    http://git-wip-us.apache.org/repos/asf/hive/blob/a1626749/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPOr.java
    ----------------------------------------------------------------------
    diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPOr.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPOr.java
    index 4160610..8de59c1 100644
    --- a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPOr.java
    +++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFOPOr.java
    @@ -98,4 +98,8 @@ public class GenericUDFOPOr extends GenericUDF {
          return sb.toString();
        }

    + @Override
    + public GenericUDF negative() {
    + return new GenericUDFOPAnd();
    + }
      }

    http://git-wip-us.apache.org/repos/asf/hive/blob/a1626749/ql/src/test/queries/clientpositive/folder_predicate.q
    ----------------------------------------------------------------------
    diff --git a/ql/src/test/queries/clientpositive/folder_predicate.q b/ql/src/test/queries/clientpositive/folder_predicate.q
    new file mode 100644
    index 0000000..2377dd4
    --- /dev/null
    +++ b/ql/src/test/queries/clientpositive/folder_predicate.q
    @@ -0,0 +1,32 @@
    +drop table if exists predicate_fold_tb;
    +
    +create table predicate_fold_tb(value int);
    +insert into predicate_fold_tb values(NULL), (1), (2), (3), (4), (5);
    +
    +explain
    +SELECT * FROM predicate_fold_tb WHERE not(value IS NOT NULL AND value = 3);
    +SELECT * FROM predicate_fold_tb WHERE not(value IS NOT NULL AND value = 3);
    +
    +explain
    +SELECT * FROM predicate_fold_tb WHERE not(value IS NOT NULL AND value >= 3);
    +SELECT * FROM predicate_fold_tb WHERE not(value IS NOT NULL AND value >= 3);
    +
    +explain
    +SELECT * FROM predicate_fold_tb WHERE not(value IS NOT NULL AND value <= 3);
    +SELECT * FROM predicate_fold_tb WHERE not(value IS NOT NULL AND value <= 3);
    +
    +explain
    +SELECT * FROM predicate_fold_tb WHERE not(value IS NOT NULL AND value > 3);
    +SELECT * FROM predicate_fold_tb WHERE not(value IS NOT NULL AND value > 3);
    +
    +explain
    +SELECT * FROM predicate_fold_tb WHERE not(value IS NOT NULL AND value < 3);
    +SELECT * FROM predicate_fold_tb WHERE not(value IS NOT NULL AND value < 3);
    +
    +explain
    +SELECT * FROM predicate_fold_tb WHERE not(value IS NOT NULL AND value <> 3);
    +SELECT * FROM predicate_fold_tb WHERE not(value IS NOT NULL AND value <> 3);
    +
    +explain
    +SELECT * FROM predicate_fold_tb WHERE not(value IS NOT NULL AND value > 1 AND value <=3);
    +SELECT * FROM predicate_fold_tb WHERE not(value IS NOT NULL AND value > 1 AND value <=3);

    http://git-wip-us.apache.org/repos/asf/hive/blob/a1626749/ql/src/test/results/clientpositive/annotate_stats_filter.q.out
    ----------------------------------------------------------------------
    diff --git a/ql/src/test/results/clientpositive/annotate_stats_filter.q.out b/ql/src/test/results/clientpositive/annotate_stats_filter.q.out
    index af1e1c3..9e0e78a 100644
    --- a/ql/src/test/results/clientpositive/annotate_stats_filter.q.out
    +++ b/ql/src/test/results/clientpositive/annotate_stats_filter.q.out
    @@ -297,15 +297,15 @@ STAGE PLANS:
                  alias: loc_orc
                  Statistics: Num rows: 8 Data size: 796 Basic stats: COMPLETE Column stats: COMPLETE
                  Filter Operator
    - predicate: (not zip is not null) (type: boolean)
    + predicate: zip is null (type: boolean)
                    Statistics: Num rows: 1 Data size: 102 Basic stats: COMPLETE Column stats: COMPLETE
                    Select Operator
    - expressions: state (type: string), locid (type: int), zip (type: bigint), year (type: int)
    + expressions: state (type: string), locid (type: int), null (type: bigint), year (type: int)
                      outputColumnNames: _col0, _col1, _col2, _col3
    - Statistics: Num rows: 1 Data size: 102 Basic stats: COMPLETE Column stats: COMPLETE
    + Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: COMPLETE
                      File Output Operator
                        compressed: false
    - Statistics: Num rows: 1 Data size: 102 Basic stats: COMPLETE Column stats: COMPLETE
    + Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: COMPLETE
                        table:
                            input format: org.apache.hadoop.mapred.TextInputFormat
                            output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
    @@ -375,7 +375,7 @@ STAGE PLANS:
                  alias: loc_orc
                  Statistics: Num rows: 8 Data size: 796 Basic stats: COMPLETE Column stats: COMPLETE
                  Filter Operator
    - predicate: (not zip is null) (type: boolean)
    + predicate: zip is not null (type: boolean)
                    Statistics: Num rows: 7 Data size: 702 Basic stats: COMPLETE Column stats: COMPLETE
                    Select Operator
                      expressions: state (type: string), locid (type: int), zip (type: bigint), year (type: int)

    http://git-wip-us.apache.org/repos/asf/hive/blob/a1626749/ql/src/test/results/clientpositive/decimal_udf.q.out
    ----------------------------------------------------------------------
    diff --git a/ql/src/test/results/clientpositive/decimal_udf.q.out b/ql/src/test/results/clientpositive/decimal_udf.q.out
    index f36670c..338b754 100644
    --- a/ql/src/test/results/clientpositive/decimal_udf.q.out
    +++ b/ql/src/test/results/clientpositive/decimal_udf.q.out
    @@ -973,12 +973,12 @@ STAGE PLANS:
                alias: decimal_udf
                Statistics: Num rows: 3 Data size: 359 Basic stats: COMPLETE Column stats: NONE
                Filter Operator
    - predicate: (key <> 0) (type: boolean)
    - Statistics: Num rows: 3 Data size: 359 Basic stats: COMPLETE Column stats: NONE
    + predicate: (key is not null and (key <> 0)) (type: boolean)
    + Statistics: Num rows: 2 Data size: 239 Basic stats: COMPLETE Column stats: NONE
                  Select Operator
                    expressions: (key / key) (type: decimal(38,24))
                    outputColumnNames: _col0
    - Statistics: Num rows: 3 Data size: 359 Basic stats: COMPLETE Column stats: NONE
    + Statistics: Num rows: 2 Data size: 239 Basic stats: COMPLETE Column stats: NONE
                    ListSink

      PREHOOK: query: SELECT key / key FROM DECIMAL_UDF WHERE key is not null and key <> 0
    @@ -1039,12 +1039,12 @@ STAGE PLANS:
                alias: decimal_udf
                Statistics: Num rows: 3 Data size: 359 Basic stats: COMPLETE Column stats: NONE
                Filter Operator
    - predicate: (value <> 0) (type: boolean)
    - Statistics: Num rows: 3 Data size: 359 Basic stats: COMPLETE Column stats: NONE
    + predicate: (value is not null and (value <> 0)) (type: boolean)
    + Statistics: Num rows: 2 Data size: 239 Basic stats: COMPLETE Column stats: NONE
                  Select Operator
                    expressions: (key / CAST( value AS decimal(10,0))) (type: decimal(31,21))
                    outputColumnNames: _col0
    - Statistics: Num rows: 3 Data size: 359 Basic stats: COMPLETE Column stats: NONE
    + Statistics: Num rows: 2 Data size: 239 Basic stats: COMPLETE Column stats: NONE
                    ListSink

      PREHOOK: query: SELECT key / value FROM DECIMAL_UDF WHERE value is not null and value <> 0
    @@ -1095,12 +1095,12 @@ STAGE PLANS:
                alias: decimal_udf
                Statistics: Num rows: 3 Data size: 359 Basic stats: COMPLETE Column stats: NONE
                Filter Operator
    - predicate: (value <> 0) (type: boolean)
    - Statistics: Num rows: 3 Data size: 359 Basic stats: COMPLETE Column stats: NONE
    + predicate: (value is not null and (value <> 0)) (type: boolean)
    + Statistics: Num rows: 2 Data size: 239 Basic stats: COMPLETE Column stats: NONE
                  Select Operator
                    expressions: (UDFToDouble(key) / (UDFToDouble(value) / 2.0)) (type: double)
                    outputColumnNames: _col0
    - Statistics: Num rows: 3 Data size: 359 Basic stats: COMPLETE Column stats: NONE
    + Statistics: Num rows: 2 Data size: 239 Basic stats: COMPLETE Column stats: NONE
                    ListSink

      PREHOOK: query: SELECT key / (value/2) FROM DECIMAL_UDF WHERE value is not null and value <> 0

    http://git-wip-us.apache.org/repos/asf/hive/blob/a1626749/ql/src/test/results/clientpositive/filter_cond_pushdown.q.out
    ----------------------------------------------------------------------
    diff --git a/ql/src/test/results/clientpositive/filter_cond_pushdown.q.out b/ql/src/test/results/clientpositive/filter_cond_pushdown.q.out
    index e09057a..af42d5c 100644
    --- a/ql/src/test/results/clientpositive/filter_cond_pushdown.q.out
    +++ b/ql/src/test/results/clientpositive/filter_cond_pushdown.q.out
    @@ -37,7 +37,7 @@ STAGE PLANS:
                  alias: f
                  Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
                  Filter Operator
    - predicate: (((value <> '') and key is not null) and value is not null) (type: boolean)
    + predicate: ((value is not null and (value <> '')) and key is not null) (type: boolean)
                    Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE
                    Select Operator
                      expressions: key (type: string), value (type: string)
    @@ -86,17 +86,17 @@ STAGE PLANS:
                  alias: f
                  Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
                  Filter Operator
    - predicate: (value <> '') (type: boolean)
    - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
    + predicate: (value is not null and (value <> '')) (type: boolean)
    + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
                    Select Operator
                      expressions: value (type: string)
                      outputColumnNames: _col0
    - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
    + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
                      Reduce Output Operator
                        key expressions: _col0 (type: string)
                        sort order: +
                        Map-reduce partition columns: _col0 (type: string)
    - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
    + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
            Reduce Operator Tree:
              Join Operator
                condition map:
    @@ -105,14 +105,14 @@ STAGE PLANS:
                  0 _col3 (type: string)
                  1 _col0 (type: string)
                outputColumnNames: _col0, _col4
    - Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE
    + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE
                Select Operator
                  expressions: _col0 (type: string), _col4 (type: string)
                  outputColumnNames: _col0, _col1
    - Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE
    + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE
                  File Output Operator
                    compressed: false
    - Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE
    + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE
                    table:
                        input format: org.apache.hadoop.mapred.TextInputFormat
                        output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
    @@ -163,7 +163,7 @@ STAGE PLANS:
                  alias: f
                  Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
                  Filter Operator
    - predicate: (((value <> '') and key is not null) and value is not null) (type: boolean)
    + predicate: ((value is not null and (value <> '')) and key is not null) (type: boolean)
                    Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE
                    Select Operator
                      expressions: key (type: string), value (type: string)
    @@ -212,17 +212,17 @@ STAGE PLANS:
                  alias: f
                  Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
                  Filter Operator
    - predicate: (value <> '') (type: boolean)
    - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
    + predicate: (value is not null and (value <> '')) (type: boolean)
    + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
                    Select Operator
                      expressions: value (type: string)
                      outputColumnNames: _col0
    - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
    + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
                      Reduce Output Operator
                        key expressions: _col0 (type: string)
                        sort order: +
                        Map-reduce partition columns: _col0 (type: string)
    - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
    + Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
            Reduce Operator Tree:
              Join Operator
                condition map:
    @@ -231,14 +231,14 @@ STAGE PLANS:
                  0 _col3 (type: string)
                  1 _col0 (type: string)
                outputColumnNames: _col0, _col4
    - Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE
    + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE
                Select Operator
                  expressions: _col0 (type: string), _col4 (type: string)
                  outputColumnNames: _col0, _col1
    - Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE
    + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE
                  File Output Operator
                    compressed: false
    - Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE
    + Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE
                    table:
                        input format: org.apache.hadoop.mapred.TextInputFormat
                        output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat

    http://git-wip-us.apache.org/repos/asf/hive/blob/a1626749/ql/src/test/results/clientpositive/filter_join_breaktask.q.out
    ----------------------------------------------------------------------
    diff --git a/ql/src/test/results/clientpositive/filter_join_breaktask.q.out b/ql/src/test/results/clientpositive/filter_join_breaktask.q.out
    index 948ad0d..83f4720 100644
    --- a/ql/src/test/results/clientpositive/filter_join_breaktask.q.out
    +++ b/ql/src/test/results/clientpositive/filter_join_breaktask.q.out
    @@ -271,13 +271,13 @@ STAGE PLANS:
                  GatherStats: false
                  Filter Operator
                    isSamplingPred: false
    - predicate: (value <> '') (type: boolean)
    - Statistics: Num rows: 25 Data size: 211 Basic stats: COMPLETE Column stats: NONE
    + predicate: ((value <> '') and value is not null) (type: boolean)
    + Statistics: Num rows: 13 Data size: 109 Basic stats: COMPLETE Column stats: NONE
                    Reduce Output Operator
                      key expressions: value (type: string)
                      sort order: +
                      Map-reduce partition columns: value (type: string)
    - Statistics: Num rows: 25 Data size: 211 Basic stats: COMPLETE Column stats: NONE
    + Statistics: Num rows: 13 Data size: 109 Basic stats: COMPLETE Column stats: NONE
                      tag: 1
                      auto parallelism: false
            Path -> Alias:
    @@ -360,17 +360,17 @@ STAGE PLANS:
                  0 _col7 (type: string)
                  1 value (type: string)
                outputColumnNames: _col0, _col13
    - Statistics: Num rows: 27 Data size: 232 Basic stats: COMPLETE Column stats: NONE
    + Statistics: Num rows: 15 Data size: 130 Basic stats: COMPLETE Column stats: NONE
                Select Operator
                  expressions: _col0 (type: int), _col13 (type: string)
                  outputColumnNames: _col0, _col1
    - Statistics: Num rows: 27 Data size: 232 Basic stats: COMPLETE Column stats: NONE
    + Statistics: Num rows: 15 Data size: 130 Basic stats: COMPLETE Column stats: NONE
                  File Output Operator
                    compressed: false
                    GlobalTableId: 0
      #### A masked pattern was here ####
                    NumFilesPerFileSink: 1
    - Statistics: Num rows: 27 Data size: 232 Basic stats: COMPLETE Column stats: NONE
    + Statistics: Num rows: 15 Data size: 130 Basic stats: COMPLETE Column stats: NONE
      #### A masked pattern was here ####
                    table:
                        input format: org.apache.hadoop.mapred.TextInputFormat

    http://git-wip-us.apache.org/repos/asf/hive/blob/a1626749/ql/src/test/results/clientpositive/fold_when.q.out
    ----------------------------------------------------------------------
    diff --git a/ql/src/test/results/clientpositive/fold_when.q.out b/ql/src/test/results/clientpositive/fold_when.q.out
    index 21748da..bf61a7d 100644
    --- a/ql/src/test/results/clientpositive/fold_when.q.out
    +++ b/ql/src/test/results/clientpositive/fold_when.q.out
    @@ -328,15 +328,15 @@ STAGE PLANS:
                  alias: src
                  Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
                  Filter Operator
    - predicate: (not (key = '238')) (type: boolean)
    - Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
    + predicate: (key <> '238') (type: boolean)
    + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
                    Select Operator
                      expressions: key (type: string)
                      outputColumnNames: _col0
    - Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
    + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
                      File Output Operator
                        compressed: false
    - Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
    + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
                        table:
                            input format: org.apache.hadoop.mapred.TextInputFormat
                            output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
    @@ -435,15 +435,15 @@ STAGE PLANS:
                  alias: src
                  Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
                  Filter Operator
    - predicate: (not (key = '11')) (type: boolean)
    - Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
    + predicate: (key <> '11') (type: boolean)
    + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
                    Select Operator
                      expressions: key (type: string)
                      outputColumnNames: _col0
    - Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
    + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
                      File Output Operator
                        compressed: false
    - Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
    + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
                        table:
                            input format: org.apache.hadoop.mapred.TextInputFormat
                            output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat

    http://git-wip-us.apache.org/repos/asf/hive/blob/a1626749/ql/src/test/results/clientpositive/folder_predicate.q.out
    ----------------------------------------------------------------------
    diff --git a/ql/src/test/results/clientpositive/folder_predicate.q.out b/ql/src/test/results/clientpositive/folder_predicate.q.out
    new file mode 100644
    index 0000000..1bc1afb
    --- /dev/null
    +++ b/ql/src/test/results/clientpositive/folder_predicate.q.out
    @@ -0,0 +1,368 @@
    +PREHOOK: query: drop table if exists predicate_fold_tb
    +PREHOOK: type: DROPTABLE
    +POSTHOOK: query: drop table if exists predicate_fold_tb
    +POSTHOOK: type: DROPTABLE
    +PREHOOK: query: create table predicate_fold_tb(value int)
    +PREHOOK: type: CREATETABLE
    +PREHOOK: Output: database:default
    +PREHOOK: Output: default@predicate_fold_tb
    +POSTHOOK: query: create table predicate_fold_tb(value int)
    +POSTHOOK: type: CREATETABLE
    +POSTHOOK: Output: database:default
    +POSTHOOK: Output: default@predicate_fold_tb
    +PREHOOK: query: insert into predicate_fold_tb values(NULL), (1), (2), (3), (4), (5)
    +PREHOOK: type: QUERY
    +PREHOOK: Input: default@values__tmp__table__1
    +PREHOOK: Output: default@predicate_fold_tb
    +POSTHOOK: query: insert into predicate_fold_tb values(NULL), (1), (2), (3), (4), (5)
    +POSTHOOK: type: QUERY
    +POSTHOOK: Input: default@values__tmp__table__1
    +POSTHOOK: Output: default@predicate_fold_tb
    +POSTHOOK: Lineage: predicate_fold_tb.value EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col1, type:string, comment:), ]
    +PREHOOK: query: explain
    +SELECT * FROM predicate_fold_tb WHERE not(value IS NOT NULL AND value = 3)
    +PREHOOK: type: QUERY
    +POSTHOOK: query: explain
    +SELECT * FROM predicate_fold_tb WHERE not(value IS NOT NULL AND value = 3)
    +POSTHOOK: type: QUERY
    +STAGE DEPENDENCIES:
    + Stage-1 is a root stage
    + Stage-0 depends on stages: Stage-1
    +
    +STAGE PLANS:
    + Stage: Stage-1
    + Map Reduce
    + Map Operator Tree:
    + TableScan
    + alias: predicate_fold_tb
    + Statistics: Num rows: 6 Data size: 7 Basic stats: COMPLETE Column stats: NONE
    + Filter Operator
    + predicate: (value is null or (value <> 3)) (type: boolean)
    + Statistics: Num rows: 6 Data size: 7 Basic stats: COMPLETE Column stats: NONE
    + Select Operator
    + expressions: value (type: int)
    + outputColumnNames: _col0
    + Statistics: Num rows: 6 Data size: 7 Basic stats: COMPLETE Column stats: NONE
    + File Output Operator
    + compressed: false
    + Statistics: Num rows: 6 Data size: 7 Basic stats: COMPLETE Column stats: NONE
    + table:
    + input format: org.apache.hadoop.mapred.TextInputFormat
    + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
    + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
    +
    + Stage: Stage-0
    + Fetch Operator
    + limit: -1
    + Processor Tree:
    + ListSink
    +
    +PREHOOK: query: SELECT * FROM predicate_fold_tb WHERE not(value IS NOT NULL AND value = 3)
    +PREHOOK: type: QUERY
    +PREHOOK: Input: default@predicate_fold_tb
    +#### A masked pattern was here ####
    +POSTHOOK: query: SELECT * FROM predicate_fold_tb WHERE not(value IS NOT NULL AND value = 3)
    +POSTHOOK: type: QUERY
    +POSTHOOK: Input: default@predicate_fold_tb
    +#### A masked pattern was here ####
    +NULL
    +1
    +2
    +4
    +5
    +PREHOOK: query: explain
    +SELECT * FROM predicate_fold_tb WHERE not(value IS NOT NULL AND value >= 3)
    +PREHOOK: type: QUERY
    +POSTHOOK: query: explain
    +SELECT * FROM predicate_fold_tb WHERE not(value IS NOT NULL AND value >= 3)
    +POSTHOOK: type: QUERY
    +STAGE DEPENDENCIES:
    + Stage-1 is a root stage
    + Stage-0 depends on stages: Stage-1
    +
    +STAGE PLANS:
    + Stage: Stage-1
    + Map Reduce
    + Map Operator Tree:
    + TableScan
    + alias: predicate_fold_tb
    + Statistics: Num rows: 6 Data size: 7 Basic stats: COMPLETE Column stats: NONE
    + Filter Operator
    + predicate: (value is null or (value < 3)) (type: boolean)
    + Statistics: Num rows: 5 Data size: 5 Basic stats: COMPLETE Column stats: NONE
    + Select Operator
    + expressions: value (type: int)
    + outputColumnNames: _col0
    + Statistics: Num rows: 5 Data size: 5 Basic stats: COMPLETE Column stats: NONE
    + File Output Operator
    + compressed: false
    + Statistics: Num rows: 5 Data size: 5 Basic stats: COMPLETE Column stats: NONE
    + table:
    + input format: org.apache.hadoop.mapred.TextInputFormat
    + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
    + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
    +
    + Stage: Stage-0
    + Fetch Operator
    + limit: -1
    + Processor Tree:
    + ListSink
    +
    +PREHOOK: query: SELECT * FROM predicate_fold_tb WHERE not(value IS NOT NULL AND value >= 3)
    +PREHOOK: type: QUERY
    +PREHOOK: Input: default@predicate_fold_tb
    +#### A masked pattern was here ####
    +POSTHOOK: query: SELECT * FROM predicate_fold_tb WHERE not(value IS NOT NULL AND value >= 3)
    +POSTHOOK: type: QUERY
    +POSTHOOK: Input: default@predicate_fold_tb
    +#### A masked pattern was here ####
    +NULL
    +1
    +2
    +PREHOOK: query: explain
    +SELECT * FROM predicate_fold_tb WHERE not(value IS NOT NULL AND value <= 3)
    +PREHOOK: type: QUERY
    +POSTHOOK: query: explain
    +SELECT * FROM predicate_fold_tb WHERE not(value IS NOT NULL AND value <= 3)
    +POSTHOOK: type: QUERY
    +STAGE DEPENDENCIES:
    + Stage-1 is a root stage
    + Stage-0 depends on stages: Stage-1
    +
    +STAGE PLANS:
    + Stage: Stage-1
    + Map Reduce
    + Map Operator Tree:
    + TableScan
    + alias: predicate_fold_tb
    + Statistics: Num rows: 6 Data size: 7 Basic stats: COMPLETE Column stats: NONE
    + Filter Operator
    + predicate: (value is null or (value > 3)) (type: boolean)
    + Statistics: Num rows: 5 Data size: 5 Basic stats: COMPLETE Column stats: NONE
    + Select Operator
    + expressions: value (type: int)
    + outputColumnNames: _col0
    + Statistics: Num rows: 5 Data size: 5 Basic stats: COMPLETE Column stats: NONE
    + File Output Operator
    + compressed: false
    + Statistics: Num rows: 5 Data size: 5 Basic stats: COMPLETE Column stats: NONE
    + table:
    + input format: org.apache.hadoop.mapred.TextInputFormat
    + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
    + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
    +
    + Stage: Stage-0
    + Fetch Operator
    + limit: -1
    + Processor Tree:
    + ListSink
    +
    +PREHOOK: query: SELECT * FROM predicate_fold_tb WHERE not(value IS NOT NULL AND value <= 3)
    +PREHOOK: type: QUERY
    +PREHOOK: Input: default@predicate_fold_tb
    +#### A masked pattern was here ####
    +POSTHOOK: query: SELECT * FROM predicate_fold_tb WHERE not(value IS NOT NULL AND value <= 3)
    +POSTHOOK: type: QUERY
    +POSTHOOK: Input: default@predicate_fold_tb
    +#### A masked pattern was here ####
    +NULL
    +4
    +5
    +PREHOOK: query: explain
    +SELECT * FROM predicate_fold_tb WHERE not(value IS NOT NULL AND value > 3)
    +PREHOOK: type: QUERY
    +POSTHOOK: query: explain
    +SELECT * FROM predicate_fold_tb WHERE not(value IS NOT NULL AND value > 3)
    +POSTHOOK: type: QUERY
    +STAGE DEPENDENCIES:
    + Stage-1 is a root stage
    + Stage-0 depends on stages: Stage-1
    +
    +STAGE PLANS:
    + Stage: Stage-1
    + Map Reduce
    + Map Operator Tree:
    + TableScan
    + alias: predicate_fold_tb
    + Statistics: Num rows: 6 Data size: 7 Basic stats: COMPLETE Column stats: NONE
    + Filter Operator
    + predicate: (value is null or (value <= 3)) (type: boolean)
    + Statistics: Num rows: 5 Data size: 5 Basic stats: COMPLETE Column stats: NONE
    + Select Operator
    + expressions: value (type: int)
    + outputColumnNames: _col0
    + Statistics: Num rows: 5 Data size: 5 Basic stats: COMPLETE Column stats: NONE
    + File Output Operator
    + compressed: false
    + Statistics: Num rows: 5 Data size: 5 Basic stats: COMPLETE Column stats: NONE
    + table:
    + input format: org.apache.hadoop.mapred.TextInputFormat
    + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
    + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
    +
    + Stage: Stage-0
    + Fetch Operator
    + limit: -1
    + Processor Tree:
    + ListSink
    +
    +PREHOOK: query: SELECT * FROM predicate_fold_tb WHERE not(value IS NOT NULL AND value > 3)
    +PREHOOK: type: QUERY
    +PREHOOK: Input: default@predicate_fold_tb
    +#### A masked pattern was here ####
    +POSTHOOK: query: SELECT * FROM predicate_fold_tb WHERE not(value IS NOT NULL AND value > 3)
    +POSTHOOK: type: QUERY
    +POSTHOOK: Input: default@predicate_fold_tb
    +#### A masked pattern was here ####
    +NULL
    +1
    +2
    +3
    +PREHOOK: query: explain
    +SELECT * FROM predicate_fold_tb WHERE not(value IS NOT NULL AND value < 3)
    +PREHOOK: type: QUERY
    +POSTHOOK: query: explain
    +SELECT * FROM predicate_fold_tb WHERE not(value IS NOT NULL AND value < 3)
    +POSTHOOK: type: QUERY
    +STAGE DEPENDENCIES:
    + Stage-1 is a root stage
    + Stage-0 depends on stages: Stage-1
    +
    +STAGE PLANS:
    + Stage: Stage-1
    + Map Reduce
    + Map Operator Tree:
    + TableScan
    + alias: predicate_fold_tb
    + Statistics: Num rows: 6 Data size: 7 Basic stats: COMPLETE Column stats: NONE
    + Filter Operator
    + predicate: (value is null or (value >= 3)) (type: boolean)
    + Statistics: Num rows: 5 Data size: 5 Basic stats: COMPLETE Column stats: NONE
    + Select Operator
    + expressions: value (type: int)
    + outputColumnNames: _col0
    + Statistics: Num rows: 5 Data size: 5 Basic stats: COMPLETE Column stats: NONE
    + File Output Operator
    + compressed: false
    + Statistics: Num rows: 5 Data size: 5 Basic stats: COMPLETE Column stats: NONE
    + table:
    + input format: org.apache.hadoop.mapred.TextInputFormat
    + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
    + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
    +
    + Stage: Stage-0
    + Fetch Operator
    + limit: -1
    + Processor Tree:
    + ListSink
    +
    +PREHOOK: query: SELECT * FROM predicate_fold_tb WHERE not(value IS NOT NULL AND value < 3)
    +PREHOOK: type: QUERY
    +PREHOOK: Input: default@predicate_fold_tb
    +#### A masked pattern was here ####
    +POSTHOOK: query: SELECT * FROM predicate_fold_tb WHERE not(value IS NOT NULL AND value < 3)
    +POSTHOOK: type: QUERY
    +POSTHOOK: Input: default@predicate_fold_tb
    +#### A masked pattern was here ####
    +NULL
    +3
    +4
    +5
    +PREHOOK: query: explain
    +SELECT * FROM predicate_fold_tb WHERE not(value IS NOT NULL AND value <> 3)
    +PREHOOK: type: QUERY
    +POSTHOOK: query: explain
    +SELECT * FROM predicate_fold_tb WHERE not(value IS NOT NULL AND value <> 3)
    +POSTHOOK: type: QUERY
    +STAGE DEPENDENCIES:
    + Stage-1 is a root stage
    + Stage-0 depends on stages: Stage-1
    +
    +STAGE PLANS:
    + Stage: Stage-1
    + Map Reduce
    + Map Operator Tree:
    + TableScan
    + alias: predicate_fold_tb
    + Statistics: Num rows: 6 Data size: 7 Basic stats: COMPLETE Column stats: NONE
    + Filter Operator
    + predicate: (value is null or (value = 3)) (type: boolean)
    + Statistics: Num rows: 6 Data size: 7 Basic stats: COMPLETE Column stats: NONE
    + Select Operator
    + expressions: value (type: int)
    + outputColumnNames: _col0
    + Statistics: Num rows: 6 Data size: 7 Basic stats: COMPLETE Column stats: NONE
    + File Output Operator
    + compressed: false
    + Statistics: Num rows: 6 Data size: 7 Basic stats: COMPLETE Column stats: NONE
    + table:
    + input format: org.apache.hadoop.mapred.TextInputFormat
    + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
    + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
    +
    + Stage: Stage-0
    + Fetch Operator
    + limit: -1
    + Processor Tree:
    + ListSink
    +
    +PREHOOK: query: SELECT * FROM predicate_fold_tb WHERE not(value IS NOT NULL AND value <> 3)
    +PREHOOK: type: QUERY
    +PREHOOK: Input: default@predicate_fold_tb
    +#### A masked pattern was here ####
    +POSTHOOK: query: SELECT * FROM predicate_fold_tb WHERE not(value IS NOT NULL AND value <> 3)
    +POSTHOOK: type: QUERY
    +POSTHOOK: Input: default@predicate_fold_tb
    +#### A masked pattern was here ####
    +NULL
    +3
    +PREHOOK: query: explain
    +SELECT * FROM predicate_fold_tb WHERE not(value IS NOT NULL AND value > 1 AND value <=3)
    +PREHOOK: type: QUERY
    +POSTHOOK: query: explain
    +SELECT * FROM predicate_fold_tb WHERE not(value IS NOT NULL AND value > 1 AND value <=3)
    +POSTHOOK: type: QUERY
    +STAGE DEPENDENCIES:
    + Stage-1 is a root stage
    + Stage-0 depends on stages: Stage-1
    +
    +STAGE PLANS:
    + Stage: Stage-1
    + Map Reduce
    + Map Operator Tree:
    + TableScan
    + alias: predicate_fold_tb
    + Statistics: Num rows: 6 Data size: 7 Basic stats: COMPLETE Column stats: NONE
    + Filter Operator
    + predicate: (value is null or (value <= 1) or (value > 3)) (type: boolean)
    + Statistics: Num rows: 6 Data size: 7 Basic stats: COMPLETE Column stats: NONE
    + Select Operator
    + expressions: value (type: int)
    + outputColumnNames: _col0
    + Statistics: Num rows: 6 Data size: 7 Basic stats: COMPLETE Column stats: NONE
    + File Output Operator
    + compressed: false
    + Statistics: Num rows: 6 Data size: 7 Basic stats: COMPLETE Column stats: NONE
    + table:
    + input format: org.apache.hadoop.mapred.TextInputFormat
    + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
    + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
    +
    + Stage: Stage-0
    + Fetch Operator
    + limit: -1
    + Processor Tree:
    + ListSink
    +
    +PREHOOK: query: SELECT * FROM predicate_fold_tb WHERE not(value IS NOT NULL AND value > 1 AND value <=3)
    +PREHOOK: type: QUERY
    +PREHOOK: Input: default@predicate_fold_tb
    +#### A masked pattern was here ####
    +POSTHOOK: query: SELECT * FROM predicate_fold_tb WHERE not(value IS NOT NULL AND value > 1 AND value <=3)
    +POSTHOOK: type: QUERY
    +POSTHOOK: Input: default@predicate_fold_tb
    +#### A masked pattern was here ####
    +NULL
    +1
    +4
    +5

    http://git-wip-us.apache.org/repos/asf/hive/blob/a1626749/ql/src/test/results/clientpositive/input_testxpath2.q.out
    ----------------------------------------------------------------------
    diff --git a/ql/src/test/results/clientpositive/input_testxpath2.q.out b/ql/src/test/results/clientpositive/input_testxpath2.q.out
    index d3a6f29..67e270f 100644
    --- a/ql/src/test/results/clientpositive/input_testxpath2.q.out
    +++ b/ql/src/test/results/clientpositive/input_testxpath2.q.out
    @@ -32,7 +32,7 @@ STAGE PLANS:
                  alias: src_thrift
                  Statistics: Num rows: 11 Data size: 3070 Basic stats: COMPLETE Column stats: NONE
                  Filter Operator
    - predicate: (lint is not null and (not mstringstring is null)) (type: boolean)
    + predicate: (lint is not null and mstringstring is not null) (type: boolean)
                    Statistics: Num rows: 3 Data size: 837 Basic stats: COMPLETE Column stats: NONE
                    Select Operator
                      expressions: size(lint) (type: int), size(lintstring) (type: int), size(mstringstring) (type: int)

    http://git-wip-us.apache.org/repos/asf/hive/blob/a1626749/ql/src/test/results/clientpositive/list_bucket_query_oneskew_3.q.out
    ----------------------------------------------------------------------
    diff --git a/ql/src/test/results/clientpositive/list_bucket_query_oneskew_3.q.out b/ql/src/test/results/clientpositive/list_bucket_query_oneskew_3.q.out
    index fabd0d6..bef079b 100644
    --- a/ql/src/test/results/clientpositive/list_bucket_query_oneskew_3.q.out
    +++ b/ql/src/test/results/clientpositive/list_bucket_query_oneskew_3.q.out
    @@ -293,12 +293,12 @@ STAGE PLANS:
                GatherStats: false
                Filter Operator
                  isSamplingPred: false
    - predicate: (not (x = 86)) (type: boolean)
    - Statistics: Num rows: 15 Data size: 60 Basic stats: COMPLETE Column stats: NONE
    + predicate: (x <> 86) (type: boolean)
    + Statistics: Num rows: 29 Data size: 117 Basic stats: COMPLETE Column stats: NONE
                  Select Operator
                    expressions: x (type: int)
                    outputColumnNames: _col0
    - Statistics: Num rows: 15 Data size: 60 Basic stats: COMPLETE Column stats: NONE
    + Statistics: Num rows: 29 Data size: 117 Basic stats: COMPLETE Column stats: NONE
                    ListSink

      PREHOOK: query: -- List Bucketing Query

    http://git-wip-us.apache.org/repos/asf/hive/blob/a1626749/ql/src/test/results/clientpositive/rand_partitionpruner3.q.out
    ----------------------------------------------------------------------
    diff --git a/ql/src/test/results/clientpositive/rand_partitionpruner3.q.out b/ql/src/test/results/clientpositive/rand_partitionpruner3.q.out
    index ef8ee2f..634e171 100644
    --- a/ql/src/test/results/clientpositive/rand_partitionpruner3.q.out
    +++ b/ql/src/test/results/clientpositive/rand_partitionpruner3.q.out
    @@ -115,12 +115,12 @@ STAGE PLANS:
                GatherStats: false
                Filter Operator
                  isSamplingPred: false
    - predicate: ((rand(1) < 0.1) and (not ((UDFToDouble(key) > 50.0) or (UDFToDouble(key) < 10.0)))) (type: boolean)
    - Statistics: Num rows: 56 Data size: 594 Basic stats: COMPLETE Column stats: NONE
    + predicate: ((rand(1) < 0.1) and ((UDFToDouble(key) <= 50.0) and (UDFToDouble(key) >= 10.0))) (type: boolean)
    + Statistics: Num rows: 18 Data size: 191 Basic stats: COMPLETE Column stats: NONE
               &n