FAQ
Repository: hive
Updated Branches:
   refs/heads/master ab1f405d1 -> b219752df


HIVE-12815: column stats NPE for a query w/o a table (Prasanth Jayachandran reviewed by Sergey Shelukhin)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/b219752d
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/b219752d
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/b219752d

Branch: refs/heads/master
Commit: b219752dfd12de862e9d9ce637e60720f3ab1b44
Parents: ab1f405
Author: Prasanth Jayachandran <j.prasanth.j@gmail.com>
Authored: Thu Jan 14 02:31:52 2016 +0530
Committer: Prasanth Jayachandran <j.prasanth.j@gmail.com>
Committed: Thu Jan 14 02:32:22 2016 +0530

----------------------------------------------------------------------
  .../apache/hadoop/hive/ql/stats/StatsUtils.java | 2 +-
  .../clientpositive/annotate_stats_table.q | 3 +
  .../clientpositive/annotate_stats_table.q.out | 99 ++++++++++++++++++++
  3 files changed, 103 insertions(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/b219752d/ql/src/java/org/apache/hadoop/hive/ql/stats/StatsUtils.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/stats/StatsUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/stats/StatsUtils.java
index 2f78fe8..b4cf58f 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/stats/StatsUtils.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/stats/StatsUtils.java
@@ -1443,7 +1443,7 @@ public class StatsUtils {
    public static long getDataSizeFromColumnStats(long numRows, List<ColStatistics> colStats) {
      long result = 0;

- if (numRows <= 0) {
+ if (numRows <= 0 || colStats == null || colStats.isEmpty()) {
        return result;
      }


http://git-wip-us.apache.org/repos/asf/hive/blob/b219752d/ql/src/test/queries/clientpositive/annotate_stats_table.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/annotate_stats_table.q b/ql/src/test/queries/clientpositive/annotate_stats_table.q
index ea1999f..b5389ff 100644
--- a/ql/src/test/queries/clientpositive/annotate_stats_table.q
+++ b/ql/src/test/queries/clientpositive/annotate_stats_table.q
@@ -51,3 +51,6 @@ explain select deptid from emp_orc;

  -- basicStatState: COMPLETE colStatState: COMPLETE
  explain select lastname,deptid from emp_orc;
+
+create table tmp as select 1;
+explain create table tmp as select 1;

http://git-wip-us.apache.org/repos/asf/hive/blob/b219752d/ql/src/test/results/clientpositive/annotate_stats_table.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/annotate_stats_table.q.out b/ql/src/test/results/clientpositive/annotate_stats_table.q.out
index 0f80755..cec3a1f 100644
--- a/ql/src/test/results/clientpositive/annotate_stats_table.q.out
+++ b/ql/src/test/results/clientpositive/annotate_stats_table.q.out
@@ -289,3 +289,102 @@ STAGE PLANS:
              Statistics: Num rows: 48 Data size: 4560 Basic stats: COMPLETE Column stats: COMPLETE
              ListSink

+PREHOOK: query: create table tmp as select 1
+PREHOOK: type: CREATETABLE_AS_SELECT
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: database:default
+PREHOOK: Output: default@tmp
+POSTHOOK: query: create table tmp as select 1
+POSTHOOK: type: CREATETABLE_AS_SELECT
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@tmp
+PREHOOK: query: explain create table tmp as select 1
+PREHOOK: type: CREATETABLE_AS_SELECT
+POSTHOOK: query: explain create table tmp as select 1
+POSTHOOK: type: CREATETABLE_AS_SELECT
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-6 depends on stages: Stage-1 , consists of Stage-3, Stage-2, Stage-4
+ Stage-3
+ Stage-0 depends on stages: Stage-3, Stage-2, Stage-5
+ Stage-7 depends on stages: Stage-0
+ Stage-2
+ Stage-4
+ Stage-5 depends on stages: Stage-4
+
+STAGE PLANS:
+ Stage: Stage-1
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ alias: _dummy_table
+ Row Limit Per Split: 1
+ Statistics: Num rows: 1 Data size: 1 Basic stats: COMPLETE Column stats: COMPLETE
+ Select Operator
+ expressions: 1 (type: int)
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.tmp
+
+ Stage: Stage-6
+ Conditional Operator
+
+ Stage: Stage-3
+ Move Operator
+ files:
+ hdfs directory: true
+#### A masked pattern was here ####
+
+ Stage: Stage-0
+ Move Operator
+ files:
+ hdfs directory: true
+#### A masked pattern was here ####
+
+ Stage: Stage-7
+ Create Table Operator:
+ Create Table
+ columns: _c0 int
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat
+ serde name: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.tmp
+
+ Stage: Stage-2
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ File Output Operator
+ compressed: false
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.tmp
+
+ Stage: Stage-4
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ File Output Operator
+ compressed: false
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.tmp
+
+ Stage: Stage-5
+ Move Operator
+ files:
+ hdfs directory: true
+#### A masked pattern was here ####
+

Search Discussions

  • Prasanthj at Jan 13, 2016 at 9:04 pm
    Repository: hive
    Updated Branches:
       refs/heads/branch-1 976938518 -> 67be6771b


    HIVE-12815: column stats NPE for a query w/o a table (Prasanth Jayachandran reviewed by Sergey Shelukhin)


    Project: http://git-wip-us.apache.org/repos/asf/hive/repo
    Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/67be6771
    Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/67be6771
    Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/67be6771

    Branch: refs/heads/branch-1
    Commit: 67be6771bb1f7ba9a6669a240e1db255d0fa5eda
    Parents: 9769385
    Author: Prasanth Jayachandran <j.prasanth.j@gmail.com>
    Authored: Thu Jan 14 02:34:19 2016 +0530
    Committer: Prasanth Jayachandran <j.prasanth.j@gmail.com>
    Committed: Thu Jan 14 02:34:19 2016 +0530

    ----------------------------------------------------------------------
      .../apache/hadoop/hive/ql/stats/StatsUtils.java | 2 +-
      .../clientpositive/annotate_stats_table.q | 3 +
      .../clientpositive/annotate_stats_table.q.out | 99 ++++++++++++++++++++
      3 files changed, 103 insertions(+), 1 deletion(-)
    ----------------------------------------------------------------------


    http://git-wip-us.apache.org/repos/asf/hive/blob/67be6771/ql/src/java/org/apache/hadoop/hive/ql/stats/StatsUtils.java
    ----------------------------------------------------------------------
    diff --git a/ql/src/java/org/apache/hadoop/hive/ql/stats/StatsUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/stats/StatsUtils.java
    index 1e4c85e..a652c1e 100644
    --- a/ql/src/java/org/apache/hadoop/hive/ql/stats/StatsUtils.java
    +++ b/ql/src/java/org/apache/hadoop/hive/ql/stats/StatsUtils.java
    @@ -1406,7 +1406,7 @@ public class StatsUtils {
        public static long getDataSizeFromColumnStats(long numRows, List<ColStatistics> colStats) {
          long result = 0;

    - if (numRows <= 0) {
    + if (numRows <= 0 || colStats == null || colStats.isEmpty()) {
            return result;
          }


    http://git-wip-us.apache.org/repos/asf/hive/blob/67be6771/ql/src/test/queries/clientpositive/annotate_stats_table.q
    ----------------------------------------------------------------------
    diff --git a/ql/src/test/queries/clientpositive/annotate_stats_table.q b/ql/src/test/queries/clientpositive/annotate_stats_table.q
    index ea1999f..b5389ff 100644
    --- a/ql/src/test/queries/clientpositive/annotate_stats_table.q
    +++ b/ql/src/test/queries/clientpositive/annotate_stats_table.q
    @@ -51,3 +51,6 @@ explain select deptid from emp_orc;

      -- basicStatState: COMPLETE colStatState: COMPLETE
      explain select lastname,deptid from emp_orc;
    +
    +create table tmp as select 1;
    +explain create table tmp as select 1;

    http://git-wip-us.apache.org/repos/asf/hive/blob/67be6771/ql/src/test/results/clientpositive/annotate_stats_table.q.out
    ----------------------------------------------------------------------
    diff --git a/ql/src/test/results/clientpositive/annotate_stats_table.q.out b/ql/src/test/results/clientpositive/annotate_stats_table.q.out
    index 87e2fa6..b3c8fca 100644
    --- a/ql/src/test/results/clientpositive/annotate_stats_table.q.out
    +++ b/ql/src/test/results/clientpositive/annotate_stats_table.q.out
    @@ -289,3 +289,102 @@ STAGE PLANS:
                  Statistics: Num rows: 48 Data size: 4560 Basic stats: COMPLETE Column stats: COMPLETE
                  ListSink

    +PREHOOK: query: create table tmp as select 1
    +PREHOOK: type: CREATETABLE_AS_SELECT
    +PREHOOK: Input: _dummy_database@_dummy_table
    +PREHOOK: Output: database:default
    +PREHOOK: Output: default@tmp
    +POSTHOOK: query: create table tmp as select 1
    +POSTHOOK: type: CREATETABLE_AS_SELECT
    +POSTHOOK: Input: _dummy_database@_dummy_table
    +POSTHOOK: Output: database:default
    +POSTHOOK: Output: default@tmp
    +PREHOOK: query: explain create table tmp as select 1
    +PREHOOK: type: CREATETABLE_AS_SELECT
    +POSTHOOK: query: explain create table tmp as select 1
    +POSTHOOK: type: CREATETABLE_AS_SELECT
    +STAGE DEPENDENCIES:
    + Stage-1 is a root stage
    + Stage-6 depends on stages: Stage-1 , consists of Stage-3, Stage-2, Stage-4
    + Stage-3
    + Stage-0 depends on stages: Stage-3, Stage-2, Stage-5
    + Stage-7 depends on stages: Stage-0
    + Stage-2
    + Stage-4
    + Stage-5 depends on stages: Stage-4
    +
    +STAGE PLANS:
    + Stage: Stage-1
    + Map Reduce
    + Map Operator Tree:
    + TableScan
    + alias: _dummy_table
    + Row Limit Per Split: 1
    + Statistics: Num rows: 1 Data size: 1 Basic stats: COMPLETE Column stats: COMPLETE
    + Select Operator
    + expressions: 1 (type: int)
    + outputColumnNames: _col0
    + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE
    + File Output Operator
    + compressed: false
    + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE
    + table:
    + input format: org.apache.hadoop.mapred.TextInputFormat
    + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
    + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
    + name: default.tmp
    +
    + Stage: Stage-6
    + Conditional Operator
    +
    + Stage: Stage-3
    + Move Operator
    + files:
    + hdfs directory: true
    +#### A masked pattern was here ####
    +
    + Stage: Stage-0
    + Move Operator
    + files:
    + hdfs directory: true
    +#### A masked pattern was here ####
    +
    + Stage: Stage-7
    + Create Table Operator:
    + Create Table
    + columns: _c0 int
    + input format: org.apache.hadoop.mapred.TextInputFormat
    + output format: org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat
    + serde name: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
    + name: default.tmp
    +
    + Stage: Stage-2
    + Map Reduce
    + Map Operator Tree:
    + TableScan
    + File Output Operator
    + compressed: false
    + table:
    + input format: org.apache.hadoop.mapred.TextInputFormat
    + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
    + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
    + name: default.tmp
    +
    + Stage: Stage-4
    + Map Reduce
    + Map Operator Tree:
    + TableScan
    + File Output Operator
    + compressed: false
    + table:
    + input format: org.apache.hadoop.mapred.TextInputFormat
    + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
    + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
    + name: default.tmp
    +
    + Stage: Stage-5
    + Move Operator
    + files:
    + hdfs directory: true
    +#### A masked pattern was here ####
    +
  • Prasanthj at Jan 13, 2016 at 9:09 pm
    Repository: hive
    Updated Branches:
       refs/heads/branch-2.0 4a3a6b34d -> 295ad94f9


    HIVE-12815: column stats NPE for a query w/o a table (Prasanth Jayachandran reviewed by Sergey Shelukhin)


    Project: http://git-wip-us.apache.org/repos/asf/hive/repo
    Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/295ad94f
    Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/295ad94f
    Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/295ad94f

    Branch: refs/heads/branch-2.0
    Commit: 295ad94f9719b64d2820eca42f3d3850f36f1bd8
    Parents: 4a3a6b3
    Author: Prasanth Jayachandran <j.prasanth.j@gmail.com>
    Authored: Thu Jan 14 02:38:41 2016 +0530
    Committer: Prasanth Jayachandran <j.prasanth.j@gmail.com>
    Committed: Thu Jan 14 02:38:41 2016 +0530

    ----------------------------------------------------------------------
      .../apache/hadoop/hive/ql/stats/StatsUtils.java | 2 +-
      .../clientpositive/annotate_stats_table.q | 3 +
      .../clientpositive/annotate_stats_table.q.out | 99 ++++++++++++++++++++
      3 files changed, 103 insertions(+), 1 deletion(-)
    ----------------------------------------------------------------------


    http://git-wip-us.apache.org/repos/asf/hive/blob/295ad94f/ql/src/java/org/apache/hadoop/hive/ql/stats/StatsUtils.java
    ----------------------------------------------------------------------
    diff --git a/ql/src/java/org/apache/hadoop/hive/ql/stats/StatsUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/stats/StatsUtils.java
    index 2f78fe8..b4cf58f 100644
    --- a/ql/src/java/org/apache/hadoop/hive/ql/stats/StatsUtils.java
    +++ b/ql/src/java/org/apache/hadoop/hive/ql/stats/StatsUtils.java
    @@ -1443,7 +1443,7 @@ public class StatsUtils {
        public static long getDataSizeFromColumnStats(long numRows, List<ColStatistics> colStats) {
          long result = 0;

    - if (numRows <= 0) {
    + if (numRows <= 0 || colStats == null || colStats.isEmpty()) {
            return result;
          }


    http://git-wip-us.apache.org/repos/asf/hive/blob/295ad94f/ql/src/test/queries/clientpositive/annotate_stats_table.q
    ----------------------------------------------------------------------
    diff --git a/ql/src/test/queries/clientpositive/annotate_stats_table.q b/ql/src/test/queries/clientpositive/annotate_stats_table.q
    index ea1999f..b5389ff 100644
    --- a/ql/src/test/queries/clientpositive/annotate_stats_table.q
    +++ b/ql/src/test/queries/clientpositive/annotate_stats_table.q
    @@ -51,3 +51,6 @@ explain select deptid from emp_orc;

      -- basicStatState: COMPLETE colStatState: COMPLETE
      explain select lastname,deptid from emp_orc;
    +
    +create table tmp as select 1;
    +explain create table tmp as select 1;

    http://git-wip-us.apache.org/repos/asf/hive/blob/295ad94f/ql/src/test/results/clientpositive/annotate_stats_table.q.out
    ----------------------------------------------------------------------
    diff --git a/ql/src/test/results/clientpositive/annotate_stats_table.q.out b/ql/src/test/results/clientpositive/annotate_stats_table.q.out
    index 0f80755..cec3a1f 100644
    --- a/ql/src/test/results/clientpositive/annotate_stats_table.q.out
    +++ b/ql/src/test/results/clientpositive/annotate_stats_table.q.out
    @@ -289,3 +289,102 @@ STAGE PLANS:
                  Statistics: Num rows: 48 Data size: 4560 Basic stats: COMPLETE Column stats: COMPLETE
                  ListSink

    +PREHOOK: query: create table tmp as select 1
    +PREHOOK: type: CREATETABLE_AS_SELECT
    +PREHOOK: Input: _dummy_database@_dummy_table
    +PREHOOK: Output: database:default
    +PREHOOK: Output: default@tmp
    +POSTHOOK: query: create table tmp as select 1
    +POSTHOOK: type: CREATETABLE_AS_SELECT
    +POSTHOOK: Input: _dummy_database@_dummy_table
    +POSTHOOK: Output: database:default
    +POSTHOOK: Output: default@tmp
    +PREHOOK: query: explain create table tmp as select 1
    +PREHOOK: type: CREATETABLE_AS_SELECT
    +POSTHOOK: query: explain create table tmp as select 1
    +POSTHOOK: type: CREATETABLE_AS_SELECT
    +STAGE DEPENDENCIES:
    + Stage-1 is a root stage
    + Stage-6 depends on stages: Stage-1 , consists of Stage-3, Stage-2, Stage-4
    + Stage-3
    + Stage-0 depends on stages: Stage-3, Stage-2, Stage-5
    + Stage-7 depends on stages: Stage-0
    + Stage-2
    + Stage-4
    + Stage-5 depends on stages: Stage-4
    +
    +STAGE PLANS:
    + Stage: Stage-1
    + Map Reduce
    + Map Operator Tree:
    + TableScan
    + alias: _dummy_table
    + Row Limit Per Split: 1
    + Statistics: Num rows: 1 Data size: 1 Basic stats: COMPLETE Column stats: COMPLETE
    + Select Operator
    + expressions: 1 (type: int)
    + outputColumnNames: _col0
    + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE
    + File Output Operator
    + compressed: false
    + Statistics: Num rows: 1 Data size: 4 Basic stats: COMPLETE Column stats: COMPLETE
    + table:
    + input format: org.apache.hadoop.mapred.TextInputFormat
    + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
    + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
    + name: default.tmp
    +
    + Stage: Stage-6
    + Conditional Operator
    +
    + Stage: Stage-3
    + Move Operator
    + files:
    + hdfs directory: true
    +#### A masked pattern was here ####
    +
    + Stage: Stage-0
    + Move Operator
    + files:
    + hdfs directory: true
    +#### A masked pattern was here ####
    +
    + Stage: Stage-7
    + Create Table Operator:
    + Create Table
    + columns: _c0 int
    + input format: org.apache.hadoop.mapred.TextInputFormat
    + output format: org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat
    + serde name: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
    + name: default.tmp
    +
    + Stage: Stage-2
    + Map Reduce
    + Map Operator Tree:
    + TableScan
    + File Output Operator
    + compressed: false
    + table:
    + input format: org.apache.hadoop.mapred.TextInputFormat
    + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
    + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
    + name: default.tmp
    +
    + Stage: Stage-4
    + Map Reduce
    + Map Operator Tree:
    + TableScan
    + File Output Operator
    + compressed: false
    + table:
    + input format: org.apache.hadoop.mapred.TextInputFormat
    + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
    + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
    + name: default.tmp
    +
    + Stage: Stage-5
    + Move Operator
    + files:
    + hdfs directory: true
    +#### A masked pattern was here ####
    +

Related Discussions

Discussion Navigation
viewthread | post
Discussion Overview
groupcommits @
categorieshive, hadoop
postedJan 13, '16 at 9:03p
activeJan 13, '16 at 9:09p
posts3
users1
websitehive.apache.org

1 user in discussion

Prasanthj: 3 posts

People

Translate

site design / logo © 2021 Grokbase