FAQ
Repository: hive
Updated Branches:
   refs/heads/master 40f348f6b -> 5fff8231e


HIVE-10965 : direct SQL for stats fails in 0-column case (Sergey Shelukhin reviewed by Ashutosh Chauhan)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/5fff8231
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/5fff8231
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/5fff8231

Branch: refs/heads/master
Commit: 5fff8231ec88e3bff9db22541bcbf9dcc1c68361
Parents: 40f348f
Author: Thejas Nair <thejas@hortonworks.com>
Authored: Wed Jun 10 15:10:57 2015 -0700
Committer: Thejas Nair <thejas@hortonworks.com>
Committed: Wed Jun 10 15:10:57 2015 -0700

----------------------------------------------------------------------
  .../org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java | 1 +
  .../java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java | 2 ++
  ql/src/java/org/apache/hadoop/hive/ql/stats/StatsUtils.java | 3 ++-
  3 files changed, 5 insertions(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/5fff8231/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java
index f5816a0..4891d42 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java
@@ -2060,6 +2060,7 @@ public class HiveMetaStoreClient implements IMetaStoreClient {
    @Override
    public AggrStats getAggrColStatsFor(String dbName, String tblName,
      List<String> colNames, List<String> partNames) throws NoSuchObjectException, MetaException, TException {
+ if (colNames.isEmpty()) return null; // Nothing to aggregate.
      PartitionsStatsRequest req = new PartitionsStatsRequest(dbName, tblName, colNames, partNames);
      return client.get_aggr_stats_for(req);
    }

http://git-wip-us.apache.org/repos/asf/hive/blob/5fff8231/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java b/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java
index 0a56bac..ed810d2 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java
@@ -1100,6 +1100,7 @@ class MetaStoreDirectSql {
    public AggrStats aggrColStatsForPartitions(String dbName, String tableName,
        List<String> partNames, List<String> colNames, boolean useDensityFunctionForNDVEstimation)
        throws MetaException {
+ if (colNames.isEmpty() || partNames.isEmpty()) return new AggrStats(); // Nothing to aggregate.
      long partsFound = partsFoundForPartitions(dbName, tableName, partNames, colNames);
      List<ColumnStatisticsObj> colStatsList;
      // Try to read from the cache first
@@ -1160,6 +1161,7 @@ class MetaStoreDirectSql {

    private long partsFoundForPartitions(String dbName, String tableName,
        List<String> partNames, List<String> colNames) throws MetaException {
+ assert !colNames.isEmpty() && !partNames.isEmpty();
      long partsFound = 0;
      boolean doTrace = LOG.isDebugEnabled();
      String queryText = "select count(\"COLUMN_NAME\") from \"PART_COL_STATS\""

http://git-wip-us.apache.org/repos/asf/hive/blob/5fff8231/ql/src/java/org/apache/hadoop/hive/ql/stats/StatsUtils.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/stats/StatsUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/stats/StatsUtils.java
index 6615a28..0bd7f0a 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/stats/StatsUtils.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/stats/StatsUtils.java
@@ -250,7 +250,8 @@ public class StatsUtils {
          neededColumns = processNeededColumns(schema, neededColumns);
          AggrStats aggrStats = Hive.get().getAggrColStatsFor(table.getDbName(), table.getTableName(),
              neededColumns, partNames);
- if (null == aggrStats) {
+ if (null == aggrStats || null == aggrStats.getColStats()
+ || aggrStats.getColStatsSize() == 0) {
            // There are some partitions with no state (or we didn't fetch any state).
            // Update the stats with empty list to reflect that in the
            // state/initialize structures.

Search Discussions

  • Thejas at Jun 10, 2015 at 10:11 pm
    Repository: hive
    Updated Branches:
       refs/heads/branch-1 82244ce67 -> 718500eab


    HIVE-10965 : direct SQL for stats fails in 0-column case (Sergey Shelukhin reviewed by Ashutosh Chauhan)


    Project: http://git-wip-us.apache.org/repos/asf/hive/repo
    Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/718500ea
    Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/718500ea
    Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/718500ea

    Branch: refs/heads/branch-1
    Commit: 718500eabee43d971d40989c38460ba0b74f5461
    Parents: 82244ce
    Author: Thejas Nair <thejas@hortonworks.com>
    Authored: Wed Jun 10 15:10:57 2015 -0700
    Committer: Thejas Nair <thejas@hortonworks.com>
    Committed: Wed Jun 10 15:11:31 2015 -0700

    ----------------------------------------------------------------------
      .../org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java | 1 +
      .../java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java | 2 ++
      ql/src/java/org/apache/hadoop/hive/ql/stats/StatsUtils.java | 3 ++-
      3 files changed, 5 insertions(+), 1 deletion(-)
    ----------------------------------------------------------------------


    http://git-wip-us.apache.org/repos/asf/hive/blob/718500ea/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java
    ----------------------------------------------------------------------
    diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java
    index f5816a0..4891d42 100644
    --- a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java
    +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java
    @@ -2060,6 +2060,7 @@ public class HiveMetaStoreClient implements IMetaStoreClient {
        @Override
        public AggrStats getAggrColStatsFor(String dbName, String tblName,
          List<String> colNames, List<String> partNames) throws NoSuchObjectException, MetaException, TException {
    + if (colNames.isEmpty()) return null; // Nothing to aggregate.
          PartitionsStatsRequest req = new PartitionsStatsRequest(dbName, tblName, colNames, partNames);
          return client.get_aggr_stats_for(req);
        }

    http://git-wip-us.apache.org/repos/asf/hive/blob/718500ea/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java
    ----------------------------------------------------------------------
    diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java b/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java
    index 0a56bac..ed810d2 100644
    --- a/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java
    +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java
    @@ -1100,6 +1100,7 @@ class MetaStoreDirectSql {
        public AggrStats aggrColStatsForPartitions(String dbName, String tableName,
            List<String> partNames, List<String> colNames, boolean useDensityFunctionForNDVEstimation)
            throws MetaException {
    + if (colNames.isEmpty() || partNames.isEmpty()) return new AggrStats(); // Nothing to aggregate.
          long partsFound = partsFoundForPartitions(dbName, tableName, partNames, colNames);
          List<ColumnStatisticsObj> colStatsList;
          // Try to read from the cache first
    @@ -1160,6 +1161,7 @@ class MetaStoreDirectSql {

        private long partsFoundForPartitions(String dbName, String tableName,
            List<String> partNames, List<String> colNames) throws MetaException {
    + assert !colNames.isEmpty() && !partNames.isEmpty();
          long partsFound = 0;
          boolean doTrace = LOG.isDebugEnabled();
          String queryText = "select count(\"COLUMN_NAME\") from \"PART_COL_STATS\""

    http://git-wip-us.apache.org/repos/asf/hive/blob/718500ea/ql/src/java/org/apache/hadoop/hive/ql/stats/StatsUtils.java
    ----------------------------------------------------------------------
    diff --git a/ql/src/java/org/apache/hadoop/hive/ql/stats/StatsUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/stats/StatsUtils.java
    index 6615a28..0bd7f0a 100644
    --- a/ql/src/java/org/apache/hadoop/hive/ql/stats/StatsUtils.java
    +++ b/ql/src/java/org/apache/hadoop/hive/ql/stats/StatsUtils.java
    @@ -250,7 +250,8 @@ public class StatsUtils {
              neededColumns = processNeededColumns(schema, neededColumns);
              AggrStats aggrStats = Hive.get().getAggrColStatsFor(table.getDbName(), table.getTableName(),
                  neededColumns, partNames);
    - if (null == aggrStats) {
    + if (null == aggrStats || null == aggrStats.getColStats()
    + || aggrStats.getColStatsSize() == 0) {
                // There are some partitions with no state (or we didn't fetch any state).
                // Update the stats with empty list to reflect that in the
                // state/initialize structures.
  • Thejas at Jun 10, 2015 at 10:11 pm
    Repository: hive
    Updated Branches:
       refs/heads/branch-1.2 e87569f49 -> d35a56f6b


    HIVE-10965 : direct SQL for stats fails in 0-column case (Sergey Shelukhin reviewed by Ashutosh Chauhan)


    Project: http://git-wip-us.apache.org/repos/asf/hive/repo
    Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/d35a56f6
    Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/d35a56f6
    Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/d35a56f6

    Branch: refs/heads/branch-1.2
    Commit: d35a56f6b872da67f9a62ad4cc68264c4e8aaf4b
    Parents: e87569f
    Author: Thejas Nair <thejas@hortonworks.com>
    Authored: Wed Jun 10 15:10:57 2015 -0700
    Committer: Thejas Nair <thejas@hortonworks.com>
    Committed: Wed Jun 10 15:11:50 2015 -0700

    ----------------------------------------------------------------------
      .../org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java | 1 +
      .../java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java | 2 ++
      ql/src/java/org/apache/hadoop/hive/ql/stats/StatsUtils.java | 3 ++-
      3 files changed, 5 insertions(+), 1 deletion(-)
    ----------------------------------------------------------------------


    http://git-wip-us.apache.org/repos/asf/hive/blob/d35a56f6/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java
    ----------------------------------------------------------------------
    diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java
    index de577ab..41b4ea0 100644
    --- a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java
    +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java
    @@ -2059,6 +2059,7 @@ public class HiveMetaStoreClient implements IMetaStoreClient {
        @Override
        public AggrStats getAggrColStatsFor(String dbName, String tblName,
          List<String> colNames, List<String> partNames) throws NoSuchObjectException, MetaException, TException {
    + if (colNames.isEmpty()) return null; // Nothing to aggregate.
          PartitionsStatsRequest req = new PartitionsStatsRequest(dbName, tblName, colNames, partNames);
          return client.get_aggr_stats_for(req);
        }

    http://git-wip-us.apache.org/repos/asf/hive/blob/d35a56f6/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java
    ----------------------------------------------------------------------
    diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java b/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java
    index 0a56bac..ed810d2 100644
    --- a/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java
    +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java
    @@ -1100,6 +1100,7 @@ class MetaStoreDirectSql {
        public AggrStats aggrColStatsForPartitions(String dbName, String tableName,
            List<String> partNames, List<String> colNames, boolean useDensityFunctionForNDVEstimation)
            throws MetaException {
    + if (colNames.isEmpty() || partNames.isEmpty()) return new AggrStats(); // Nothing to aggregate.
          long partsFound = partsFoundForPartitions(dbName, tableName, partNames, colNames);
          List<ColumnStatisticsObj> colStatsList;
          // Try to read from the cache first
    @@ -1160,6 +1161,7 @@ class MetaStoreDirectSql {

        private long partsFoundForPartitions(String dbName, String tableName,
            List<String> partNames, List<String> colNames) throws MetaException {
    + assert !colNames.isEmpty() && !partNames.isEmpty();
          long partsFound = 0;
          boolean doTrace = LOG.isDebugEnabled();
          String queryText = "select count(\"COLUMN_NAME\") from \"PART_COL_STATS\""

    http://git-wip-us.apache.org/repos/asf/hive/blob/d35a56f6/ql/src/java/org/apache/hadoop/hive/ql/stats/StatsUtils.java
    ----------------------------------------------------------------------
    diff --git a/ql/src/java/org/apache/hadoop/hive/ql/stats/StatsUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/stats/StatsUtils.java
    index 6615a28..0bd7f0a 100644
    --- a/ql/src/java/org/apache/hadoop/hive/ql/stats/StatsUtils.java
    +++ b/ql/src/java/org/apache/hadoop/hive/ql/stats/StatsUtils.java
    @@ -250,7 +250,8 @@ public class StatsUtils {
              neededColumns = processNeededColumns(schema, neededColumns);
              AggrStats aggrStats = Hive.get().getAggrColStatsFor(table.getDbName(), table.getTableName(),
                  neededColumns, partNames);
    - if (null == aggrStats) {
    + if (null == aggrStats || null == aggrStats.getColStats()
    + || aggrStats.getColStatsSize() == 0) {
                // There are some partitions with no state (or we didn't fetch any state).
                // Update the stats with empty list to reflect that in the
                // state/initialize structures.

Related Discussions

Discussion Navigation
viewthread | post
Discussion Overview
groupcommits @
categorieshive, hadoop
postedJun 10, '15 at 10:11p
activeJun 10, '15 at 10:11p
posts3
users1
websitehive.apache.org

1 user in discussion

Thejas: 3 posts

People

Translate

site design / logo © 2021 Grokbase