FAQ
Repository: hive
Updated Branches:
   refs/heads/master f87b2b637 -> d5b1adbb6


HIVE-13243: Hive drop table on encyption zone fails for external tables (Chaoyu Tang, reviewed by Sergio Pena)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/d5b1adbb
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/d5b1adbb
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/d5b1adbb

Branch: refs/heads/master
Commit: d5b1adbb6638acd74fd4162a6ed408ebd72abeae
Parents: f87b2b6
Author: ctang <ctang@cloudera.com>
Authored: Tue Mar 15 11:41:41 2016 -0400
Committer: ctang <ctang@cloudera.com>
Committed: Tue Mar 15 11:41:41 2016 -0400

----------------------------------------------------------------------
  .../hadoop/hive/metastore/HiveMetaStore.java | 34 ++++++---
  .../clientpositive/encryption_drop_partition.q | 10 +++
  .../clientpositive/encryption_drop_table.q | 9 ++-
  .../encrypted/encryption_drop_partition.q.out | 76 ++++++++++++++++++++
  .../encrypted/encryption_drop_table.q.out | 27 +++++++
  5 files changed, 144 insertions(+), 12 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/d5b1adbb/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
index f0bc560..2fa0e9a 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
@@ -1508,8 +1508,7 @@ public class HiveMetaStore extends ThriftHiveMetastore {
            }
          }

- // tblPath will be null when tbl is a view. We skip the following if block in that case.
- checkTrashPurgeCombination(tblPath, dbname + "." + name, ifPurge);
+ checkTrashPurgeCombination(tblPath, dbname + "." + name, ifPurge, deleteData && !isExternal);
          // Drop the partitions and get a list of locations which need to be deleted
          partPaths = dropPartitionsAndGetLocations(ms, dbname, name, tblPath,
              tbl.getPartitionKeys(), deleteData && !isExternal);
@@ -1546,15 +1545,20 @@ public class HiveMetaStore extends ThriftHiveMetastore {
       * @param objectName db.table, or db.table.part
       * @param ifPurge if PURGE options is specified
       */
- private void checkTrashPurgeCombination(Path pathToData, String objectName, boolean ifPurge)
- throws MetaException {
- if (!(pathToData != null && !ifPurge)) {//pathToData may be NULL for a view
+ private void checkTrashPurgeCombination(Path pathToData, String objectName, boolean ifPurge,
+ boolean deleteData) throws MetaException {
+ // There is no need to check TrashPurgeCombination in following cases since Purge/Trash
+ // is not applicable:
+ // a) deleteData is false -- drop an external table
+ // b) pathToData is null -- a view
+ // c) ifPurge is true -- force delete without Trash
+ if (!deleteData || pathToData == null || ifPurge) {
          return;
        }

        boolean trashEnabled = false;
        try {
- trashEnabled = 0 < hiveConf.getFloat("fs.trash.interval", -1);
+ trashEnabled = 0 < hiveConf.getFloat("fs.trash.interval", -1);
        } catch(NumberFormatException ex) {
    // nothing to do
        }
@@ -2644,11 +2648,13 @@ public class HiveMetaStore extends ThriftHiveMetastore {
        boolean isArchived = false;
        Path archiveParentDir = null;
        boolean mustPurge = false;
+ boolean isExternalTbl = false;

        try {
          ms.openTransaction();
          part = ms.getPartition(db_name, tbl_name, part_vals);
          tbl = get_table_core(db_name, tbl_name);
+ isExternalTbl = isExternal(tbl);
          firePreEvent(new PreDropPartitionEvent(tbl, part, deleteData, this));
          mustPurge = isMustPurge(envContext, tbl);

@@ -2661,7 +2667,8 @@ public class HiveMetaStore extends ThriftHiveMetastore {
          if (isArchived) {
            archiveParentDir = MetaStoreUtils.getOriginalLocation(part);
            verifyIsWritablePath(archiveParentDir);
- checkTrashPurgeCombination(archiveParentDir, db_name + "." + tbl_name + "." + part_vals, mustPurge);
+ checkTrashPurgeCombination(archiveParentDir, db_name + "." + tbl_name + "." + part_vals,
+ mustPurge, deleteData && !isExternalTbl);
          }
          if (!ms.dropPartition(db_name, tbl_name, part_vals)) {
            throw new MetaException("Unable to drop partition");
@@ -2670,13 +2677,14 @@ public class HiveMetaStore extends ThriftHiveMetastore {
          if ((part.getSd() != null) && (part.getSd().getLocation() != null)) {
            partPath = new Path(part.getSd().getLocation());
            verifyIsWritablePath(partPath);
- checkTrashPurgeCombination(partPath, db_name + "." + tbl_name + "." + part_vals, mustPurge);
+ checkTrashPurgeCombination(partPath, db_name + "." + tbl_name + "." + part_vals,
+ mustPurge, deleteData && !isExternalTbl);
          }
        } finally {
          if (!success) {
            ms.rollbackTransaction();
          } else if (deleteData && ((partPath != null) || (archiveParentDir != null))) {
- if (tbl != null && !isExternal(tbl)) {
+ if (!isExternalTbl) {
              if (mustPurge) {
                LOG.info("dropPartition() will purge " + partPath + " directly, skipping trash.");
              }
@@ -2761,10 +2769,12 @@ public class HiveMetaStore extends ThriftHiveMetastore {
        Table tbl = null;
        List<Partition> parts = null;
        boolean mustPurge = false;
+ boolean isExternalTbl = false;
        try {
          // We need Partition-s for firing events and for result; DN needs MPartition-s to drop.
          // Great... Maybe we could bypass fetching MPartitions by issuing direct SQL deletes.
          tbl = get_table_core(dbName, tblName);
+ isExternalTbl = isExternal(tbl);
          mustPurge = isMustPurge(envContext, tbl);
          int minCount = 0;
          RequestPartsSpec spec = request.getParts();
@@ -2829,13 +2839,15 @@ public class HiveMetaStore extends ThriftHiveMetastore {
            if (MetaStoreUtils.isArchived(part)) {
              Path archiveParentDir = MetaStoreUtils.getOriginalLocation(part);
              verifyIsWritablePath(archiveParentDir);
- checkTrashPurgeCombination(archiveParentDir, dbName + "." + tblName + "." + part.getValues(), mustPurge);
+ checkTrashPurgeCombination(archiveParentDir, dbName + "." + tblName + "." +
+ part.getValues(), mustPurge, deleteData && !isExternalTbl);
              archToDelete.add(archiveParentDir);
            }
            if ((part.getSd() != null) && (part.getSd().getLocation() != null)) {
              Path partPath = new Path(part.getSd().getLocation());
              verifyIsWritablePath(partPath);
- checkTrashPurgeCombination(partPath, dbName + "." + tblName + "." + part.getValues(), mustPurge);
+ checkTrashPurgeCombination(partPath, dbName + "." + tblName + "." + part.getValues(),
+ mustPurge, deleteData && !isExternalTbl);
              dirsToDelete.add(new PathAndPartValSize(partPath, part.getValues().size()));
            }
          }

http://git-wip-us.apache.org/repos/asf/hive/blob/d5b1adbb/ql/src/test/queries/clientpositive/encryption_drop_partition.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/encryption_drop_partition.q b/ql/src/test/queries/clientpositive/encryption_drop_partition.q
index a26aa19..57dfabd 100644
--- a/ql/src/test/queries/clientpositive/encryption_drop_partition.q
+++ b/ql/src/test/queries/clientpositive/encryption_drop_partition.q
@@ -12,7 +12,17 @@ CRYPTO CREATE_ZONE --keyName key_128 --path ${hiveconf:hive.metastore.warehouse.

  INSERT INTO encrypted_table_dp PARTITION(p)(p,key,value) values('2014-09-23', 1, 'foo'),('2014-09-24', 2, 'bar');
  SELECT * FROM encrypted_table_dp;
+
+CREATE EXTERNAL TABLE encrypted_ext_table_dp (key INT, value STRING) partitioned by (p STRING) LOCATION '${hiveconf:hive.metastore.warehouse.dir}/default/encrypted_table_dp';
+ALTER TABLE encrypted_ext_table_dp ADD PARTITION (p='2014-09-23') LOCATION '${hiveconf:hive.metastore.warehouse.dir}/default/encrypted_table_dp/p=2014-09-23';
+SELECT * FROM encrypted_ext_table_dp;
+ALTER TABLE encrypted_ext_table_dp DROP PARTITION (p='2014-09-23');
+SELECT * FROM encrypted_ext_table_dp;
+DROP TABLE encrypted_ext_table_dp;
+
+SELECT * FROM encrypted_table_dp;
  ALTER TABLE encrypted_table_dp DROP PARTITION (p='2014-09-23');
  SELECT * FROM encrypted_table_dp;
  ALTER TABLE encrypted_table_dp DROP PARTITION (p='2014-09-23') PURGE;
  SELECT * FROM encrypted_table_dp;
+DROP TABLE encrypted_table_dp PURGE;

http://git-wip-us.apache.org/repos/asf/hive/blob/d5b1adbb/ql/src/test/queries/clientpositive/encryption_drop_table.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/encryption_drop_table.q b/ql/src/test/queries/clientpositive/encryption_drop_table.q
index 193beea..2ae3c69 100644
--- a/ql/src/test/queries/clientpositive/encryption_drop_table.q
+++ b/ql/src/test/queries/clientpositive/encryption_drop_table.q
@@ -10,9 +10,16 @@ CRYPTO CREATE_KEY --keyName key_128 --bitLength 128;
  CRYPTO CREATE_ZONE --keyName key_128 --path ${hiveconf:hive.metastore.warehouse.dir}/default/encrypted_table;

  INSERT OVERWRITE TABLE encrypted_table SELECT * FROM src;
+
+CREATE EXTERNAL TABLE encrypted_ext_table (key INT, value STRING) LOCATION '${hiveconf:hive.metastore.warehouse.dir}/default/encrypted_table';
+SHOW TABLES;
+
+DROP TABLE default.encrypted_ext_table;
  SHOW TABLES;
+
  DROP TABLE default.encrypted_table;
  SHOW TABLES;
+
  DROP TABLE default.encrypted_table PURGE;
  SHOW TABLES;
-CRYPTO DELETE_KEY --keyName key_128;
\ No newline at end of file
+CRYPTO DELETE_KEY --keyName key_128;

http://git-wip-us.apache.org/repos/asf/hive/blob/d5b1adbb/ql/src/test/results/clientpositive/encrypted/encryption_drop_partition.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/encrypted/encryption_drop_partition.q.out b/ql/src/test/results/clientpositive/encrypted/encryption_drop_partition.q.out
index 067bf82..2643006 100644
--- a/ql/src/test/results/clientpositive/encrypted/encryption_drop_partition.q.out
+++ b/ql/src/test/results/clientpositive/encrypted/encryption_drop_partition.q.out
@@ -41,6 +41,74 @@ POSTHOOK: Input: default@encrypted_table_dp@p=2014-09-24
  #### A PARTIAL masked pattern was here #### data/warehouse/default/encrypted_table_dp/.hive-staging
  1 foo 2014-09-23
  2 bar 2014-09-24
+#### A masked pattern was here ####
+PREHOOK: type: CREATETABLE
+#### A masked pattern was here ####
+PREHOOK: Output: database:default
+PREHOOK: Output: default@encrypted_ext_table_dp
+#### A masked pattern was here ####
+POSTHOOK: type: CREATETABLE
+#### A masked pattern was here ####
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@encrypted_ext_table_dp
+#### A masked pattern was here ####
+PREHOOK: type: ALTERTABLE_ADDPARTS
+#### A masked pattern was here ####
+PREHOOK: Output: default@encrypted_ext_table_dp
+#### A masked pattern was here ####
+POSTHOOK: type: ALTERTABLE_ADDPARTS
+#### A masked pattern was here ####
+POSTHOOK: Output: default@encrypted_ext_table_dp
+POSTHOOK: Output: default@encrypted_ext_table_dp@p=2014-09-23
+PREHOOK: query: SELECT * FROM encrypted_ext_table_dp
+PREHOOK: type: QUERY
+PREHOOK: Input: default@encrypted_ext_table_dp
+PREHOOK: Input: default@encrypted_ext_table_dp@p=2014-09-23
+#### A PARTIAL masked pattern was here #### data/warehouse/default/encrypted_table_dp/.hive-staging
+POSTHOOK: query: SELECT * FROM encrypted_ext_table_dp
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@encrypted_ext_table_dp
+POSTHOOK: Input: default@encrypted_ext_table_dp@p=2014-09-23
+#### A PARTIAL masked pattern was here #### data/warehouse/default/encrypted_table_dp/.hive-staging
+1 foo 2014-09-23
+PREHOOK: query: ALTER TABLE encrypted_ext_table_dp DROP PARTITION (p='2014-09-23')
+PREHOOK: type: ALTERTABLE_DROPPARTS
+PREHOOK: Input: default@encrypted_ext_table_dp
+PREHOOK: Output: default@encrypted_ext_table_dp@p=2014-09-23
+POSTHOOK: query: ALTER TABLE encrypted_ext_table_dp DROP PARTITION (p='2014-09-23')
+POSTHOOK: type: ALTERTABLE_DROPPARTS
+POSTHOOK: Input: default@encrypted_ext_table_dp
+POSTHOOK: Output: default@encrypted_ext_table_dp@p=2014-09-23
+PREHOOK: query: SELECT * FROM encrypted_ext_table_dp
+PREHOOK: type: QUERY
+PREHOOK: Input: default@encrypted_ext_table_dp
+#### A PARTIAL masked pattern was here #### data/warehouse/default/encrypted_table_dp/.hive-staging
+POSTHOOK: query: SELECT * FROM encrypted_ext_table_dp
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@encrypted_ext_table_dp
+#### A PARTIAL masked pattern was here #### data/warehouse/default/encrypted_table_dp/.hive-staging
+PREHOOK: query: DROP TABLE encrypted_ext_table_dp
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@encrypted_ext_table_dp
+PREHOOK: Output: default@encrypted_ext_table_dp
+POSTHOOK: query: DROP TABLE encrypted_ext_table_dp
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@encrypted_ext_table_dp
+POSTHOOK: Output: default@encrypted_ext_table_dp
+PREHOOK: query: SELECT * FROM encrypted_table_dp
+PREHOOK: type: QUERY
+PREHOOK: Input: default@encrypted_table_dp
+PREHOOK: Input: default@encrypted_table_dp@p=2014-09-23
+PREHOOK: Input: default@encrypted_table_dp@p=2014-09-24
+#### A PARTIAL masked pattern was here #### data/warehouse/default/encrypted_table_dp/.hive-staging
+POSTHOOK: query: SELECT * FROM encrypted_table_dp
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@encrypted_table_dp
+POSTHOOK: Input: default@encrypted_table_dp@p=2014-09-23
+POSTHOOK: Input: default@encrypted_table_dp@p=2014-09-24
+#### A PARTIAL masked pattern was here #### data/warehouse/default/encrypted_table_dp/.hive-staging
+1 foo 2014-09-23
+2 bar 2014-09-24
  PREHOOK: query: ALTER TABLE encrypted_table_dp DROP PARTITION (p='2014-09-23')
  PREHOOK: type: ALTERTABLE_DROPPARTS
  PREHOOK: Input: default@encrypted_table_dp
@@ -79,3 +147,11 @@ POSTHOOK: Input: default@encrypted_table_dp
  POSTHOOK: Input: default@encrypted_table_dp@p=2014-09-24
  #### A PARTIAL masked pattern was here #### data/warehouse/default/encrypted_table_dp/.hive-staging
  2 bar 2014-09-24
+PREHOOK: query: DROP TABLE encrypted_table_dp PURGE
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@encrypted_table_dp
+PREHOOK: Output: default@encrypted_table_dp
+POSTHOOK: query: DROP TABLE encrypted_table_dp PURGE
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@encrypted_table_dp
+POSTHOOK: Output: default@encrypted_table_dp

http://git-wip-us.apache.org/repos/asf/hive/blob/d5b1adbb/ql/src/test/results/clientpositive/encrypted/encryption_drop_table.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/encrypted/encryption_drop_table.q.out b/ql/src/test/results/clientpositive/encrypted/encryption_drop_table.q.out
index 55eefa0..c5007ee 100644
--- a/ql/src/test/results/clientpositive/encrypted/encryption_drop_table.q.out
+++ b/ql/src/test/results/clientpositive/encrypted/encryption_drop_table.q.out
@@ -24,6 +24,33 @@ POSTHOOK: Input: default@src
  POSTHOOK: Output: default@encrypted_table
  POSTHOOK: Lineage: encrypted_table.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ]
  POSTHOOK: Lineage: encrypted_table.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+#### A masked pattern was here ####
+PREHOOK: type: CREATETABLE
+#### A masked pattern was here ####
+PREHOOK: Output: database:default
+PREHOOK: Output: default@encrypted_ext_table
+#### A masked pattern was here ####
+POSTHOOK: type: CREATETABLE
+#### A masked pattern was here ####
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@encrypted_ext_table
+PREHOOK: query: SHOW TABLES
+PREHOOK: type: SHOWTABLES
+PREHOOK: Input: database:default
+POSTHOOK: query: SHOW TABLES
+POSTHOOK: type: SHOWTABLES
+POSTHOOK: Input: database:default
+encrypted_ext_table
+encrypted_table
+src
+PREHOOK: query: DROP TABLE default.encrypted_ext_table
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@encrypted_ext_table
+PREHOOK: Output: default@encrypted_ext_table
+POSTHOOK: query: DROP TABLE default.encrypted_ext_table
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@encrypted_ext_table
+POSTHOOK: Output: default@encrypted_ext_table
  PREHOOK: query: SHOW TABLES
  PREHOOK: type: SHOWTABLES
  PREHOOK: Input: database:default

Search Discussions

Discussion Posts

Follow ups

Related Discussions

Discussion Navigation
viewthread | post
posts ‹ prev | 1 of 2 | next ›
Discussion Overview
groupcommits @
categorieshive, hadoop
postedMar 15, '16 at 3:41p
activeMar 15, '16 at 3:53p
posts2
users1
websitehive.apache.org

1 user in discussion

Ctang: 2 posts

People

Translate

site design / logo © 2021 Grokbase