Grokbase Groups Hive commits May 2015
FAQ
Repository: hive
Updated Branches:
   refs/heads/master fc05556ce -> 8d3108c41


HIVE-10629 Dropping table in an encrypted zone does not drop warehouse directory(Eugene Koifman, reviewed by Sergio Pena)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/8d3108c4
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/8d3108c4
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/8d3108c4

Branch: refs/heads/master
Commit: 8d3108c41802d7d1766c0d85bc81e895ad348dfb
Parents: fc05556
Author: Eugene Koifman <ekoifman@hortonworks.com>
Authored: Wed May 20 10:05:31 2015 -0700
Committer: Eugene Koifman <ekoifman@hortonworks.com>
Committed: Wed May 20 10:05:31 2015 -0700

----------------------------------------------------------------------
  data/scripts/q_test_cleanup_for_encryption.sql | 2 +-
  data/scripts/q_test_init_for_encryption.sql | 2 +-
  .../test/resources/testconfiguration.properties | 3 +-
  .../org/apache/hadoop/hive/ql/QTestUtil.java | 3 +-
  .../hadoop/hive/metastore/HiveMetaStore.java | 36 +++++++++----
  .../clientpositive/encryption_drop_table.q | 18 +++++++
  .../encryption_insert_partition_dynamic.q | 8 +--
  .../encryption_insert_partition_static.q | 8 +--
  .../encryption_join_unencrypted_tbl.q | 4 +-
  ...yption_join_with_different_encryption_keys.q | 8 +--
  .../encryption_load_data_to_encrypted_tables.q | 4 +-
  .../clientpositive/encryption_move_tbl.q | 4 +-
  .../encryption_select_read_only_encrypted_tbl.q | 4 +-
  .../encrypted/encryption_drop_table.q.out | 56 ++++++++++++++++++++
  .../encryption_insert_partition_dynamic.q.out | 16 +++---
  .../encryption_insert_partition_static.q.out | 30 +++++------
  .../encryption_join_unencrypted_tbl.q.out | 8 +--
  ...on_join_with_different_encryption_keys.q.out | 16 +++---
  ...cryption_load_data_to_encrypted_tables.q.out | 8 +--
  .../encrypted/encryption_move_tbl.q.out | 12 ++++-
  ...ryption_select_read_only_encrypted_tbl.q.out | 8 +--
  .../apache/hadoop/hive/shims/Hadoop23Shims.java | 3 ++
  22 files changed, 184 insertions(+), 77 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/8d3108c4/data/scripts/q_test_cleanup_for_encryption.sql
----------------------------------------------------------------------
diff --git a/data/scripts/q_test_cleanup_for_encryption.sql b/data/scripts/q_test_cleanup_for_encryption.sql
index 08264ee..070cdbe 100644
--- a/data/scripts/q_test_cleanup_for_encryption.sql
+++ b/data/scripts/q_test_cleanup_for_encryption.sql
@@ -1 +1 @@
-DROP TABLE IF EXISTS src;
+DROP TABLE IF EXISTS src PURGE;

http://git-wip-us.apache.org/repos/asf/hive/blob/8d3108c4/data/scripts/q_test_init_for_encryption.sql
----------------------------------------------------------------------
diff --git a/data/scripts/q_test_init_for_encryption.sql b/data/scripts/q_test_init_for_encryption.sql
index 1822ebb..56b44e0 100644
--- a/data/scripts/q_test_init_for_encryption.sql
+++ b/data/scripts/q_test_init_for_encryption.sql
@@ -1,4 +1,4 @@
-DROP TABLE IF EXISTS src;
+DROP TABLE IF EXISTS src PURGE;

  CREATE TABLE src(key STRING COMMENT 'default', value STRING COMMENT 'default') STORED AS TEXTFILE;


http://git-wip-us.apache.org/repos/asf/hive/blob/8d3108c4/itests/src/test/resources/testconfiguration.properties
----------------------------------------------------------------------
diff --git a/itests/src/test/resources/testconfiguration.properties b/itests/src/test/resources/testconfiguration.properties
index a485408..b9d85f6 100644
--- a/itests/src/test/resources/testconfiguration.properties
+++ b/itests/src/test/resources/testconfiguration.properties
@@ -349,7 +349,8 @@ encrypted.query.files=encryption_join_unencrypted_tbl.q,\
    encryption_select_read_only_unencrypted_tbl.q,\
    encryption_load_data_to_encrypted_tables.q, \
    encryption_unencrypted_nonhdfs_external_tables.q \
- encryption_move_tbl.q
+ encryption_move_tbl.q \
+ encryption_drop_table.q

  beeline.positive.exclude=add_part_exist.q,\
    alter1.q,\

http://git-wip-us.apache.org/repos/asf/hive/blob/8d3108c4/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
----------------------------------------------------------------------
diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
index 2b4bd85..39d5d9e 100644
--- a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
+++ b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
@@ -372,6 +372,7 @@ public class QTestUtil {
          // Set the security key provider so that the MiniDFS cluster is initialized
          // with encryption
          conf.set(SECURITY_KEY_PROVIDER_URI_NAME, getKeyProviderURI());
+ conf.setInt("fs.trash.interval", 50);

          dfs = shims.getMiniDfs(conf, numberOfDataNodes, true, null);
          fs = dfs.getFileSystem();
@@ -681,7 +682,7 @@ public class QTestUtil {
            if(tblObj.isIndexTable()) {
              continue;
            }
- db.dropTable(dbName, tblName);
+ db.dropTable(dbName, tblName, true, true, clusterType == MiniClusterType.encrypted);
          } else {
            // this table is defined in srcTables, drop all indexes on it
           List<Index> indexes = db.getIndexes(dbName, tblName, (short)-1);

http://git-wip-us.apache.org/repos/asf/hive/blob/8d3108c4/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
index e796048..5d8718c 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
@@ -55,6 +55,7 @@ import org.apache.commons.cli.OptionBuilder;
  import org.apache.commons.logging.Log;
  import org.apache.commons.logging.LogFactory;
  import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
  import org.apache.hadoop.fs.Path;
  import org.apache.hadoop.hive.common.FileUtils;
  import org.apache.hadoop.hive.common.LogUtils;
@@ -198,6 +199,7 @@ import org.apache.hadoop.hive.metastore.partition.spec.PartitionSpecProxy;
  import org.apache.hadoop.hive.metastore.txn.TxnHandler;
  import org.apache.hadoop.hive.serde2.Deserializer;
  import org.apache.hadoop.hive.serde2.SerDeException;
+import org.apache.hadoop.hive.shims.HadoopShims;
  import org.apache.hadoop.hive.shims.ShimLoader;
  import org.apache.hadoop.hive.shims.Utils;
  import org.apache.hadoop.hive.thrift.HadoopThriftAuthBridge;
@@ -1478,6 +1480,7 @@ public class HiveMetaStore extends ThriftHiveMetastore {
        Path tblPath = null;
        List<Path> partPaths = null;
        Table tbl = null;
+ boolean ifPurge = false;
        try {
          ms.openTransaction();
          // drop any partitions
@@ -1489,6 +1492,17 @@ public class HiveMetaStore extends ThriftHiveMetastore {
            throw new MetaException("Table metadata is corrupted");
          }

+ /**
+ * Trash may be skipped iff:
+ * 1. deleteData == true, obviously.
+ * 2. tbl is external.
+ * 3. Either
+ * 3.1. User has specified PURGE from the commandline, and if not,
+ * 3.2. User has set the table to auto-purge.
+ */
+ ifPurge = ((envContext != null) && Boolean.parseBoolean(envContext.getProperties().get("ifPurge")))
+ || (tbl.isSetParameters() && "true".equalsIgnoreCase(tbl.getParameters().get("auto.purge")));
+
          firePreEvent(new PreDropTableEvent(tbl, deleteData, this));

          boolean isIndexTable = isIndexTable(tbl);
@@ -1521,6 +1535,19 @@ public class HiveMetaStore extends ThriftHiveMetastore {
            }
          }

+ if(!ifPurge) {
+ String trashInterval = hiveConf.get("fs.trash.interval");
+ boolean trashEnabled = trashInterval != null && trashInterval.length() > 0
+ && Float.parseFloat(trashInterval) > 0;
+ if (trashEnabled) {
+ HadoopShims.HdfsEncryptionShim shim =
+ ShimLoader.getHadoopShims().createHdfsEncryptionShim(FileSystem.get(hiveConf), hiveConf);
+ if (shim.isPathEncrypted(tblPath)) {
+ throw new MetaException("Unable to drop table because it is in an encryption zone" +
+ " and trash is enabled. Use PURGE option to skip trash.");
+ }
+ }
+ }
          // Drop the partitions and get a list of locations which need to be deleted
          partPaths = dropPartitionsAndGetLocations(ms, dbname, name, tblPath,
              tbl.getPartitionKeys(), deleteData && !isExternal);
@@ -1536,15 +1563,6 @@ public class HiveMetaStore extends ThriftHiveMetastore {
            ms.rollbackTransaction();
          } else if (deleteData && !isExternal) {
            // Data needs deletion. Check if trash may be skipped.
- // Trash may be skipped iff:
- // 1. deleteData == true, obviously.
- // 2. tbl is external.
- // 3. Either
- // 3.1. User has specified PURGE from the commandline, and if not,
- // 3.2. User has set the table to auto-purge.
- boolean ifPurge = ((envContext != null) && Boolean.parseBoolean(envContext.getProperties().get("ifPurge")))
- ||
- (tbl.isSetParameters() && "true".equalsIgnoreCase(tbl.getParameters().get("auto.purge")));
            // Delete the data in the partitions which have other locations
            deletePartitionData(partPaths, ifPurge);
            // Delete the data in the table

http://git-wip-us.apache.org/repos/asf/hive/blob/8d3108c4/ql/src/test/queries/clientpositive/encryption_drop_table.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/encryption_drop_table.q b/ql/src/test/queries/clientpositive/encryption_drop_table.q
new file mode 100644
index 0000000..193beea
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/encryption_drop_table.q
@@ -0,0 +1,18 @@
+-- SORT_QUERY_RESULTS;
+
+-- we're setting this so that TestNegaiveCliDriver.vm doesn't stop processing after DROP TABLE fails;
+
+set hive.cli.errors.ignore=true;
+
+DROP TABLE IF EXISTS encrypted_table PURGE;
+CREATE TABLE encrypted_table (key INT, value STRING) LOCATION '${hiveconf:hive.metastore.warehouse.dir}/default/encrypted_table';
+CRYPTO CREATE_KEY --keyName key_128 --bitLength 128;
+CRYPTO CREATE_ZONE --keyName key_128 --path ${hiveconf:hive.metastore.warehouse.dir}/default/encrypted_table;
+
+INSERT OVERWRITE TABLE encrypted_table SELECT * FROM src;
+SHOW TABLES;
+DROP TABLE default.encrypted_table;
+SHOW TABLES;
+DROP TABLE default.encrypted_table PURGE;
+SHOW TABLES;
+CRYPTO DELETE_KEY --keyName key_128;
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hive/blob/8d3108c4/ql/src/test/queries/clientpositive/encryption_insert_partition_dynamic.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/encryption_insert_partition_dynamic.q b/ql/src/test/queries/clientpositive/encryption_insert_partition_dynamic.q
index 8674972..9556eed 100644
--- a/ql/src/test/queries/clientpositive/encryption_insert_partition_dynamic.q
+++ b/ql/src/test/queries/clientpositive/encryption_insert_partition_dynamic.q
@@ -6,8 +6,8 @@ set hive.enforce.bucketing=true;
  -- SORT_QUERY_RESULTS

  -- init
-drop table IF EXISTS encryptedTable;
-drop table IF EXISTS unencryptedTable;
+drop table IF EXISTS encryptedTable PURGE;
+drop table IF EXISTS unencryptedTable PURGE;

  create table encryptedTable(value string)
      partitioned by (key string) clustered by (value) into 2 buckets stored as orc
@@ -52,6 +52,6 @@ insert into table unencryptedTable partition (key)
  select * from unencryptedTable order by key;

  -- clean up
-drop table encryptedTable;
+drop table encryptedTable PURGE;
  CRYPTO DELETE_KEY --keyName key_1;
-drop table unencryptedTable;
\ No newline at end of file
+drop table unencryptedTable PURGE;
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hive/blob/8d3108c4/ql/src/test/queries/clientpositive/encryption_insert_partition_static.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/encryption_insert_partition_static.q b/ql/src/test/queries/clientpositive/encryption_insert_partition_static.q
index 10fb965..9bff0bd 100644
--- a/ql/src/test/queries/clientpositive/encryption_insert_partition_static.q
+++ b/ql/src/test/queries/clientpositive/encryption_insert_partition_static.q
@@ -5,8 +5,8 @@ set hive.enforce.bucketing=true;
  -- SORT_QUERY_RESULTS

  -- init
-drop table IF EXISTS encryptedTable;
-drop table IF EXISTS unencryptedTable;
+drop table IF EXISTS encryptedTable PURGE;
+drop table IF EXISTS unencryptedTable PURGE;

  create table encryptedTable(key string,
      value string) partitioned by (ds string) clustered by (key) into 2 buckets stored as orc
@@ -57,6 +57,6 @@ insert into table unencryptedTable partition
  select * from unencryptedTable order by key;

  -- clean up
-drop table encryptedTable;
+drop table encryptedTable PURGE;
  CRYPTO DELETE_KEY --keyName key_1;
-drop table unencryptedTable;
\ No newline at end of file
+drop table unencryptedTable PURGE;
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hive/blob/8d3108c4/ql/src/test/queries/clientpositive/encryption_join_unencrypted_tbl.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/encryption_join_unencrypted_tbl.q b/ql/src/test/queries/clientpositive/encryption_join_unencrypted_tbl.q
index adf82de..cf4d040 100644
--- a/ql/src/test/queries/clientpositive/encryption_join_unencrypted_tbl.q
+++ b/ql/src/test/queries/clientpositive/encryption_join_unencrypted_tbl.q
@@ -1,6 +1,6 @@
  --SORT_QUERY_RESULTS

-DROP TABLE IF EXISTS encrypted_table;
+DROP TABLE IF EXISTS encrypted_table PURGE;
  CREATE TABLE encrypted_table (key INT, value STRING) LOCATION '${hiveconf:hive.metastore.warehouse.dir}/default/encrypted_table';
  CRYPTO CREATE_KEY --keyName key_128 --bitLength 128;
  CRYPTO CREATE_ZONE --keyName key_128 --path ${hiveconf:hive.metastore.warehouse.dir}/default/encrypted_table;
@@ -11,5 +11,5 @@ SELECT * FROM encrypted_table;

  EXPLAIN EXTENDED SELECT * FROM src t1 JOIN encrypted_table t2 WHERE t1.key = t2.key;

-drop table encrypted_table;
+drop table encrypted_table PURGE;
  CRYPTO DELETE_KEY --keyName key_128;
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hive/blob/8d3108c4/ql/src/test/queries/clientpositive/encryption_join_with_different_encryption_keys.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/encryption_join_with_different_encryption_keys.q b/ql/src/test/queries/clientpositive/encryption_join_with_different_encryption_keys.q
index c830247..dfd3406 100644
--- a/ql/src/test/queries/clientpositive/encryption_join_with_different_encryption_keys.q
+++ b/ql/src/test/queries/clientpositive/encryption_join_with_different_encryption_keys.q
@@ -1,12 +1,12 @@
  --SORT_QUERY_RESULTS

  -- Java JCE must be installed in order to hava a key length of 256 bits
-DROP TABLE IF EXISTS table_key_1;
+DROP TABLE IF EXISTS table_key_1 PURGE;
  CREATE TABLE table_key_1 (key INT, value STRING) LOCATION '${hiveconf:hive.metastore.warehouse.dir}/table_key_1';
  CRYPTO CREATE_KEY --keyName key_1 --bitLength 128;
  CRYPTO CREATE_ZONE --keyName key_1 --path ${hiveconf:hive.metastore.warehouse.dir}/table_key_1;

-DROP TABLE IF EXISTS table_key_2;
+DROP TABLE IF EXISTS table_key_2 PURGE;
  CREATE TABLE table_key_2 (key INT, value STRING) LOCATION '${hiveconf:hive.metastore.warehouse.dir}/table_key_2';
  CRYPTO CREATE_KEY --keyName key_2 --bitLength 256;
  CRYPTO CREATE_ZONE --keyName key_2 --path ${hiveconf:hive.metastore.warehouse.dir}/table_key_2;
@@ -17,8 +17,8 @@ INSERT OVERWRITE TABLE table_key_2 SELECT * FROM src;
  EXPLAIN EXTENDED SELECT * FROM table_key_1 t1 JOIN table_key_2 t2 WHERE (t1.key = t2.key);
  SELECT * FROM table_key_1 t1 JOIN table_key_2 t2 WHERE (t1.key = t2.key);

-DROP TABLE table_key_1;
-DROP TABLE table_key_2;
+DROP TABLE table_key_1 PURGE;
+DROP TABLE table_key_2 PURGE;

  CRYPTO DELETE_KEY --keyName key_1;
  CRYPTO DELETE_KEY --keyName key_2;
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hive/blob/8d3108c4/ql/src/test/queries/clientpositive/encryption_load_data_to_encrypted_tables.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/encryption_load_data_to_encrypted_tables.q b/ql/src/test/queries/clientpositive/encryption_load_data_to_encrypted_tables.q
index e4d3a4c..a96807e 100644
--- a/ql/src/test/queries/clientpositive/encryption_load_data_to_encrypted_tables.q
+++ b/ql/src/test/queries/clientpositive/encryption_load_data_to_encrypted_tables.q
@@ -1,4 +1,4 @@
-DROP TABLE IF EXISTS encrypted_table;
+DROP TABLE IF EXISTS encrypted_table PURGE;

  CREATE TABLE encrypted_table (key STRING, value STRING) LOCATION '${hiveconf:hive.metastore.warehouse.dir}/encrypted_table';

@@ -15,6 +15,6 @@ dfs -copyFromLocal ../../data/files/kv1.txt hdfs:///tmp/kv1.txt;
  LOAD DATA INPATH '/tmp/kv1.txt' OVERWRITE INTO TABLE encrypted_table;
  SELECT * FROM encrypted_table;

-DROP TABLE encrypted_table;
+DROP TABLE encrypted_table PURGE;

  crypto delete_key --keyName key1;
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hive/blob/8d3108c4/ql/src/test/queries/clientpositive/encryption_move_tbl.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/encryption_move_tbl.q b/ql/src/test/queries/clientpositive/encryption_move_tbl.q
index 1168c6c..5a8c036 100644
--- a/ql/src/test/queries/clientpositive/encryption_move_tbl.q
+++ b/ql/src/test/queries/clientpositive/encryption_move_tbl.q
@@ -4,7 +4,7 @@

  set hive.cli.errors.ignore=true;

-DROP TABLE IF EXISTS encrypted_table;
+DROP TABLE IF EXISTS encrypted_table PURGE;
  CREATE TABLE encrypted_table (key INT, value STRING) LOCATION '${hiveconf:hive.metastore.warehouse.dir}/default/encrypted_table';
  CRYPTO CREATE_KEY --keyName key_128 --bitLength 128;
  CRYPTO CREATE_ZONE --keyName key_128 --path ${hiveconf:hive.metastore.warehouse.dir}/default/encrypted_table;
@@ -14,5 +14,7 @@ SHOW TABLES;
  ALTER TABLE default.encrypted_table RENAME TO default.plain_table;
  SHOW TABLES;

+DROP TABLE encrypted_table PURGE;
+
  CRYPTO DELETE_KEY --keyName key_128;


http://git-wip-us.apache.org/repos/asf/hive/blob/8d3108c4/ql/src/test/queries/clientpositive/encryption_select_read_only_encrypted_tbl.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/encryption_select_read_only_encrypted_tbl.q b/ql/src/test/queries/clientpositive/encryption_select_read_only_encrypted_tbl.q
index c013425..c9ac035 100644
--- a/ql/src/test/queries/clientpositive/encryption_select_read_only_encrypted_tbl.q
+++ b/ql/src/test/queries/clientpositive/encryption_select_read_only_encrypted_tbl.q
@@ -1,6 +1,6 @@
  -- SORT_QUERY_RESULTS

-DROP TABLE IF EXISTS encrypted_table;
+DROP TABLE IF EXISTS encrypted_table PURGE;
  CREATE TABLE encrypted_table (key INT, value STRING) LOCATION '${hiveconf:hive.metastore.warehouse.dir}/default/encrypted_table';

  CRYPTO CREATE_KEY --keyName key_128 --bitLength 128;
@@ -12,5 +12,5 @@ dfs -chmod -R 555 ${hiveconf:hive.metastore.warehouse.dir}/default/encrypted_tab

  SELECT count(*) FROM encrypted_table;

-drop table encrypted_table;
+drop table encrypted_table PURGE;
  CRYPTO DELETE_KEY --keyName key_128;

http://git-wip-us.apache.org/repos/asf/hive/blob/8d3108c4/ql/src/test/results/clientpositive/encrypted/encryption_drop_table.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/encrypted/encryption_drop_table.q.out b/ql/src/test/results/clientpositive/encrypted/encryption_drop_table.q.out
new file mode 100644
index 0000000..8f51fb9
--- /dev/null
+++ b/ql/src/test/results/clientpositive/encrypted/encryption_drop_table.q.out
@@ -0,0 +1,56 @@
+PREHOOK: query: DROP TABLE IF EXISTS encrypted_table PURGE
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: DROP TABLE IF EXISTS encrypted_table PURGE
+POSTHOOK: type: DROPTABLE
+#### A masked pattern was here ####
+PREHOOK: type: CREATETABLE
+#### A masked pattern was here ####
+PREHOOK: Output: database:default
+PREHOOK: Output: default@encrypted_table
+#### A masked pattern was here ####
+POSTHOOK: type: CREATETABLE
+#### A masked pattern was here ####
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@encrypted_table
+Encryption key created: 'key_128'
+Encryption zone created: '/build/ql/test/data/warehouse/default/encrypted_table' using key: 'key_128'
+PREHOOK: query: INSERT OVERWRITE TABLE encrypted_table SELECT * FROM src
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Output: default@encrypted_table
+POSTHOOK: query: INSERT OVERWRITE TABLE encrypted_table SELECT * FROM src
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Output: default@encrypted_table
+POSTHOOK: Lineage: encrypted_table.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: encrypted_table.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+PREHOOK: query: SHOW TABLES
+PREHOOK: type: SHOWTABLES
+POSTHOOK: query: SHOW TABLES
+POSTHOOK: type: SHOWTABLES
+encrypted_table
+src
+PREHOOK: query: DROP TABLE default.encrypted_table
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@encrypted_table
+PREHOOK: Output: default@encrypted_table
+FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.exec.DDLTask. MetaException(message:Unable to drop table because it is in an encryption zone and trash is enabled. Use PURGE option to skip trash.)
+PREHOOK: query: SHOW TABLES
+PREHOOK: type: SHOWTABLES
+POSTHOOK: query: SHOW TABLES
+POSTHOOK: type: SHOWTABLES
+encrypted_table
+src
+PREHOOK: query: DROP TABLE default.encrypted_table PURGE
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@encrypted_table
+PREHOOK: Output: default@encrypted_table
+POSTHOOK: query: DROP TABLE default.encrypted_table PURGE
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@encrypted_table
+POSTHOOK: Output: default@encrypted_table
+PREHOOK: query: SHOW TABLES
+PREHOOK: type: SHOWTABLES
+POSTHOOK: query: SHOW TABLES
+POSTHOOK: type: SHOWTABLES
+src

http://git-wip-us.apache.org/repos/asf/hive/blob/8d3108c4/ql/src/test/results/clientpositive/encrypted/encryption_insert_partition_dynamic.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/encrypted/encryption_insert_partition_dynamic.q.out b/ql/src/test/results/clientpositive/encrypted/encryption_insert_partition_dynamic.q.out
index 22cbce9..cb6dc5c 100644
--- a/ql/src/test/results/clientpositive/encrypted/encryption_insert_partition_dynamic.q.out
+++ b/ql/src/test/results/clientpositive/encrypted/encryption_insert_partition_dynamic.q.out
@@ -1,16 +1,16 @@
  PREHOOK: query: -- SORT_QUERY_RESULTS

  -- init
-drop table IF EXISTS encryptedTable
+drop table IF EXISTS encryptedTable PURGE
  PREHOOK: type: DROPTABLE
  POSTHOOK: query: -- SORT_QUERY_RESULTS

  -- init
-drop table IF EXISTS encryptedTable
+drop table IF EXISTS encryptedTable PURGE
  POSTHOOK: type: DROPTABLE
-PREHOOK: query: drop table IF EXISTS unencryptedTable
+PREHOOK: query: drop table IF EXISTS unencryptedTable PURGE
  PREHOOK: type: DROPTABLE
-POSTHOOK: query: drop table IF EXISTS unencryptedTable
+POSTHOOK: query: drop table IF EXISTS unencryptedTable PURGE
  POSTHOOK: type: DROPTABLE
  PREHOOK: query: create table encryptedTable(value string)
      partitioned by (key string) clustered by (value) into 2 buckets stored as orc
@@ -882,21 +882,21 @@ val_501 501
  val_502 502
  val_86 86
  PREHOOK: query: -- clean up
-drop table encryptedTable
+drop table encryptedTable PURGE
  PREHOOK: type: DROPTABLE
  PREHOOK: Input: default@encryptedtable
  PREHOOK: Output: default@encryptedtable
  POSTHOOK: query: -- clean up
-drop table encryptedTable
+drop table encryptedTable PURGE
  POSTHOOK: type: DROPTABLE
  POSTHOOK: Input: default@encryptedtable
  POSTHOOK: Output: default@encryptedtable
  Encryption key deleted: 'key_1'
-PREHOOK: query: drop table unencryptedTable
+PREHOOK: query: drop table unencryptedTable PURGE
  PREHOOK: type: DROPTABLE
  PREHOOK: Input: default@unencryptedtable
  PREHOOK: Output: default@unencryptedtable
-POSTHOOK: query: drop table unencryptedTable
+POSTHOOK: query: drop table unencryptedTable PURGE
  POSTHOOK: type: DROPTABLE
  POSTHOOK: Input: default@unencryptedtable
  POSTHOOK: Output: default@unencryptedtable

http://git-wip-us.apache.org/repos/asf/hive/blob/8d3108c4/ql/src/test/results/clientpositive/encrypted/encryption_insert_partition_static.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/encrypted/encryption_insert_partition_static.q.out b/ql/src/test/results/clientpositive/encrypted/encryption_insert_partition_static.q.out
index 96f8b6a..8966608 100644
--- a/ql/src/test/results/clientpositive/encrypted/encryption_insert_partition_static.q.out
+++ b/ql/src/test/results/clientpositive/encrypted/encryption_insert_partition_static.q.out
@@ -1,16 +1,16 @@
  PREHOOK: query: -- SORT_QUERY_RESULTS

  -- init
-drop table IF EXISTS encryptedTable
+drop table IF EXISTS encryptedTable PURGE
  PREHOOK: type: DROPTABLE
  POSTHOOK: query: -- SORT_QUERY_RESULTS

  -- init
-drop table IF EXISTS encryptedTable
+drop table IF EXISTS encryptedTable PURGE
  POSTHOOK: type: DROPTABLE
-PREHOOK: query: drop table IF EXISTS unencryptedTable
+PREHOOK: query: drop table IF EXISTS unencryptedTable PURGE
  PREHOOK: type: DROPTABLE
-POSTHOOK: query: drop table IF EXISTS unencryptedTable
+POSTHOOK: query: drop table IF EXISTS unencryptedTable PURGE
  POSTHOOK: type: DROPTABLE
  PREHOOK: query: create table encryptedTable(key string,
      value string) partitioned by (ds string) clustered by (key) into 2 buckets stored as orc
@@ -555,16 +555,16 @@ STAGE PLANS:
        Map Operator Tree:
            TableScan
              alias: encryptedtable
- Statistics: Num rows: 12 Data size: 2767 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 12 Data size: 2777 Basic stats: COMPLETE Column stats: NONE
              GatherStats: false
              Select Operator
                expressions: key (type: string), value (type: string)
                outputColumnNames: _col0, _col1
- Statistics: Num rows: 12 Data size: 2767 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 12 Data size: 2777 Basic stats: COMPLETE Column stats: NONE
                Reduce Output Operator
                  sort order:
                  Map-reduce partition columns: _col0 (type: string)
- Statistics: Num rows: 12 Data size: 2767 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 12 Data size: 2777 Basic stats: COMPLETE Column stats: NONE
                  tag: -1
                  value expressions: _col0 (type: string), _col1 (type: string)
                  auto parallelism: false
@@ -595,7 +595,7 @@ STAGE PLANS:
                serialization.ddl struct encryptedtable { string key, string value}
                serialization.format 1
                serialization.lib org.apache.hadoop.hive.ql.io.orc.OrcSerde
- totalSize 1385
+ totalSize 1392
  #### A masked pattern was here ####
              serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde

@@ -643,7 +643,7 @@ STAGE PLANS:
                serialization.ddl struct encryptedtable { string key, string value}
                serialization.format 1
                serialization.lib org.apache.hadoop.hive.ql.io.orc.OrcSerde
- totalSize 1382
+ totalSize 1385
  #### A masked pattern was here ####
              serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde

@@ -675,14 +675,14 @@ STAGE PLANS:
          Select Operator
            expressions: VALUE._col0 (type: string), VALUE._col1 (type: string)
            outputColumnNames: _col0, _col1
- Statistics: Num rows: 12 Data size: 2767 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 12 Data size: 2777 Basic stats: COMPLETE Column stats: NONE
            File Output Operator
              compressed: false
              GlobalTableId: 1
  #### A PARTIAL masked pattern was here #### data/warehouse/unencryptedtable/ds=today/.hive-staging
              NumFilesPerFileSink: 1
              Static Partition Specification: ds=today/
- Statistics: Num rows: 12 Data size: 2767 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 12 Data size: 2777 Basic stats: COMPLETE Column stats: NONE
  #### A PARTIAL masked pattern was here #### data/warehouse/unencryptedtable/ds=today/.hive-staging
              table:
                  input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
@@ -775,21 +775,21 @@ POSTHOOK: Input: default@unencryptedtable@ds=today
  502 val_502 today
  86 val_86 today
  PREHOOK: query: -- clean up
-drop table encryptedTable
+drop table encryptedTable PURGE
  PREHOOK: type: DROPTABLE
  PREHOOK: Input: default@encryptedtable
  PREHOOK: Output: default@encryptedtable
  POSTHOOK: query: -- clean up
-drop table encryptedTable
+drop table encryptedTable PURGE
  POSTHOOK: type: DROPTABLE
  POSTHOOK: Input: default@encryptedtable
  POSTHOOK: Output: default@encryptedtable
  Encryption key deleted: 'key_1'
-PREHOOK: query: drop table unencryptedTable
+PREHOOK: query: drop table unencryptedTable PURGE
  PREHOOK: type: DROPTABLE
  PREHOOK: Input: default@unencryptedtable
  PREHOOK: Output: default@unencryptedtable
-POSTHOOK: query: drop table unencryptedTable
+POSTHOOK: query: drop table unencryptedTable PURGE
  POSTHOOK: type: DROPTABLE
  POSTHOOK: Input: default@unencryptedtable
  POSTHOOK: Output: default@unencryptedtable

http://git-wip-us.apache.org/repos/asf/hive/blob/8d3108c4/ql/src/test/results/clientpositive/encrypted/encryption_join_unencrypted_tbl.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/encrypted/encryption_join_unencrypted_tbl.q.out b/ql/src/test/results/clientpositive/encrypted/encryption_join_unencrypted_tbl.q.out
index 0caf694..9b6f750 100644
--- a/ql/src/test/results/clientpositive/encrypted/encryption_join_unencrypted_tbl.q.out
+++ b/ql/src/test/results/clientpositive/encrypted/encryption_join_unencrypted_tbl.q.out
@@ -1,10 +1,10 @@
  PREHOOK: query: --SORT_QUERY_RESULTS

-DROP TABLE IF EXISTS encrypted_table
+DROP TABLE IF EXISTS encrypted_table PURGE
  PREHOOK: type: DROPTABLE
  POSTHOOK: query: --SORT_QUERY_RESULTS

-DROP TABLE IF EXISTS encrypted_table
+DROP TABLE IF EXISTS encrypted_table PURGE
  POSTHOOK: type: DROPTABLE
  #### A masked pattern was here ####
  PREHOOK: type: CREATETABLE
@@ -752,11 +752,11 @@ STAGE PLANS:
        Processor Tree:
          ListSink

-PREHOOK: query: drop table encrypted_table
+PREHOOK: query: drop table encrypted_table PURGE
  PREHOOK: type: DROPTABLE
  PREHOOK: Input: default@encrypted_table
  PREHOOK: Output: default@encrypted_table
-POSTHOOK: query: drop table encrypted_table
+POSTHOOK: query: drop table encrypted_table PURGE
  POSTHOOK: type: DROPTABLE
  POSTHOOK: Input: default@encrypted_table
  POSTHOOK: Output: default@encrypted_table

http://git-wip-us.apache.org/repos/asf/hive/blob/8d3108c4/ql/src/test/results/clientpositive/encrypted/encryption_join_with_different_encryption_keys.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/encrypted/encryption_join_with_different_encryption_keys.q.out b/ql/src/test/results/clientpositive/encrypted/encryption_join_with_different_encryption_keys.q.out
index 7d8ffce..40f6898 100644
--- a/ql/src/test/results/clientpositive/encrypted/encryption_join_with_different_encryption_keys.q.out
+++ b/ql/src/test/results/clientpositive/encrypted/encryption_join_with_different_encryption_keys.q.out
@@ -1,12 +1,12 @@
  PREHOOK: query: --SORT_QUERY_RESULTS

  -- Java JCE must be installed in order to hava a key length of 256 bits
-DROP TABLE IF EXISTS table_key_1
+DROP TABLE IF EXISTS table_key_1 PURGE
  PREHOOK: type: DROPTABLE
  POSTHOOK: query: --SORT_QUERY_RESULTS

  -- Java JCE must be installed in order to hava a key length of 256 bits
-DROP TABLE IF EXISTS table_key_1
+DROP TABLE IF EXISTS table_key_1 PURGE
  POSTHOOK: type: DROPTABLE
  #### A masked pattern was here ####
  PREHOOK: type: CREATETABLE
@@ -20,9 +20,9 @@ POSTHOOK: Output: database:default
  POSTHOOK: Output: default@table_key_1
  Encryption key created: 'key_1'
  Encryption zone created: '/build/ql/test/data/warehouse/table_key_1' using key: 'key_1'
-PREHOOK: query: DROP TABLE IF EXISTS table_key_2
+PREHOOK: query: DROP TABLE IF EXISTS table_key_2 PURGE
  PREHOOK: type: DROPTABLE
-POSTHOOK: query: DROP TABLE IF EXISTS table_key_2
+POSTHOOK: query: DROP TABLE IF EXISTS table_key_2 PURGE
  POSTHOOK: type: DROPTABLE
  #### A masked pattern was here ####
  PREHOOK: type: CREATETABLE
@@ -1310,19 +1310,19 @@ POSTHOOK: Input: default@table_key_2
  498 val_498 498 val_498
  498 val_498 498 val_498
  498 val_498 498 val_498
-PREHOOK: query: DROP TABLE table_key_1
+PREHOOK: query: DROP TABLE table_key_1 PURGE
  PREHOOK: type: DROPTABLE
  PREHOOK: Input: default@table_key_1
  PREHOOK: Output: default@table_key_1
-POSTHOOK: query: DROP TABLE table_key_1
+POSTHOOK: query: DROP TABLE table_key_1 PURGE
  POSTHOOK: type: DROPTABLE
  POSTHOOK: Input: default@table_key_1
  POSTHOOK: Output: default@table_key_1
-PREHOOK: query: DROP TABLE table_key_2
+PREHOOK: query: DROP TABLE table_key_2 PURGE
  PREHOOK: type: DROPTABLE
  PREHOOK: Input: default@table_key_2
  PREHOOK: Output: default@table_key_2
-POSTHOOK: query: DROP TABLE table_key_2
+POSTHOOK: query: DROP TABLE table_key_2 PURGE
  POSTHOOK: type: DROPTABLE
  POSTHOOK: Input: default@table_key_2
  POSTHOOK: Output: default@table_key_2

http://git-wip-us.apache.org/repos/asf/hive/blob/8d3108c4/ql/src/test/results/clientpositive/encrypted/encryption_load_data_to_encrypted_tables.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/encrypted/encryption_load_data_to_encrypted_tables.q.out b/ql/src/test/results/clientpositive/encrypted/encryption_load_data_to_encrypted_tables.q.out
index 46f36d8..35de14f 100644
--- a/ql/src/test/results/clientpositive/encrypted/encryption_load_data_to_encrypted_tables.q.out
+++ b/ql/src/test/results/clientpositive/encrypted/encryption_load_data_to_encrypted_tables.q.out
@@ -1,6 +1,6 @@
-PREHOOK: query: DROP TABLE IF EXISTS encrypted_table
+PREHOOK: query: DROP TABLE IF EXISTS encrypted_table PURGE
  PREHOOK: type: DROPTABLE
-POSTHOOK: query: DROP TABLE IF EXISTS encrypted_table
+POSTHOOK: query: DROP TABLE IF EXISTS encrypted_table PURGE
  POSTHOOK: type: DROPTABLE
  #### A masked pattern was here ####
  PREHOOK: type: CREATETABLE
@@ -1048,11 +1048,11 @@ POSTHOOK: Input: default@encrypted_table
  400 val_400
  200 val_200
  97 val_97
-PREHOOK: query: DROP TABLE encrypted_table
+PREHOOK: query: DROP TABLE encrypted_table PURGE
  PREHOOK: type: DROPTABLE
  PREHOOK: Input: default@encrypted_table
  PREHOOK: Output: default@encrypted_table
-POSTHOOK: query: DROP TABLE encrypted_table
+POSTHOOK: query: DROP TABLE encrypted_table PURGE
  POSTHOOK: type: DROPTABLE
  POSTHOOK: Input: default@encrypted_table
  POSTHOOK: Output: default@encrypted_table

http://git-wip-us.apache.org/repos/asf/hive/blob/8d3108c4/ql/src/test/results/clientpositive/encrypted/encryption_move_tbl.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/encrypted/encryption_move_tbl.q.out b/ql/src/test/results/clientpositive/encrypted/encryption_move_tbl.q.out
index d18827e..e0098f3 100644
--- a/ql/src/test/results/clientpositive/encrypted/encryption_move_tbl.q.out
+++ b/ql/src/test/results/clientpositive/encrypted/encryption_move_tbl.q.out
@@ -1,6 +1,6 @@
-PREHOOK: query: DROP TABLE IF EXISTS encrypted_table
+PREHOOK: query: DROP TABLE IF EXISTS encrypted_table PURGE
  PREHOOK: type: DROPTABLE
-POSTHOOK: query: DROP TABLE IF EXISTS encrypted_table
+POSTHOOK: query: DROP TABLE IF EXISTS encrypted_table PURGE
  POSTHOOK: type: DROPTABLE
  #### A masked pattern was here ####
  PREHOOK: type: CREATETABLE
@@ -41,3 +41,11 @@ POSTHOOK: query: SHOW TABLES
  POSTHOOK: type: SHOWTABLES
  encrypted_table
  src
+PREHOOK: query: DROP TABLE encrypted_table PURGE
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@encrypted_table
+PREHOOK: Output: default@encrypted_table
+POSTHOOK: query: DROP TABLE encrypted_table PURGE
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@encrypted_table
+POSTHOOK: Output: default@encrypted_table

http://git-wip-us.apache.org/repos/asf/hive/blob/8d3108c4/ql/src/test/results/clientpositive/encrypted/encryption_select_read_only_encrypted_tbl.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/encrypted/encryption_select_read_only_encrypted_tbl.q.out b/ql/src/test/results/clientpositive/encrypted/encryption_select_read_only_encrypted_tbl.q.out
index c4c0dde..5d4d2af 100644
--- a/ql/src/test/results/clientpositive/encrypted/encryption_select_read_only_encrypted_tbl.q.out
+++ b/ql/src/test/results/clientpositive/encrypted/encryption_select_read_only_encrypted_tbl.q.out
@@ -1,10 +1,10 @@
  PREHOOK: query: -- SORT_QUERY_RESULTS

-DROP TABLE IF EXISTS encrypted_table
+DROP TABLE IF EXISTS encrypted_table PURGE
  PREHOOK: type: DROPTABLE
  POSTHOOK: query: -- SORT_QUERY_RESULTS

-DROP TABLE IF EXISTS encrypted_table
+DROP TABLE IF EXISTS encrypted_table PURGE
  POSTHOOK: type: DROPTABLE
  #### A masked pattern was here ####
  PREHOOK: type: CREATETABLE
@@ -35,11 +35,11 @@ POSTHOOK: type: QUERY
  POSTHOOK: Input: default@encrypted_table
  #### A masked pattern was here ####
  500
-PREHOOK: query: drop table encrypted_table
+PREHOOK: query: drop table encrypted_table PURGE
  PREHOOK: type: DROPTABLE
  PREHOOK: Input: default@encrypted_table
  PREHOOK: Output: default@encrypted_table
-POSTHOOK: query: drop table encrypted_table
+POSTHOOK: query: drop table encrypted_table PURGE
  POSTHOOK: type: DROPTABLE
  POSTHOOK: Input: default@encrypted_table
  POSTHOOK: Output: default@encrypted_table

http://git-wip-us.apache.org/repos/asf/hive/blob/8d3108c4/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java
----------------------------------------------------------------------
diff --git a/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java b/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java
index d349068..19324b8 100644
--- a/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java
+++ b/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java
@@ -1215,6 +1215,9 @@ public class Hadoop23Shims extends HadoopShimsSecure {
        } else {
          fullPath = path.getFileSystem(conf).makeQualified(path);
        }
+ if(!"hdfs".equalsIgnoreCase(path.toUri().getScheme())) {
+ return false;
+ }
        return (hdfsAdmin.getEncryptionZoneForPath(fullPath) != null);
      }

Search Discussions

Related Discussions

Discussion Navigation
viewthread | post
posts ‹ prev | 1 of 1 | next ›
Discussion Overview
groupcommits @
categorieshive, hadoop
postedMay 20, '15 at 5:06p
activeMay 20, '15 at 5:06p
posts1
users1
websitehive.apache.org

1 user in discussion

Ekoifman: 1 post

People

Translate

site design / logo © 2021 Grokbase