Grokbase Groups Hive commits May 2015
FAQ
Repository: hive
Updated Branches:
   refs/heads/master 05a3d2ab7 -> bc0138c43


HIVE-10630:Renaming tables across encryption zones renames table even though the operation throws error (Eugene Koifman, reviewed by Sergio Pena and Ashutosh Chauhan)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/bc0138c4
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/bc0138c4
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/bc0138c4

Branch: refs/heads/master
Commit: bc0138c436add2335d2045b6c7bf86bc6a15cc27
Parents: 05a3d2a
Author: Eugene Koifman <ekoifman@hortonworks.com>
Authored: Thu May 14 10:29:00 2015 -0700
Committer: Eugene Koifman <ekoifman@hortonworks.com>
Committed: Thu May 14 10:29:00 2015 -0700

----------------------------------------------------------------------
  .../test/resources/testconfiguration.properties | 3 +-
  .../org/apache/hadoop/hive/ql/QTestUtil.java | 10 ++++-
  .../hadoop/hive/metastore/HiveAlterHandler.java | 28 ++++++++++---
  .../hadoop/hive/metastore/ObjectStore.java | 2 +-
  .../clientpositive/encryption_move_tbl.q | 18 ++++++++
  .../encrypted/encryption_move_tbl.q.out | 43 ++++++++++++++++++++
  6 files changed, 96 insertions(+), 8 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/bc0138c4/itests/src/test/resources/testconfiguration.properties
----------------------------------------------------------------------
diff --git a/itests/src/test/resources/testconfiguration.properties b/itests/src/test/resources/testconfiguration.properties
index eeb46cc..f9c9351 100644
--- a/itests/src/test/resources/testconfiguration.properties
+++ b/itests/src/test/resources/testconfiguration.properties
@@ -337,7 +337,8 @@ encrypted.query.files=encryption_join_unencrypted_tbl.q,\
    encryption_select_read_only_encrypted_tbl.q,\
    encryption_select_read_only_unencrypted_tbl.q,\
    encryption_load_data_to_encrypted_tables.q, \
- encryption_unencrypted_nonhdfs_external_tables.q
+ encryption_unencrypted_nonhdfs_external_tables.q \
+ encryption_move_tbl.q

  beeline.positive.exclude=add_part_exist.q,\
    alter1.q,\

http://git-wip-us.apache.org/repos/asf/hive/blob/bc0138c4/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
----------------------------------------------------------------------
diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
index 3e29d3c..f357c14 100644
--- a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
+++ b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
@@ -1029,7 +1029,7 @@ public class QTestUtil {
          rc = cliDriver.processLine(command);
        }

- if (rc != 0) {
+ if (rc != 0 && !ignoreErrors()) {
          break;
        }
        command = "";
@@ -1040,6 +1040,14 @@ public class QTestUtil {
      return rc;
    }

+ /**
+ * This allows a .q file to continue executing after a statement runs into an error which is convenient
+ * if you want to use another hive cmd after the failure to sanity check the state of the system.
+ */
+ private boolean ignoreErrors() {
+ return conf.getBoolVar(HiveConf.ConfVars.CLIIGNOREERRORS);
+ }
+
    private boolean isHiveCommand(String command) {
      String[] cmd = command.trim().split("\\s+");
      if (HiveCommand.find(cmd) != null) {

http://git-wip-us.apache.org/repos/asf/hive/blob/bc0138c4/metastore/src/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java
index 50ec1e8..a3f2359 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java
@@ -45,6 +45,7 @@ import org.apache.hadoop.hive.metastore.api.NoSuchObjectException;
  import org.apache.hadoop.hive.metastore.api.Partition;
  import org.apache.hadoop.hive.metastore.api.Table;
  import org.apache.hadoop.hive.metastore.api.hive_metastoreConstants;
+import org.apache.hadoop.ipc.RemoteException;
  import org.apache.hive.common.util.HiveStringUtils;

  import com.google.common.collect.Lists;
@@ -260,17 +261,18 @@ public class HiveAlterHandler implements AlterHandler {
          // rename the src to destination
          try {
            if (srcFs.exists(srcPath) && !srcFs.rename(srcPath, destPath)) {
- throw new IOException("Renaming " + srcPath + " to " + destPath + " is failed");
+ throw new IOException("Renaming " + srcPath + " to " + destPath + " failed");
            }
          } catch (IOException e) {
+ LOG.error("Alter Table operation for " + dbname + "." + name + " failed.", e);
            boolean revertMetaDataTransaction = false;
            try {
              msdb.openTransaction();
- msdb.alterTable(dbname, newt.getTableName(), oldt);
+ msdb.alterTable(newt.getDbName(), newt.getTableName(), oldt);
              for (ObjectPair<Partition, String> pair : altps) {
                Partition part = pair.getFirst();
                part.getSd().setLocation(pair.getSecond());
- msdb.alterPartition(dbname, name, part.getValues(), part);
+ msdb.alterPartition(newt.getDbName(), name, part.getValues(), part);
              }
              revertMetaDataTransaction = msdb.commitTransaction();
            } catch (Exception e1) {
@@ -288,8 +290,8 @@ public class HiveAlterHandler implements AlterHandler {
                msdb.rollbackTransaction();
              }
            }
- throw new InvalidOperationException("Unable to access old location "
- + srcPath + " for table " + dbname + "." + name);
+ throw new InvalidOperationException("Alter Table operation for " + dbname + "." + name +
+ " failed to move data due to: '" + getSimpleMessage(e) + "' See hive log file for details.");
          }
        }
      }
@@ -298,6 +300,22 @@ public class HiveAlterHandler implements AlterHandler {
      }
    }

+ /**
+ * RemoteExceptionS from hadoop RPC wrap the stack trace into e.getMessage() which makes
+ * logs/stack traces confusing.
+ * @param ex
+ * @return
+ */
+ String getSimpleMessage(IOException ex) {
+ if(ex instanceof RemoteException) {
+ String msg = ex.getMessage();
+ if(msg == null || !msg.contains("\n")) {
+ return msg;
+ }
+ return msg.substring(0, msg.indexOf('\n'));
+ }
+ return ex.getMessage();
+ }
    public Partition alterPartition(final RawStore msdb, Warehouse wh, final String dbname,
        final String name, final List<String> part_vals, final Partition new_part)
        throws InvalidOperationException, InvalidObjectException, AlreadyExistsException,

http://git-wip-us.apache.org/repos/asf/hive/blob/bc0138c4/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java b/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java
index 583e23f..fd61333 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java
@@ -2758,7 +2758,7 @@ public class ObjectStore implements RawStore, Configurable {

        MTable oldt = getMTable(dbname, name);
        if (oldt == null) {
- throw new MetaException("table " + name + " doesn't exist");
+ throw new MetaException("table " + dbname + "." + name + " doesn't exist");
        }

        // For now only alter name, owner, parameters, cols, bucketcols are allowed

http://git-wip-us.apache.org/repos/asf/hive/blob/bc0138c4/ql/src/test/queries/clientpositive/encryption_move_tbl.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/encryption_move_tbl.q b/ql/src/test/queries/clientpositive/encryption_move_tbl.q
new file mode 100644
index 0000000..1168c6c
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/encryption_move_tbl.q
@@ -0,0 +1,18 @@
+-- SORT_QUERY_RESULTS;
+
+-- we're setting this so that TestNegaiveCliDriver.vm doesn't stop processing after ALTER TABLE fails;
+
+set hive.cli.errors.ignore=true;
+
+DROP TABLE IF EXISTS encrypted_table;
+CREATE TABLE encrypted_table (key INT, value STRING) LOCATION '${hiveconf:hive.metastore.warehouse.dir}/default/encrypted_table';
+CRYPTO CREATE_KEY --keyName key_128 --bitLength 128;
+CRYPTO CREATE_ZONE --keyName key_128 --path ${hiveconf:hive.metastore.warehouse.dir}/default/encrypted_table;
+
+INSERT OVERWRITE TABLE encrypted_table SELECT * FROM src;
+SHOW TABLES;
+ALTER TABLE default.encrypted_table RENAME TO default.plain_table;
+SHOW TABLES;
+
+CRYPTO DELETE_KEY --keyName key_128;
+

http://git-wip-us.apache.org/repos/asf/hive/blob/bc0138c4/ql/src/test/results/clientpositive/encrypted/encryption_move_tbl.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/encrypted/encryption_move_tbl.q.out b/ql/src/test/results/clientpositive/encrypted/encryption_move_tbl.q.out
new file mode 100644
index 0000000..d18827e
--- /dev/null
+++ b/ql/src/test/results/clientpositive/encrypted/encryption_move_tbl.q.out
@@ -0,0 +1,43 @@
+PREHOOK: query: DROP TABLE IF EXISTS encrypted_table
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: DROP TABLE IF EXISTS encrypted_table
+POSTHOOK: type: DROPTABLE
+#### A masked pattern was here ####
+PREHOOK: type: CREATETABLE
+#### A masked pattern was here ####
+PREHOOK: Output: database:default
+PREHOOK: Output: default@encrypted_table
+#### A masked pattern was here ####
+POSTHOOK: type: CREATETABLE
+#### A masked pattern was here ####
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@encrypted_table
+Encryption key created: 'key_128'
+Encryption zone created: '/build/ql/test/data/warehouse/default/encrypted_table' using key: 'key_128'
+PREHOOK: query: INSERT OVERWRITE TABLE encrypted_table SELECT * FROM src
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Output: default@encrypted_table
+POSTHOOK: query: INSERT OVERWRITE TABLE encrypted_table SELECT * FROM src
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Output: default@encrypted_table
+POSTHOOK: Lineage: encrypted_table.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: encrypted_table.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+PREHOOK: query: SHOW TABLES
+PREHOOK: type: SHOWTABLES
+POSTHOOK: query: SHOW TABLES
+POSTHOOK: type: SHOWTABLES
+encrypted_table
+src
+PREHOOK: query: ALTER TABLE default.encrypted_table RENAME TO default.plain_table
+PREHOOK: type: ALTERTABLE_RENAME
+PREHOOK: Input: default@encrypted_table
+PREHOOK: Output: default@encrypted_table
+FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.exec.DDLTask. Unable to alter table. Alter Table operation for default.encrypted_table failed to move data due to: '/build/ql/test/data/warehouse/default/encrypted_table can't be moved from an encryption zone.' See hive log file for details.
+PREHOOK: query: SHOW TABLES
+PREHOOK: type: SHOWTABLES
+POSTHOOK: query: SHOW TABLES
+POSTHOOK: type: SHOWTABLES
+encrypted_table
+src

Search Discussions

Related Discussions

Discussion Navigation
viewthread | post
posts ‹ prev | 1 of 1 | next ›
Discussion Overview
groupcommits @
categorieshive, hadoop
postedMay 14, '15 at 5:29p
activeMay 14, '15 at 5:29p
posts1
users1
websitehive.apache.org

1 user in discussion

Ekoifman: 1 post

People

Translate

site design / logo © 2021 Grokbase