FAQ
Repository: hive
Updated Branches:
   refs/heads/master 114eabcbe -> d2c6b132f


HIVE-12506:SHOW CREATE TABLE command creates a table that does not work for RCFile format (Chaoyu Tang, reviewed by Jimmy Xiang)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/d2c6b132
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/d2c6b132
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/d2c6b132

Branch: refs/heads/master
Commit: d2c6b132f2fde6820a26c059612486fd37229c70
Parents: 114eabc
Author: ctang <ctang.ma@gmail.com>
Authored: Fri Dec 4 11:04:14 2015 -0500
Committer: ctang <ctang.ma@gmail.com>
Committed: Fri Dec 4 11:04:14 2015 -0500

----------------------------------------------------------------------
  .../hadoop/hive/metastore/MetaStoreUtils.java | 5 +-
  .../org/apache/hadoop/hive/ql/exec/DDLTask.java | 51 ++++----------------
  .../results/clientpositive/nullformat.q.out | 6 ++-
  .../results/clientpositive/nullformatCTAS.q.out | 6 ++-
  .../show_create_table_delimited.q.out | 13 +++--
  5 files changed, 28 insertions(+), 53 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/d2c6b132/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java b/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java
index 02cbd76..23068f8 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java
@@ -90,6 +90,7 @@ public class MetaStoreUtils {

    public static final String DEFAULT_DATABASE_NAME = "default";
    public static final String DEFAULT_DATABASE_COMMENT = "Default Hive database";
+ public static final String DEFAULT_SERIALIZATION_FORMAT = "1";

    public static final String DATABASE_WAREHOUSE_SUFFIX = ".db";

@@ -116,8 +117,8 @@ public class MetaStoreUtils {
      SerDeInfo serdeInfo = sd.getSerdeInfo();
      serdeInfo.setSerializationLib(LazySimpleSerDe.class.getName());
      serdeInfo.setParameters(new HashMap<String, String>());
- serdeInfo.getParameters().put(
- org.apache.hadoop.hive.serde.serdeConstants.SERIALIZATION_FORMAT, "1");
+ serdeInfo.getParameters().put(org.apache.hadoop.hive.serde.serdeConstants.SERIALIZATION_FORMAT,
+ DEFAULT_SERIALIZATION_FORMAT);

      List<FieldSchema> fields = new ArrayList<FieldSchema>();
      sd.setCols(fields);

http://git-wip-us.apache.org/repos/asf/hive/blob/d2c6b132/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
index 8762e12..1a9469a 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
@@ -2117,37 +2117,15 @@ public class DDLTask extends Task<DDLWork> implements Serializable {
        StringBuilder tbl_row_format = new StringBuilder();
        StorageDescriptor sd = tbl.getTTable().getSd();
        SerDeInfo serdeInfo = sd.getSerdeInfo();
- tbl_row_format.append("ROW FORMAT");
+ Map<String, String> serdeParams = serdeInfo.getParameters();
+ tbl_row_format.append("ROW FORMAT SERDE \n");
+ tbl_row_format.append(" '" + escapeHiveCommand(serdeInfo.getSerializationLib()) + "' \n");
        if (tbl.getStorageHandler() == null) {
- Map<String, String> serdeParams = serdeInfo.getParameters();
- String[] delimiters = new String[] {
- serdeParams.remove(serdeConstants.FIELD_DELIM),
- serdeParams.remove(serdeConstants.COLLECTION_DELIM),
- serdeParams.remove(serdeConstants.MAPKEY_DELIM),
- serdeParams.remove(serdeConstants.LINE_DELIM),
- serdeParams.remove(serdeConstants.SERIALIZATION_NULL_FORMAT)
- };
- serdeParams.remove(serdeConstants.SERIALIZATION_FORMAT);
- if (containsNonNull(delimiters)) {
- // There is a "serialization.format" property by default,
- // even with a delimited row format.
- // But our result will only cover the following four delimiters.
- tbl_row_format.append(" DELIMITED \n");
-
- // Warn:
- // If the four delimiters all exist in a CREATE TABLE query,
- // this following order needs to be strictly followed,
- // or the query will fail with a ParseException.
- for (int i = 0; i < DELIMITER_PREFIXES.length; i++) {
- if (delimiters[i] != null) {
- tbl_row_format.append(" ").append(DELIMITER_PREFIXES[i]).append(" '");
- tbl_row_format.append(escapeHiveCommand(StringEscapeUtils.escapeJava(delimiters[i])));
- tbl_row_format.append("' \n");
- }
- }
- } else {
- tbl_row_format.append(" SERDE \n '" +
- escapeHiveCommand(serdeInfo.getSerializationLib()) + "' \n");
+ // If serialization.format property has the default value, it will not to be included in
+ // SERDE properties
+ if (MetaStoreUtils.DEFAULT_SERIALIZATION_FORMAT.equals(serdeParams.get(
+ serdeConstants.SERIALIZATION_FORMAT))){
+ serdeParams.remove(serdeConstants.SERIALIZATION_FORMAT);
          }
          if (!serdeParams.isEmpty()) {
            appendSerdeParams(tbl_row_format, serdeParams).append(" \n");
@@ -2158,12 +2136,10 @@ public class DDLTask extends Task<DDLWork> implements Serializable {
              escapeHiveCommand(sd.getOutputFormat()) + "'");
        } else {
          duplicateProps.add(META_TABLE_STORAGE);
- tbl_row_format.append(" SERDE \n '" +
- escapeHiveCommand(serdeInfo.getSerializationLib()) + "' \n");
          tbl_row_format.append("STORED BY \n '" + escapeHiveCommand(tbl.getParameters().get(
              META_TABLE_STORAGE)) + "' \n");
          // SerDe Properties
- if (serdeInfo.getParametersSize() > 0) {
+ if (!serdeParams.isEmpty()) {
            appendSerdeParams(tbl_row_format, serdeInfo.getParameters());
          }
        }
@@ -2210,15 +2186,6 @@ public class DDLTask extends Task<DDLWork> implements Serializable {
      return prop_string;
    }

- private boolean containsNonNull(String[] values) {
- for (String value : values) {
- if (value != null) {
- return true;
- }
- }
- return false;
- }
-
    private StringBuilder appendSerdeParams(StringBuilder builder, Map<String, String> serdeParam) {
      serdeParam = new TreeMap<String, String>(serdeParam);
      builder.append("WITH SERDEPROPERTIES ( \n");

http://git-wip-us.apache.org/repos/asf/hive/blob/d2c6b132/ql/src/test/results/clientpositive/nullformat.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/nullformat.q.out b/ql/src/test/results/clientpositive/nullformat.q.out
index 6cfc2b8..af91470 100644
--- a/ql/src/test/results/clientpositive/nullformat.q.out
+++ b/ql/src/test/results/clientpositive/nullformat.q.out
@@ -82,8 +82,10 @@ POSTHOOK: Input: default@null_tab1
  CREATE TABLE `null_tab1`(
    `a` string,
    `b` string)
-ROW FORMAT DELIMITED
- NULL DEFINED AS 'fooNull'
+ROW FORMAT SERDE
+ 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe'
+WITH SERDEPROPERTIES (
+ 'serialization.null.format'='fooNull')
  STORED AS INPUTFORMAT
    'org.apache.hadoop.mapred.TextInputFormat'
  OUTPUTFORMAT

http://git-wip-us.apache.org/repos/asf/hive/blob/d2c6b132/ql/src/test/results/clientpositive/nullformatCTAS.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/nullformatCTAS.q.out b/ql/src/test/results/clientpositive/nullformatCTAS.q.out
index 7adca68..d6b22b1 100644
--- a/ql/src/test/results/clientpositive/nullformatCTAS.q.out
+++ b/ql/src/test/results/clientpositive/nullformatCTAS.q.out
@@ -163,8 +163,10 @@ POSTHOOK: Input: default@null_tab3
  CREATE TABLE `null_tab3`(
    `a` string,
    `b` string)
-ROW FORMAT DELIMITED
- NULL DEFINED AS 'fooNull'
+ROW FORMAT SERDE
+ 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe'
+WITH SERDEPROPERTIES (
+ 'serialization.null.format'='fooNull')
  STORED AS INPUTFORMAT
    'org.apache.hadoop.mapred.TextInputFormat'
  OUTPUTFORMAT

http://git-wip-us.apache.org/repos/asf/hive/blob/d2c6b132/ql/src/test/results/clientpositive/show_create_table_delimited.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/show_create_table_delimited.q.out b/ql/src/test/results/clientpositive/show_create_table_delimited.q.out
index 79f29de..e14f850 100644
--- a/ql/src/test/results/clientpositive/show_create_table_delimited.q.out
+++ b/ql/src/test/results/clientpositive/show_create_table_delimited.q.out
@@ -28,11 +28,14 @@ CREATE TABLE `tmp_showcrt1`(
    `key` int,
    `value` string,
    `newvalue` bigint)
-ROW FORMAT DELIMITED
- FIELDS TERMINATED BY ','
- COLLECTION ITEMS TERMINATED BY '|'
- MAP KEYS TERMINATED BY '%'
- LINES TERMINATED BY '\n'
+ROW FORMAT SERDE
+ 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe'
+WITH SERDEPROPERTIES (
+ 'colelction.delim'='|',
+ 'field.delim'=',',
+ 'line.delim'='\n',
+ 'mapkey.delim'='%',
+ 'serialization.format'=',')
  STORED AS INPUTFORMAT
    'org.apache.hadoop.mapred.TextInputFormat'
  OUTPUTFORMAT

Search Discussions

  • Ctang at Dec 4, 2015 at 7:39 pm
    Repository: hive
    Updated Branches:
       refs/heads/branch-1 e57d2db87 -> 09e7b19bf


    HIVE-12505:Insert overwrite in same encrypted zone silently fails to remove some existing files (Chaoyu Tang, reviewed by Aihua Xu)


    Project: http://git-wip-us.apache.org/repos/asf/hive/repo
    Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/44eab8e5
    Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/44eab8e5
    Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/44eab8e5

    Branch: refs/heads/branch-1
    Commit: 44eab8e5bfe822b3d2eb74433aed117aa31b166b
    Parents: e57d2db
    Author: ctang <ctang.ma@gmail.com>
    Authored: Fri Dec 4 10:46:23 2015 -0500
    Committer: ctang <ctang.ma@gmail.com>
    Committed: Fri Dec 4 13:44:04 2015 -0500

    ----------------------------------------------------------------------
      .../apache/hadoop/hive/common/FileUtils.java | 66 ++++++++--
      .../test/resources/testconfiguration.properties | 3 +-
      .../apache/hadoop/hive/ql/metadata/Hive.java | 26 +++-
      .../clientpositive/encryption_with_trash.q | 33 +++++
      .../encrypted/encryption_with_trash.q.out | 122 +++++++++++++++++++
      5 files changed, 234 insertions(+), 16 deletions(-)
    ----------------------------------------------------------------------


    http://git-wip-us.apache.org/repos/asf/hive/blob/44eab8e5/common/src/java/org/apache/hadoop/hive/common/FileUtils.java
    ----------------------------------------------------------------------
    diff --git a/common/src/java/org/apache/hadoop/hive/common/FileUtils.java b/common/src/java/org/apache/hadoop/hive/common/FileUtils.java
    index d781f08..28f67b3 100644
    --- a/common/src/java/org/apache/hadoop/hive/common/FileUtils.java
    +++ b/common/src/java/org/apache/hadoop/hive/common/FileUtils.java
    @@ -572,7 +572,7 @@ public final class FileUtils {
        }

        /**
    - * Deletes all files under a directory, sending them to the trash. Leaves the directory as is.
    + * Trashes or deletes all files under a directory. Leaves the directory as is.
         * @param fs FileSystem to use
         * @param f path of directory
         * @param conf hive configuration
    @@ -580,17 +580,34 @@ public final class FileUtils {
         * @throws FileNotFoundException
         * @throws IOException
         */
    - public static boolean trashFilesUnderDir(FileSystem fs, Path f, Configuration conf) throws FileNotFoundException, IOException {
    + public static boolean trashFilesUnderDir(FileSystem fs, Path f, Configuration conf)
    + throws FileNotFoundException, IOException {
    + return trashFilesUnderDir(fs, f, conf, true);
    + }
    +
    + /**
    + * Trashes or deletes all files under a directory. Leaves the directory as is.
    + * @param fs FileSystem to use
    + * @param f path of directory
    + * @param conf hive configuration
    + * @param forceDelete whether to force delete files if trashing does not succeed
    + * @return true if deletion successful
    + * @throws FileNotFoundException
    + * @throws IOException
    + */
    + public static boolean trashFilesUnderDir(FileSystem fs, Path f, Configuration conf,
    + boolean forceDelete) throws FileNotFoundException, IOException {
          FileStatus[] statuses = fs.listStatus(f, HIDDEN_FILES_PATH_FILTER);
          boolean result = true;
          for (FileStatus status : statuses) {
    - result = result & moveToTrash(fs, status.getPath(), conf);
    + result = result & moveToTrash(fs, status.getPath(), conf, forceDelete);
          }
          return result;
        }

        /**
    - * Move a particular file or directory to the trash.
    + * Move a particular file or directory to the trash. If for a certain reason the trashing fails
    + * it will force deletes the file or directory
         * @param fs FileSystem to use
         * @param f path of file or directory to move to trash.
         * @param conf
    @@ -598,18 +615,47 @@ public final class FileUtils {
         * @throws IOException
         */
        public static boolean moveToTrash(FileSystem fs, Path f, Configuration conf) throws IOException {
    + return moveToTrash(fs, f, conf, true);
    + }
    +
    + /**
    + * Move a particular file or directory to the trash.
    + * @param fs FileSystem to use
    + * @param f path of file or directory to move to trash.
    + * @param conf
    + * @param forceDelete whether force delete the file or directory if trashing fails
    + * @return true if move successful
    + * @throws IOException
    + */
    + public static boolean moveToTrash(FileSystem fs, Path f, Configuration conf, boolean forceDelete)
    + throws IOException {
          LOG.info("deleting " + f);
          HadoopShims hadoopShim = ShimLoader.getHadoopShims();

    - if (hadoopShim.moveToAppropriateTrash(fs, f, conf)) {
    - LOG.info("Moved to trash: " + f);
    - return true;
    + boolean result = false;
    + try {
    + result = hadoopShim.moveToAppropriateTrash(fs, f, conf);
    + if (result) {
    + LOG.info("Moved to trash: " + f);
    + return true;
    + }
    + } catch (IOException ioe) {
    + if (forceDelete) {
    + // for whatever failure reason including that trash has lower encryption zone
    + // retry with force delete
    + LOG.warn(ioe.getMessage() + "; Force to delete it.");
    + } else {
    + throw ioe;
    + }
          }

    - boolean result = fs.delete(f, true);
    - if (!result) {
    - LOG.error("Failed to delete " + f);
    + if (forceDelete) {
    + result = fs.delete(f, true);
    + if (!result) {
    + LOG.error("Failed to delete " + f);
    + }
          }
    +
          return result;
        }


    http://git-wip-us.apache.org/repos/asf/hive/blob/44eab8e5/itests/src/test/resources/testconfiguration.properties
    ----------------------------------------------------------------------
    diff --git a/itests/src/test/resources/testconfiguration.properties b/itests/src/test/resources/testconfiguration.properties
    index 660fec9..8421106 100644
    --- a/itests/src/test/resources/testconfiguration.properties
    +++ b/itests/src/test/resources/testconfiguration.properties
    @@ -382,7 +382,8 @@ encrypted.query.files=encryption_join_unencrypted_tbl.q,\
        encryption_drop_table.q \
        encryption_insert_values.q \
        encryption_drop_view.q \
    - encryption_drop_partition.q
    + encryption_drop_partition.q \
    + encryption_with_trash.q

      beeline.positive.exclude=add_part_exist.q,\
        alter1.q,\

    http://git-wip-us.apache.org/repos/asf/hive/blob/44eab8e5/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
    ----------------------------------------------------------------------
    diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
    index b5bba9f..c72b3ec 100644
    --- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
    +++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
    @@ -2871,22 +2871,38 @@ private void constructOneLBLocationMap(FileStatus fSta,
                true);

            if (oldPath != null) {
    + boolean oldPathDeleted = false;
    + boolean isOldPathUnderDestf = false;
              try {
                FileSystem fs2 = oldPath.getFileSystem(conf);
                if (fs2.exists(oldPath)) {
                  // Do not delete oldPath if:
                  // - destf is subdir of oldPath
                  //if ( !(fs2.equals(destf.getFileSystem(conf)) && FileUtils.isSubDir(oldPath, destf, fs2)))
    - if (FileUtils.isSubDir(oldPath, destf, fs2)) {
    - FileUtils.trashFilesUnderDir(fs2, oldPath, conf);
    + isOldPathUnderDestf = FileUtils.isSubDir(oldPath, destf, fs2);
    + if (isOldPathUnderDestf) {
    + // if oldPath is destf or its subdir, its should definitely be deleted, otherwise its
    + // existing content might result in incorrect (extra) data.
    + // But not sure why we changed not to delete the oldPath in HIVE-8750 if it is
    + // not the destf or its subdir?
    + oldPathDeleted = FileUtils.trashFilesUnderDir(fs2, oldPath, conf);
                  }
                  if (inheritPerms) {
                    inheritFromTable(tablePath, destf, conf, destFs);
                  }
                }
    - } catch (Exception e) {
    - //swallow the exception
    - LOG.warn("Directory " + oldPath.toString() + " cannot be removed: " + e, e);
    + } catch (IOException e) {
    + if (isOldPathUnderDestf) {
    + // if oldPath is a subdir of destf but it could not be cleaned
    + throw new HiveException("Directory " + oldPath.toString()
    + + " could not be cleaned up.", e);
    + } else {
    + //swallow the exception since it won't affect the final result
    + LOG.warn("Directory " + oldPath.toString() + " cannot be cleaned: " + e, e);
    + }
    + }
    + if (isOldPathUnderDestf && !oldPathDeleted) {
    + throw new HiveException("Destination directory " + destf + " has not be cleaned up.");
              }
            }


    http://git-wip-us.apache.org/repos/asf/hive/blob/44eab8e5/ql/src/test/queries/clientpositive/encryption_with_trash.q
    ----------------------------------------------------------------------
    diff --git a/ql/src/test/queries/clientpositive/encryption_with_trash.q b/ql/src/test/queries/clientpositive/encryption_with_trash.q
    new file mode 100644
    index 0000000..8f8789a
    --- /dev/null
    +++ b/ql/src/test/queries/clientpositive/encryption_with_trash.q
    @@ -0,0 +1,33 @@
    +set fs.trash.interval=5
    +
    +-- SORT_QUERY_RESULTS
    +
    +-- init
    +drop table IF EXISTS encryptedTableSrc PURGE;
    +drop table IF EXISTS unencryptedTable PURGE;
    +
    +create table encryptedTableSrc(key string, value string)
    +LOCATION '${hiveconf:hive.metastore.warehouse.dir}/encryptedTableSrc';
    +
    +create table encryptedTable(key string, value string) partitioned by (ds string)
    + LOCATION '${hiveconf:hive.metastore.warehouse.dir}/encryptedTable';
    +CRYPTO CREATE_KEY --keyName key_1 --bitLength 128;
    +CRYPTO CREATE_ZONE --keyName key_1 --path ${hiveconf:hive.metastore.warehouse.dir}/encryptedTableSrc;
    +CRYPTO CREATE_ZONE --keyName key_1 --path ${hiveconf:hive.metastore.warehouse.dir}/encryptedTable;
    +
    +-- insert src table from values
    +insert into table encryptedTableSrc values ('501', 'val_501'), ('502', 'val_502');
    +
    +insert into table encryptedTable partition (ds='today') select key, value from encryptedTableSrc;
    +select count(*) from encryptedTable where ds='today';
    +insert into table encryptedTable partition (ds='today') select key, value from encryptedTableSrc;
    +select count(*) from encryptedTable where ds='today';
    +
    +insert overwrite table encryptedTable partition (ds='today') select key, value from encryptedTableSrc;
    +select count(*) from encryptedTable where ds='today';
    +
    +-- clean up
    +drop table encryptedTable PURGE;
    +drop table unencryptedTable PURGE;
    +CRYPTO DELETE_KEY --keyName key_1;
    +set fs.trash.interval=0

    http://git-wip-us.apache.org/repos/asf/hive/blob/44eab8e5/ql/src/test/results/clientpositive/encrypted/encryption_with_trash.q.out
    ----------------------------------------------------------------------
    diff --git a/ql/src/test/results/clientpositive/encrypted/encryption_with_trash.q.out b/ql/src/test/results/clientpositive/encrypted/encryption_with_trash.q.out
    new file mode 100644
    index 0000000..3d1f75f
    --- /dev/null
    +++ b/ql/src/test/results/clientpositive/encrypted/encryption_with_trash.q.out
    @@ -0,0 +1,122 @@
    +Warning: Value had a \n character in it.
    +PREHOOK: query: drop table IF EXISTS unencryptedTable PURGE
    +PREHOOK: type: DROPTABLE
    +POSTHOOK: query: drop table IF EXISTS unencryptedTable PURGE
    +POSTHOOK: type: DROPTABLE
    +PREHOOK: query: create table encryptedTableSrc(key string, value string)
    +#### A masked pattern was here ####
    +PREHOOK: type: CREATETABLE
    +#### A masked pattern was here ####
    +PREHOOK: Output: database:default
    +PREHOOK: Output: default@encryptedTableSrc
    +POSTHOOK: query: create table encryptedTableSrc(key string, value string)
    +#### A masked pattern was here ####
    +POSTHOOK: type: CREATETABLE
    +#### A masked pattern was here ####
    +POSTHOOK: Output: database:default
    +POSTHOOK: Output: default@encryptedTableSrc
    +PREHOOK: query: create table encryptedTable(key string, value string) partitioned by (ds string)
    +#### A masked pattern was here ####
    +PREHOOK: type: CREATETABLE
    +#### A masked pattern was here ####
    +PREHOOK: Output: database:default
    +PREHOOK: Output: default@encryptedTable
    +POSTHOOK: query: create table encryptedTable(key string, value string) partitioned by (ds string)
    +#### A masked pattern was here ####
    +POSTHOOK: type: CREATETABLE
    +#### A masked pattern was here ####
    +POSTHOOK: Output: database:default
    +POSTHOOK: Output: default@encryptedTable
    +Encryption key created: 'key_1'
    +Encryption zone created: '/build/ql/test/data/warehouse/encryptedTableSrc' using key: 'key_1'
    +Encryption zone created: '/build/ql/test/data/warehouse/encryptedTable' using key: 'key_1'
    +PREHOOK: query: -- insert src table from values
    +insert into table encryptedTableSrc values ('501', 'val_501'), ('502', 'val_502')
    +PREHOOK: type: QUERY
    +PREHOOK: Input: default@values__tmp__table__1
    +PREHOOK: Output: default@encryptedtablesrc
    +POSTHOOK: query: -- insert src table from values
    +insert into table encryptedTableSrc values ('501', 'val_501'), ('502', 'val_502')
    +POSTHOOK: type: QUERY
    +POSTHOOK: Input: default@values__tmp__table__1
    +POSTHOOK: Output: default@encryptedtablesrc
    +POSTHOOK: Lineage: encryptedtablesrc.key SIMPLE [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col1, type:string, comment:), ]
    +POSTHOOK: Lineage: encryptedtablesrc.value SIMPLE [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col2, type:string, comment:), ]
    +PREHOOK: query: insert into table encryptedTable partition (ds='today') select key, value from encryptedTableSrc
    +PREHOOK: type: QUERY
    +PREHOOK: Input: default@encryptedtablesrc
    +PREHOOK: Output: default@encryptedtable@ds=today
    +POSTHOOK: query: insert into table encryptedTable partition (ds='today') select key, value from encryptedTableSrc
    +POSTHOOK: type: QUERY
    +POSTHOOK: Input: default@encryptedtablesrc
    +POSTHOOK: Output: default@encryptedtable@ds=today
    +POSTHOOK: Lineage: encryptedtable PARTITION(ds=today).key SIMPLE [(encryptedtablesrc)encryptedtablesrc.FieldSchema(name:key, type:string, comment:null), ]
    +POSTHOOK: Lineage: encryptedtable PARTITION(ds=today).value SIMPLE [(encryptedtablesrc)encryptedtablesrc.FieldSchema(name:value, type:string, comment:null), ]
    +PREHOOK: query: select count(*) from encryptedTable where ds='today'
    +PREHOOK: type: QUERY
    +PREHOOK: Input: default@encryptedtable
    +PREHOOK: Input: default@encryptedtable@ds=today
    +#### A PARTIAL masked pattern was here #### data/warehouse/encryptedTable/.hive-staging
    +POSTHOOK: query: select count(*) from encryptedTable where ds='today'
    +POSTHOOK: type: QUERY
    +POSTHOOK: Input: default@encryptedtable
    +POSTHOOK: Input: default@encryptedtable@ds=today
    +#### A PARTIAL masked pattern was here #### data/warehouse/encryptedTable/.hive-staging
    +2
    +PREHOOK: query: insert into table encryptedTable partition (ds='today') select key, value from encryptedTableSrc
    +PREHOOK: type: QUERY
    +PREHOOK: Input: default@encryptedtablesrc
    +PREHOOK: Output: default@encryptedtable@ds=today
    +POSTHOOK: query: insert into table encryptedTable partition (ds='today') select key, value from encryptedTableSrc
    +POSTHOOK: type: QUERY
    +POSTHOOK: Input: default@encryptedtablesrc
    +POSTHOOK: Output: default@encryptedtable@ds=today
    +POSTHOOK: Lineage: encryptedtable PARTITION(ds=today).key SIMPLE [(encryptedtablesrc)encryptedtablesrc.FieldSchema(name:key, type:string, comment:null), ]
    +POSTHOOK: Lineage: encryptedtable PARTITION(ds=today).value SIMPLE [(encryptedtablesrc)encryptedtablesrc.FieldSchema(name:value, type:string, comment:null), ]
    +PREHOOK: query: select count(*) from encryptedTable where ds='today'
    +PREHOOK: type: QUERY
    +PREHOOK: Input: default@encryptedtable
    +PREHOOK: Input: default@encryptedtable@ds=today
    +#### A PARTIAL masked pattern was here #### data/warehouse/encryptedTable/.hive-staging
    +POSTHOOK: query: select count(*) from encryptedTable where ds='today'
    +POSTHOOK: type: QUERY
    +POSTHOOK: Input: default@encryptedtable
    +POSTHOOK: Input: default@encryptedtable@ds=today
    +#### A PARTIAL masked pattern was here #### data/warehouse/encryptedTable/.hive-staging
    +4
    +PREHOOK: query: insert overwrite table encryptedTable partition (ds='today') select key, value from encryptedTableSrc
    +PREHOOK: type: QUERY
    +PREHOOK: Input: default@encryptedtablesrc
    +PREHOOK: Output: default@encryptedtable@ds=today
    +POSTHOOK: query: insert overwrite table encryptedTable partition (ds='today') select key, value from encryptedTableSrc
    +POSTHOOK: type: QUERY
    +POSTHOOK: Input: default@encryptedtablesrc
    +POSTHOOK: Output: default@encryptedtable@ds=today
    +POSTHOOK: Lineage: encryptedtable PARTITION(ds=today).key SIMPLE [(encryptedtablesrc)encryptedtablesrc.FieldSchema(name:key, type:string, comment:null), ]
    +POSTHOOK: Lineage: encryptedtable PARTITION(ds=today).value SIMPLE [(encryptedtablesrc)encryptedtablesrc.FieldSchema(name:value, type:string, comment:null), ]
    +PREHOOK: query: select count(*) from encryptedTable where ds='today'
    +PREHOOK: type: QUERY
    +PREHOOK: Input: default@encryptedtable
    +PREHOOK: Input: default@encryptedtable@ds=today
    +#### A PARTIAL masked pattern was here #### data/warehouse/encryptedTable/.hive-staging
    +POSTHOOK: query: select count(*) from encryptedTable where ds='today'
    +POSTHOOK: type: QUERY
    +POSTHOOK: Input: default@encryptedtable
    +POSTHOOK: Input: default@encryptedtable@ds=today
    +#### A PARTIAL masked pattern was here #### data/warehouse/encryptedTable/.hive-staging
    +2
    +PREHOOK: query: -- clean up
    +drop table encryptedTable PURGE
    +PREHOOK: type: DROPTABLE
    +PREHOOK: Input: default@encryptedtable
    +PREHOOK: Output: default@encryptedtable
    +POSTHOOK: query: -- clean up
    +drop table encryptedTable PURGE
    +POSTHOOK: type: DROPTABLE
    +POSTHOOK: Input: default@encryptedtable
    +POSTHOOK: Output: default@encryptedtable
    +PREHOOK: query: drop table unencryptedTable PURGE
    +PREHOOK: type: DROPTABLE
    +POSTHOOK: query: drop table unencryptedTable PURGE
    +POSTHOOK: type: DROPTABLE
    +Encryption key deleted: 'key_1'
  • Ctang at Dec 4, 2015 at 7:39 pm
    HIVE-12506:SHOW CREATE TABLE command creates a table that does not work for RCFile format (Chaoyu Tang, reviewed by Jimmy Xiang)


    Project: http://git-wip-us.apache.org/repos/asf/hive/repo
    Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/09e7b19b
    Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/09e7b19b
    Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/09e7b19b

    Branch: refs/heads/branch-1
    Commit: 09e7b19bf0fc629291d1da5aea9f0c63baa4d952
    Parents: 44eab8e
    Author: ctang <ctang.ma@gmail.com>
    Authored: Fri Dec 4 11:04:14 2015 -0500
    Committer: ctang <ctang.ma@gmail.com>
    Committed: Fri Dec 4 14:37:12 2015 -0500

    ----------------------------------------------------------------------
      .../hadoop/hive/metastore/MetaStoreUtils.java | 5 +-
      .../org/apache/hadoop/hive/ql/exec/DDLTask.java | 51 ++++----------------
      .../results/clientpositive/nullformat.q.out | 6 ++-
      .../results/clientpositive/nullformatCTAS.q.out | 6 ++-
      .../show_create_table_delimited.q.out | 13 +++--
      5 files changed, 28 insertions(+), 53 deletions(-)
    ----------------------------------------------------------------------


    http://git-wip-us.apache.org/repos/asf/hive/blob/09e7b19b/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java
    ----------------------------------------------------------------------
    diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java b/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java
    index 425549b..f5153b3 100644
    --- a/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java
    +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java
    @@ -90,6 +90,7 @@ public class MetaStoreUtils {

        public static final String DEFAULT_DATABASE_NAME = "default";
        public static final String DEFAULT_DATABASE_COMMENT = "Default Hive database";
    + public static final String DEFAULT_SERIALIZATION_FORMAT = "1";

        public static final String DATABASE_WAREHOUSE_SUFFIX = ".db";

    @@ -108,8 +109,8 @@ public class MetaStoreUtils {
          SerDeInfo serdeInfo = sd.getSerdeInfo();
          serdeInfo.setSerializationLib(LazySimpleSerDe.class.getName());
          serdeInfo.setParameters(new HashMap<String, String>());
    - serdeInfo.getParameters().put(
    - org.apache.hadoop.hive.serde.serdeConstants.SERIALIZATION_FORMAT, "1");
    + serdeInfo.getParameters().put(org.apache.hadoop.hive.serde.serdeConstants.SERIALIZATION_FORMAT,
    + DEFAULT_SERIALIZATION_FORMAT);

          List<FieldSchema> fields = new ArrayList<FieldSchema>();
          sd.setCols(fields);

    http://git-wip-us.apache.org/repos/asf/hive/blob/09e7b19b/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
    ----------------------------------------------------------------------
    diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
    index 78c7872..c3693de 100644
    --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
    +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
    @@ -2065,37 +2065,15 @@ public class DDLTask extends Task<DDLWork> implements Serializable {
            StringBuilder tbl_row_format = new StringBuilder();
            StorageDescriptor sd = tbl.getTTable().getSd();
            SerDeInfo serdeInfo = sd.getSerdeInfo();
    - tbl_row_format.append("ROW FORMAT");
    + Map<String, String> serdeParams = serdeInfo.getParameters();
    + tbl_row_format.append("ROW FORMAT SERDE \n");
    + tbl_row_format.append(" '" + escapeHiveCommand(serdeInfo.getSerializationLib()) + "' \n");
            if (tbl.getStorageHandler() == null) {
    - Map<String, String> serdeParams = serdeInfo.getParameters();
    - String[] delimiters = new String[] {
    - serdeParams.remove(serdeConstants.FIELD_DELIM),
    - serdeParams.remove(serdeConstants.COLLECTION_DELIM),
    - serdeParams.remove(serdeConstants.MAPKEY_DELIM),
    - serdeParams.remove(serdeConstants.LINE_DELIM),
    - serdeParams.remove(serdeConstants.SERIALIZATION_NULL_FORMAT)
    - };
    - serdeParams.remove(serdeConstants.SERIALIZATION_FORMAT);
    - if (containsNonNull(delimiters)) {
    - // There is a "serialization.format" property by default,
    - // even with a delimited row format.
    - // But our result will only cover the following four delimiters.
    - tbl_row_format.append(" DELIMITED \n");
    -
    - // Warn:
    - // If the four delimiters all exist in a CREATE TABLE query,
    - // this following order needs to be strictly followed,
    - // or the query will fail with a ParseException.
    - for (int i = 0; i < DELIMITER_PREFIXES.length; i++) {
    - if (delimiters[i] != null) {
    - tbl_row_format.append(" ").append(DELIMITER_PREFIXES[i]).append(" '");
    - tbl_row_format.append(escapeHiveCommand(StringEscapeUtils.escapeJava(delimiters[i])));
    - tbl_row_format.append("' \n");
    - }
    - }
    - } else {
    - tbl_row_format.append(" SERDE \n '" +
    - escapeHiveCommand(serdeInfo.getSerializationLib()) + "' \n");
    + // If serialization.format property has the default value, it will not to be included in
    + // SERDE properties
    + if (MetaStoreUtils.DEFAULT_SERIALIZATION_FORMAT.equals(serdeParams.get(
    + serdeConstants.SERIALIZATION_FORMAT))){
    + serdeParams.remove(serdeConstants.SERIALIZATION_FORMAT);
              }
              if (!serdeParams.isEmpty()) {
                appendSerdeParams(tbl_row_format, serdeParams).append(" \n");
    @@ -2106,12 +2084,10 @@ public class DDLTask extends Task<DDLWork> implements Serializable {
                  escapeHiveCommand(sd.getOutputFormat()) + "'");
            } else {
              duplicateProps.add(META_TABLE_STORAGE);
    - tbl_row_format.append(" SERDE \n '" +
    - escapeHiveCommand(serdeInfo.getSerializationLib()) + "' \n");
              tbl_row_format.append("STORED BY \n '" + escapeHiveCommand(tbl.getParameters().get(
                  META_TABLE_STORAGE)) + "' \n");
              // SerDe Properties
    - if (serdeInfo.getParametersSize() > 0) {
    + if (!serdeParams.isEmpty()) {
                appendSerdeParams(tbl_row_format, serdeInfo.getParameters());
              }
            }
    @@ -2162,15 +2138,6 @@ public class DDLTask extends Task<DDLWork> implements Serializable {
          return 0;
        }

    - private boolean containsNonNull(String[] values) {
    - for (String value : values) {
    - if (value != null) {
    - return true;
    - }
    - }
    - return false;
    - }
    -
        private StringBuilder appendSerdeParams(StringBuilder builder, Map<String, String> serdeParam) {
          serdeParam = new TreeMap<String, String>(serdeParam);
          builder.append("WITH SERDEPROPERTIES ( \n");

    http://git-wip-us.apache.org/repos/asf/hive/blob/09e7b19b/ql/src/test/results/clientpositive/nullformat.q.out
    ----------------------------------------------------------------------
    diff --git a/ql/src/test/results/clientpositive/nullformat.q.out b/ql/src/test/results/clientpositive/nullformat.q.out
    index 6cfc2b8..af91470 100644
    --- a/ql/src/test/results/clientpositive/nullformat.q.out
    +++ b/ql/src/test/results/clientpositive/nullformat.q.out
    @@ -82,8 +82,10 @@ POSTHOOK: Input: default@null_tab1
      CREATE TABLE `null_tab1`(
        `a` string,
        `b` string)
    -ROW FORMAT DELIMITED
    - NULL DEFINED AS 'fooNull'
    +ROW FORMAT SERDE
    + 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe'
    +WITH SERDEPROPERTIES (
    + 'serialization.null.format'='fooNull')
      STORED AS INPUTFORMAT
        'org.apache.hadoop.mapred.TextInputFormat'
      OUTPUTFORMAT

    http://git-wip-us.apache.org/repos/asf/hive/blob/09e7b19b/ql/src/test/results/clientpositive/nullformatCTAS.q.out
    ----------------------------------------------------------------------
    diff --git a/ql/src/test/results/clientpositive/nullformatCTAS.q.out b/ql/src/test/results/clientpositive/nullformatCTAS.q.out
    index 7adca68..d6b22b1 100644
    --- a/ql/src/test/results/clientpositive/nullformatCTAS.q.out
    +++ b/ql/src/test/results/clientpositive/nullformatCTAS.q.out
    @@ -163,8 +163,10 @@ POSTHOOK: Input: default@null_tab3
      CREATE TABLE `null_tab3`(
        `a` string,
        `b` string)
    -ROW FORMAT DELIMITED
    - NULL DEFINED AS 'fooNull'
    +ROW FORMAT SERDE
    + 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe'
    +WITH SERDEPROPERTIES (
    + 'serialization.null.format'='fooNull')
      STORED AS INPUTFORMAT
        'org.apache.hadoop.mapred.TextInputFormat'
      OUTPUTFORMAT

    http://git-wip-us.apache.org/repos/asf/hive/blob/09e7b19b/ql/src/test/results/clientpositive/show_create_table_delimited.q.out
    ----------------------------------------------------------------------
    diff --git a/ql/src/test/results/clientpositive/show_create_table_delimited.q.out b/ql/src/test/results/clientpositive/show_create_table_delimited.q.out
    index 79f29de..e14f850 100644
    --- a/ql/src/test/results/clientpositive/show_create_table_delimited.q.out
    +++ b/ql/src/test/results/clientpositive/show_create_table_delimited.q.out
    @@ -28,11 +28,14 @@ CREATE TABLE `tmp_showcrt1`(
        `key` int,
        `value` string,
        `newvalue` bigint)
    -ROW FORMAT DELIMITED
    - FIELDS TERMINATED BY ','
    - COLLECTION ITEMS TERMINATED BY '|'
    - MAP KEYS TERMINATED BY '%'
    - LINES TERMINATED BY '\n'
    +ROW FORMAT SERDE
    + 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe'
    +WITH SERDEPROPERTIES (
    + 'colelction.delim'='|',
    + 'field.delim'=',',
    + 'line.delim'='\n',
    + 'mapkey.delim'='%',
    + 'serialization.format'=',')
      STORED AS INPUTFORMAT
        'org.apache.hadoop.mapred.TextInputFormat'
      OUTPUTFORMAT

Related Discussions

Discussion Navigation
viewthread | post
Discussion Overview
groupcommits @
categorieshive, hadoop
postedDec 4, '15 at 4:05p
activeDec 4, '15 at 7:39p
posts3
users1
websitehive.apache.org

1 user in discussion

Ctang: 3 posts

People

Translate

site design / logo © 2021 Grokbase