FAQ
Repository: hive
Updated Branches:
   refs/heads/master 65e9fcf05 -> 8121b9ab6


HIVE-11194 - Exchange partition on external tables should fail with error message when target folder already exists (Aihua Xu, reviewd by Chao Sun)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/8121b9ab
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/8121b9ab
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/8121b9ab

Branch: refs/heads/master
Commit: 8121b9ab644cbe477df477827dd82a9859a7791b
Parents: 65e9fcf
Author: Chao Sun <sunchao@apache.org>
Authored: Mon Jul 13 09:36:22 2015 -0700
Committer: Chao Sun <sunchao@apache.org>
Committed: Mon Jul 13 09:36:22 2015 -0700

----------------------------------------------------------------------
  .../apache/hadoop/hive/common/FileUtils.java | 9 +++-
  .../hadoop/hive/metastore/HiveMetaStore.java | 2 +-
  .../queries/clientnegative/exchange_partition.q | 19 +++++++
  .../clientnegative/exchange_partition.q.out | 54 ++++++++++++++++++++
  4 files changed, 82 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/8121b9ab/common/src/java/org/apache/hadoop/hive/common/FileUtils.java
----------------------------------------------------------------------
diff --git a/common/src/java/org/apache/hadoop/hive/common/FileUtils.java b/common/src/java/org/apache/hadoop/hive/common/FileUtils.java
index c2c54bc..7e4f386 100644
--- a/common/src/java/org/apache/hadoop/hive/common/FileUtils.java
+++ b/common/src/java/org/apache/hadoop/hive/common/FileUtils.java
@@ -30,7 +30,6 @@ import java.util.List;
  import org.apache.commons.logging.Log;
  import org.apache.commons.logging.LogFactory;
  import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.DefaultFileAccess;
  import org.apache.hadoop.fs.FileStatus;
  import org.apache.hadoop.fs.FileSystem;
  import org.apache.hadoop.fs.FileUtil;
@@ -635,6 +634,14 @@ public final class FileUtils {
                                 Path destPath, boolean inheritPerms,
                                 Configuration conf) throws IOException {
      LOG.info("Renaming " + sourcePath + " to " + destPath);
+
+ // If destPath directory exists, rename call will move the sourcePath
+ // into destPath without failing. So check it before renaming.
+ if (fs.exists(destPath)) {
+ throw new IOException("Cannot rename the source path. The destination "
+ + "path already exists.");
+ }
+
      if (!inheritPerms) {
        //just rename the directory
        return fs.rename(sourcePath, destPath);

http://git-wip-us.apache.org/repos/asf/hive/blob/8121b9ab/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
index 4c9cd79..920e762 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
@@ -2581,7 +2581,7 @@ public class HiveMetaStore extends ThriftHiveMetastore {
          pathCreated = wh.renameDir(sourcePath, destPath);
          success = ms.commitTransaction();
        } finally {
- if (!success) {
+ if (!success || !pathCreated) {
            ms.rollbackTransaction();
            if (pathCreated) {
              wh.renameDir(destPath, sourcePath);

http://git-wip-us.apache.org/repos/asf/hive/blob/8121b9ab/ql/src/test/queries/clientnegative/exchange_partition.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientnegative/exchange_partition.q b/ql/src/test/queries/clientnegative/exchange_partition.q
new file mode 100644
index 0000000..7dc4f57
--- /dev/null
+++ b/ql/src/test/queries/clientnegative/exchange_partition.q
@@ -0,0 +1,19 @@
+dfs ${system:test.dfs.mkdir} ${system:test.tmp.dir}/ex_table1;
+dfs ${system:test.dfs.mkdir} ${system:test.tmp.dir}/ex_table1/part=part1;
+CREATE EXTERNAL TABLE ex_table1 ( key INT, value STRING)
+ PARTITIONED BY (part STRING)
+ STORED AS textfile
+ LOCATION 'file:${system:test.tmp.dir}/ex_table1';
+
+dfs ${system:test.dfs.mkdir} ${system:test.tmp.dir}/ex_table2;
+
+CREATE EXTERNAL TABLE ex_table2 ( key INT, value STRING)
+ PARTITIONED BY (part STRING)
+ STORED AS textfile
+ LOCATION 'file:${system:test.tmp.dir}/ex_table2';
+
+INSERT OVERWRITE TABLE ex_table2 PARTITION (part='part1')
+SELECT key, value FROM src WHERE key < 10;
+SHOW PARTITIONS ex_table2;
+
+ALTER TABLE ex_table1 EXCHANGE PARTITION (part='part1') WITH TABLE ex_table2;

http://git-wip-us.apache.org/repos/asf/hive/blob/8121b9ab/ql/src/test/results/clientnegative/exchange_partition.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientnegative/exchange_partition.q.out b/ql/src/test/results/clientnegative/exchange_partition.q.out
new file mode 100644
index 0000000..b81fb99
--- /dev/null
+++ b/ql/src/test/results/clientnegative/exchange_partition.q.out
@@ -0,0 +1,54 @@
+PREHOOK: query: CREATE EXTERNAL TABLE ex_table1 ( key INT, value STRING)
+ PARTITIONED BY (part STRING)
+ STORED AS textfile
+#### A masked pattern was here ####
+PREHOOK: type: CREATETABLE
+#### A masked pattern was here ####
+PREHOOK: Output: database:default
+PREHOOK: Output: default@ex_table1
+POSTHOOK: query: CREATE EXTERNAL TABLE ex_table1 ( key INT, value STRING)
+ PARTITIONED BY (part STRING)
+ STORED AS textfile
+#### A masked pattern was here ####
+POSTHOOK: type: CREATETABLE
+#### A masked pattern was here ####
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@ex_table1
+PREHOOK: query: CREATE EXTERNAL TABLE ex_table2 ( key INT, value STRING)
+ PARTITIONED BY (part STRING)
+ STORED AS textfile
+#### A masked pattern was here ####
+PREHOOK: type: CREATETABLE
+#### A masked pattern was here ####
+PREHOOK: Output: database:default
+PREHOOK: Output: default@ex_table2
+POSTHOOK: query: CREATE EXTERNAL TABLE ex_table2 ( key INT, value STRING)
+ PARTITIONED BY (part STRING)
+ STORED AS textfile
+#### A masked pattern was here ####
+POSTHOOK: type: CREATETABLE
+#### A masked pattern was here ####
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@ex_table2
+PREHOOK: query: INSERT OVERWRITE TABLE ex_table2 PARTITION (part='part1')
+SELECT key, value FROM src WHERE key < 10
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Output: default@ex_table2@part=part1
+POSTHOOK: query: INSERT OVERWRITE TABLE ex_table2 PARTITION (part='part1')
+SELECT key, value FROM src WHERE key < 10
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Output: default@ex_table2@part=part1
+POSTHOOK: Lineage: ex_table2 PARTITION(part=part1).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: ex_table2 PARTITION(part=part1).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+PREHOOK: query: SHOW PARTITIONS ex_table2
+PREHOOK: type: SHOWPARTITIONS
+PREHOOK: Input: default@ex_table2
+POSTHOOK: query: SHOW PARTITIONS ex_table2
+POSTHOOK: type: SHOWPARTITIONS
+POSTHOOK: Input: default@ex_table2
+part=part1
+PREHOOK: query: ALTER TABLE ex_table1 EXCHANGE PARTITION (part='part1') WITH TABLE ex_table2
+PREHOOK: type: null
+FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.exec.DDLTask. MetaException(message:Got exception: java.io.IOException Cannot rename the source path. The destination path already exists.)

Search Discussions

  • Sunchao at Jul 13, 2015 at 4:59 pm
    Repository: hive
    Updated Branches:
       refs/heads/branch-1 c30ab4686 -> 1b5bca0a5


    HIVE-11194 - Exchange partition on external tables should fail with error message when target folder already exists (Aihua Xu, reviewd by Chao Sun)


    Project: http://git-wip-us.apache.org/repos/asf/hive/repo
    Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/1b5bca0a
    Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/1b5bca0a
    Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/1b5bca0a

    Branch: refs/heads/branch-1
    Commit: 1b5bca0a5a4556058d84003126161c81863268d7
    Parents: c30ab46
    Author: Chao Sun <sunchao@apache.org>
    Authored: Mon Jul 13 09:38:36 2015 -0700
    Committer: Chao Sun <sunchao@apache.org>
    Committed: Mon Jul 13 09:38:36 2015 -0700

    ----------------------------------------------------------------------
      .../apache/hadoop/hive/common/FileUtils.java | 9 +++-
      .../hadoop/hive/metastore/HiveMetaStore.java | 2 +-
      .../queries/clientnegative/exchange_partition.q | 19 +++++++
      .../clientnegative/exchange_partition.q.out | 54 ++++++++++++++++++++
      4 files changed, 82 insertions(+), 2 deletions(-)
    ----------------------------------------------------------------------


    http://git-wip-us.apache.org/repos/asf/hive/blob/1b5bca0a/common/src/java/org/apache/hadoop/hive/common/FileUtils.java
    ----------------------------------------------------------------------
    diff --git a/common/src/java/org/apache/hadoop/hive/common/FileUtils.java b/common/src/java/org/apache/hadoop/hive/common/FileUtils.java
    index c2c54bc..7e4f386 100644
    --- a/common/src/java/org/apache/hadoop/hive/common/FileUtils.java
    +++ b/common/src/java/org/apache/hadoop/hive/common/FileUtils.java
    @@ -30,7 +30,6 @@ import java.util.List;
      import org.apache.commons.logging.Log;
      import org.apache.commons.logging.LogFactory;
      import org.apache.hadoop.conf.Configuration;
    -import org.apache.hadoop.fs.DefaultFileAccess;
      import org.apache.hadoop.fs.FileStatus;
      import org.apache.hadoop.fs.FileSystem;
      import org.apache.hadoop.fs.FileUtil;
    @@ -635,6 +634,14 @@ public final class FileUtils {
                                     Path destPath, boolean inheritPerms,
                                     Configuration conf) throws IOException {
          LOG.info("Renaming " + sourcePath + " to " + destPath);
    +
    + // If destPath directory exists, rename call will move the sourcePath
    + // into destPath without failing. So check it before renaming.
    + if (fs.exists(destPath)) {
    + throw new IOException("Cannot rename the source path. The destination "
    + + "path already exists.");
    + }
    +
          if (!inheritPerms) {
            //just rename the directory
            return fs.rename(sourcePath, destPath);

    http://git-wip-us.apache.org/repos/asf/hive/blob/1b5bca0a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
    ----------------------------------------------------------------------
    diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
    index 580d9fc..8dde05a 100644
    --- a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
    +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
    @@ -2581,7 +2581,7 @@ public class HiveMetaStore extends ThriftHiveMetastore {
              pathCreated = wh.renameDir(sourcePath, destPath);
              success = ms.commitTransaction();
            } finally {
    - if (!success) {
    + if (!success || !pathCreated) {
                ms.rollbackTransaction();
                if (pathCreated) {
                  wh.renameDir(destPath, sourcePath);

    http://git-wip-us.apache.org/repos/asf/hive/blob/1b5bca0a/ql/src/test/queries/clientnegative/exchange_partition.q
    ----------------------------------------------------------------------
    diff --git a/ql/src/test/queries/clientnegative/exchange_partition.q b/ql/src/test/queries/clientnegative/exchange_partition.q
    new file mode 100644
    index 0000000..7dc4f57
    --- /dev/null
    +++ b/ql/src/test/queries/clientnegative/exchange_partition.q
    @@ -0,0 +1,19 @@
    +dfs ${system:test.dfs.mkdir} ${system:test.tmp.dir}/ex_table1;
    +dfs ${system:test.dfs.mkdir} ${system:test.tmp.dir}/ex_table1/part=part1;
    +CREATE EXTERNAL TABLE ex_table1 ( key INT, value STRING)
    + PARTITIONED BY (part STRING)
    + STORED AS textfile
    + LOCATION 'file:${system:test.tmp.dir}/ex_table1';
    +
    +dfs ${system:test.dfs.mkdir} ${system:test.tmp.dir}/ex_table2;
    +
    +CREATE EXTERNAL TABLE ex_table2 ( key INT, value STRING)
    + PARTITIONED BY (part STRING)
    + STORED AS textfile
    + LOCATION 'file:${system:test.tmp.dir}/ex_table2';
    +
    +INSERT OVERWRITE TABLE ex_table2 PARTITION (part='part1')
    +SELECT key, value FROM src WHERE key < 10;
    +SHOW PARTITIONS ex_table2;
    +
    +ALTER TABLE ex_table1 EXCHANGE PARTITION (part='part1') WITH TABLE ex_table2;

    http://git-wip-us.apache.org/repos/asf/hive/blob/1b5bca0a/ql/src/test/results/clientnegative/exchange_partition.q.out
    ----------------------------------------------------------------------
    diff --git a/ql/src/test/results/clientnegative/exchange_partition.q.out b/ql/src/test/results/clientnegative/exchange_partition.q.out
    new file mode 100644
    index 0000000..b81fb99
    --- /dev/null
    +++ b/ql/src/test/results/clientnegative/exchange_partition.q.out
    @@ -0,0 +1,54 @@
    +PREHOOK: query: CREATE EXTERNAL TABLE ex_table1 ( key INT, value STRING)
    + PARTITIONED BY (part STRING)
    + STORED AS textfile
    +#### A masked pattern was here ####
    +PREHOOK: type: CREATETABLE
    +#### A masked pattern was here ####
    +PREHOOK: Output: database:default
    +PREHOOK: Output: default@ex_table1
    +POSTHOOK: query: CREATE EXTERNAL TABLE ex_table1 ( key INT, value STRING)
    + PARTITIONED BY (part STRING)
    + STORED AS textfile
    +#### A masked pattern was here ####
    +POSTHOOK: type: CREATETABLE
    +#### A masked pattern was here ####
    +POSTHOOK: Output: database:default
    +POSTHOOK: Output: default@ex_table1
    +PREHOOK: query: CREATE EXTERNAL TABLE ex_table2 ( key INT, value STRING)
    + PARTITIONED BY (part STRING)
    + STORED AS textfile
    +#### A masked pattern was here ####
    +PREHOOK: type: CREATETABLE
    +#### A masked pattern was here ####
    +PREHOOK: Output: database:default
    +PREHOOK: Output: default@ex_table2
    +POSTHOOK: query: CREATE EXTERNAL TABLE ex_table2 ( key INT, value STRING)
    + PARTITIONED BY (part STRING)
    + STORED AS textfile
    +#### A masked pattern was here ####
    +POSTHOOK: type: CREATETABLE
    +#### A masked pattern was here ####
    +POSTHOOK: Output: database:default
    +POSTHOOK: Output: default@ex_table2
    +PREHOOK: query: INSERT OVERWRITE TABLE ex_table2 PARTITION (part='part1')
    +SELECT key, value FROM src WHERE key < 10
    +PREHOOK: type: QUERY
    +PREHOOK: Input: default@src
    +PREHOOK: Output: default@ex_table2@part=part1
    +POSTHOOK: query: INSERT OVERWRITE TABLE ex_table2 PARTITION (part='part1')
    +SELECT key, value FROM src WHERE key < 10
    +POSTHOOK: type: QUERY
    +POSTHOOK: Input: default@src
    +POSTHOOK: Output: default@ex_table2@part=part1
    +POSTHOOK: Lineage: ex_table2 PARTITION(part=part1).key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ]
    +POSTHOOK: Lineage: ex_table2 PARTITION(part=part1).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
    +PREHOOK: query: SHOW PARTITIONS ex_table2
    +PREHOOK: type: SHOWPARTITIONS
    +PREHOOK: Input: default@ex_table2
    +POSTHOOK: query: SHOW PARTITIONS ex_table2
    +POSTHOOK: type: SHOWPARTITIONS
    +POSTHOOK: Input: default@ex_table2
    +part=part1
    +PREHOOK: query: ALTER TABLE ex_table1 EXCHANGE PARTITION (part='part1') WITH TABLE ex_table2
    +PREHOOK: type: null
    +FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.exec.DDLTask. MetaException(message:Got exception: java.io.IOException Cannot rename the source path. The destination path already exists.)

Related Discussions

Discussion Navigation
viewthread | post
Discussion Overview
groupcommits @
categorieshive, hadoop
postedJul 13, '15 at 4:36p
activeJul 13, '15 at 4:59p
posts2
users1
websitehive.apache.org

1 user in discussion

Sunchao: 2 posts

People

Translate

site design / logo © 2021 Grokbase