FAQ
Repository: hive
Updated Branches:
   refs/heads/master 4935cfda7 -> ef706b76b


HIVE-12724 ACID: Major compaction fails to include the original bucket files into MR job(Wei Zheng via Eugene Koifman)(ADDENDUM)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/ef706b76
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/ef706b76
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/ef706b76

Branch: refs/heads/master
Commit: ef706b76bf17578fdc4a95b94f2abfc866010fd7
Parents: 4935cfd
Author: Eugene Koifman <ekoifman@hortonworks.com>
Authored: Fri Jan 15 09:09:48 2016 -0800
Committer: Eugene Koifman <ekoifman@hortonworks.com>
Committed: Fri Jan 15 09:09:48 2016 -0800

----------------------------------------------------------------------
  ql/src/java/org/apache/hadoop/hive/ql/io/AcidUtils.java | 4 ++++
  .../org/apache/hadoop/hive/ql/TestTxnCommands2.java | 12 ++++++++++++
  2 files changed, 16 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/ef706b76/ql/src/java/org/apache/hadoop/hive/ql/io/AcidUtils.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/AcidUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/io/AcidUtils.java
index 3437fc8..14f7374 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/AcidUtils.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/AcidUtils.java
@@ -497,12 +497,16 @@ public class AcidUtils {

      // If we have a base, the original files are obsolete.
      if (bestBase.status != null) {
+ // Add original files to obsolete list if any
        for (HdfsFileStatusWithId fswid : original) {
          obsolete.add(fswid.getFileStatus());
        }
+ // Add original direcotries to obsolete list if any
+ obsolete.addAll(originalDirectories);
        // remove the entries so we don't get confused later and think we should
        // use them.
        original.clear();
+ originalDirectories.clear();
      } else {
        // Okay, we're going to need these originals. Recurse through them and figure out what we
        // really need.

http://git-wip-us.apache.org/repos/asf/hive/blob/ef706b76/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands2.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands2.java b/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands2.java
index 890012a..b784585 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands2.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands2.java
@@ -348,6 +348,18 @@ public class TestTxnCommands2 {
      Assert.assertEquals(resultCount, Integer.parseInt(rs.get(0)));

      // 5. Let Cleaner delete obsolete files/dirs
+ // Note, here we create a fake directory along with fake files as original directories/files
+ String fakeFile0 = TEST_WAREHOUSE_DIR + "/" + (Table.NONACIDORCTBL).toString().toLowerCase() +
+ "/subdir/000000_0";
+ String fakeFile1 = TEST_WAREHOUSE_DIR + "/" + (Table.NONACIDORCTBL).toString().toLowerCase() +
+ "/subdir/000000_1";
+ fs.create(new Path(fakeFile0));
+ fs.create(new Path(fakeFile1));
+ status = fs.listStatus(new Path(TEST_WAREHOUSE_DIR + "/" +
+ (Table.NONACIDORCTBL).toString().toLowerCase()), FileUtils.STAGING_DIR_PATH_FILTER);
+ // Before Cleaner, there should be 5 items:
+ // 2 original files, 1 original directory, 1 base directory and 1 delta directory
+ Assert.assertEquals(5, status.length);
      Cleaner c = new Cleaner();
      c.setThreadId((int) c.getId());
      c.setHiveConf(hiveConf);

Search Discussions

Related Discussions

Discussion Navigation
viewthread | post
Discussion Overview
groupcommits @
categorieshive, hadoop
postedJan 15, '16 at 5:10p
activeJan 15, '16 at 5:10p
posts1
users1
websitehive.apache.org

1 user in discussion

Ekoifman: 1 post

People

Translate

site design / logo © 2021 Grokbase