FAQ
Repository: hive
Updated Branches:
   refs/heads/hbase-metastore 6ea5c0956 -> 1a1c0d817


HIVE-11061 Table renames not propagated to partition table in HBase metastore (gates)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/1a1c0d81
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/1a1c0d81
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/1a1c0d81

Branch: refs/heads/hbase-metastore
Commit: 1a1c0d817da036025f1c3b953af7b52403c048ba
Parents: 6ea5c09
Author: Alan Gates <gates@hortonworks.com>
Authored: Wed Jul 1 15:47:03 2015 -0700
Committer: Alan Gates <gates@hortonworks.com>
Committed: Wed Jul 1 15:47:03 2015 -0700

----------------------------------------------------------------------
  .../metastore/hbase/TestHBaseMetastoreSql.java | 27 +++++++++++++
  .../hadoop/hive/metastore/HiveAlterHandler.java | 16 ++++----
  .../hive/metastore/hbase/HBaseReadWrite.java | 40 ++++++++++++++++----
  .../hadoop/hive/metastore/hbase/HBaseStore.java | 21 +++++++++-
  4 files changed, 85 insertions(+), 19 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/1a1c0d81/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/hbase/TestHBaseMetastoreSql.java
----------------------------------------------------------------------
diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/hbase/TestHBaseMetastoreSql.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/hbase/TestHBaseMetastoreSql.java
index 119e5aa..c61ebb7 100644
--- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/hbase/TestHBaseMetastoreSql.java
+++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/hbase/TestHBaseMetastoreSql.java
@@ -192,5 +192,32 @@ public class TestHBaseMetastoreSql extends HBaseIntegrationTests {
      Assert.assertEquals(0, rsp.getResponseCode());
    }

+ @Test
+ public void alterRenamePartitioned() throws Exception {
+ driver.run("create table alterrename (c int) partitioned by (ds string)");
+ driver.run("alter table alterrename add partition (ds = 'a')");
+ CommandProcessorResponse rsp = driver.run("describe extended alterrename partition (ds='a')");
+ Assert.assertEquals(0, rsp.getResponseCode());
+ rsp = driver.run("alter table alterrename rename to alter_renamed");
+ Assert.assertEquals(0, rsp.getResponseCode());
+ rsp = driver.run("describe extended alter_renamed partition (ds='a')");
+ Assert.assertEquals(0, rsp.getResponseCode());
+ rsp = driver.run("describe extended alterrename partition (ds='a')");
+ Assert.assertEquals(10001, rsp.getResponseCode());
+ }
+
+ @Test
+ public void alterRename() throws Exception {
+ driver.run("create table alterrename1 (c int)");
+ CommandProcessorResponse rsp = driver.run("describe alterrename1");
+ Assert.assertEquals(0, rsp.getResponseCode());
+ rsp = driver.run("alter table alterrename1 rename to alter_renamed1");
+ Assert.assertEquals(0, rsp.getResponseCode());
+ rsp = driver.run("describe alter_renamed1");
+ Assert.assertEquals(0, rsp.getResponseCode());
+ rsp = driver.run("describe alterrename1");
+ Assert.assertEquals(10001, rsp.getResponseCode());
+ }
+

  }

http://git-wip-us.apache.org/repos/asf/hive/blob/1a1c0d81/metastore/src/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java
index 99d6680..f402f73 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java
@@ -17,12 +17,7 @@
   */
  package org.apache.hadoop.hive.metastore;

-import java.io.IOException;
-import java.net.URI;
-import java.util.ArrayList;
-import java.util.Iterator;
-import java.util.List;
-
+import com.google.common.collect.Lists;
  import org.apache.commons.lang.StringUtils;
  import org.apache.commons.logging.Log;
  import org.apache.commons.logging.LogFactory;
@@ -48,7 +43,11 @@ import org.apache.hadoop.hive.metastore.api.hive_metastoreConstants;
  import org.apache.hadoop.ipc.RemoteException;
  import org.apache.hive.common.util.HiveStringUtils;

-import com.google.common.collect.Lists;
+import java.io.IOException;
+import java.net.URI;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;

  /**
   * Hive specific implementation of alter
@@ -121,8 +120,7 @@ public class HiveAlterHandler implements AlterHandler {
        // get old table
        oldt = msdb.getTable(dbname, name);
        if (oldt == null) {
- throw new InvalidOperationException("table " + newt.getDbName() + "."
- + newt.getTableName() + " doesn't exist");
+ throw new InvalidOperationException("table " + dbname + "." + name + " doesn't exist");
        }

        if (HiveConf.getBoolVar(hiveConf,

http://git-wip-us.apache.org/repos/asf/hive/blob/1a1c0d81/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseReadWrite.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseReadWrite.java b/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseReadWrite.java
index b54afb9..fd6f9f5 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseReadWrite.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseReadWrite.java
@@ -541,6 +541,9 @@ class HBaseReadWrite {
      byte[][] serialized = HBaseUtils.serializePartition(newPart, hash);
      store(PART_TABLE, serialized[0], CATALOG_CF, CATALOG_COL, serialized[1]);
      partCache.put(newPart.getDbName(), newPart.getTableName(), newPart);
+ if (!oldPart.getTableName().equals(newPart.getTableName())) {
+ deletePartition(oldPart.getDbName(), oldPart.getTableName(), oldPart.getValues());
+ }
    }

    /**
@@ -568,7 +571,7 @@ class HBaseReadWrite {
      if (oldParts.size() != newParts.size()) {
        throw new RuntimeException("Number of old and new partitions must match.");
      }
- List<Put> puts = new ArrayList<Put>(newParts.size());
+ List<Put> puts = new ArrayList<>(newParts.size());
      for (int i = 0; i < newParts.size(); i++) {
        byte[] hash;
        byte[] oldHash = HBaseUtils.hashStorageDescriptor(oldParts.get(i).getSd(), md);
@@ -584,6 +587,11 @@ class HBaseReadWrite {
        p.add(CATALOG_CF, CATALOG_COL, serialized[1]);
        puts.add(p);
        partCache.put(newParts.get(i).getDbName(), newParts.get(i).getTableName(), newParts.get(i));
+ if (!newParts.get(i).getTableName().equals(oldParts.get(i).getTableName())) {
+ // We need to remove the old record as well.
+ deletePartition(oldParts.get(i).getDbName(), oldParts.get(i).getTableName(),
+ oldParts.get(i).getValues(), false);
+ }
      }
      HTableInterface htab = conn.getHBaseTable(PART_TABLE);
      htab.put(puts);
@@ -734,10 +742,17 @@ class HBaseReadWrite {
     * @throws IOException
     */
    void deletePartition(String dbName, String tableName, List<String> partVals) throws IOException {
+ deletePartition(dbName, tableName, partVals, true);
+ }
+
+ private void deletePartition(String dbName, String tableName, List<String> partVals,
+ boolean decrementRefCnt) throws IOException {
      // Find the partition so I can get the storage descriptor and drop it
      partCache.remove(dbName, tableName, partVals);
- Partition p = getPartition(dbName, tableName, partVals, false);
- decrementStorageDescriptorRefCount(p.getSd());
+ if (decrementRefCnt) {
+ Partition p = getPartition(dbName, tableName, partVals, false);
+ decrementStorageDescriptorRefCount(p.getSd());
+ }
      byte[] key = HBaseUtils.buildPartitionKey(dbName, tableName, partVals);
      delete(PART_TABLE, key, null, null);
    }
@@ -1287,8 +1302,10 @@ class HBaseReadWrite {
      }
      byte[][] serialized = HBaseUtils.serializeTable(newTable, hash);
      store(TABLE_TABLE, serialized[0], CATALOG_CF, CATALOG_COL, serialized[1]);
- tableCache.put(new ObjectPair<String, String>(newTable.getDbName(), newTable.getTableName()),
- newTable);
+ tableCache.put(new ObjectPair<>(newTable.getDbName(), newTable.getTableName()), newTable);
+ if (!oldTable.getTableName().equals(newTable.getTableName())) {
+ deleteTable(oldTable.getDbName(), oldTable.getTableName());
+ }
    }

    /**
@@ -1298,10 +1315,17 @@ class HBaseReadWrite {
     * @throws IOException
     */
    void deleteTable(String dbName, String tableName) throws IOException {
+ deleteTable(dbName, tableName, true);
+ }
+
+ private void deleteTable(String dbName, String tableName, boolean decrementRefCnt)
+ throws IOException {
      tableCache.remove(new ObjectPair<String, String>(dbName, tableName));
- // Find the table so I can get the storage descriptor and drop it
- Table t = getTable(dbName, tableName, false);
- decrementStorageDescriptorRefCount(t.getSd());
+ if (decrementRefCnt) {
+ // Find the table so I can get the storage descriptor and drop it
+ Table t = getTable(dbName, tableName, false);
+ decrementStorageDescriptorRefCount(t.getSd());
+ }
      byte[] key = HBaseUtils.buildKey(dbName, tableName);
      delete(TABLE_TABLE, key, null, null);
    }

http://git-wip-us.apache.org/repos/asf/hive/blob/1a1c0d81/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseStore.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseStore.java b/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseStore.java
index 0dbdba2..4fa2ae5 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseStore.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseStore.java
@@ -404,8 +404,25 @@ public class HBaseStore implements RawStore {
      boolean commit = false;
      openTransaction();
      try {
- Table oldTable = getHBase().getTable(dbname, name);
- getHBase().replaceTable(oldTable, newTable);
+ getHBase().replaceTable(getHBase().getTable(dbname, name), newTable);
+ if (newTable.getPartitionKeys() != null && newTable.getPartitionKeys().size() > 0
+ && !name.equals(newTable.getTableName())) {
+ // They renamed the table, so we need to change each partition as well, since it changes
+ // the key.
+ try {
+ List<Partition> oldParts = getPartitions(dbname, name, -1);
+ List<Partition> newParts = new ArrayList<>(oldParts.size());
+ for (Partition oldPart : oldParts) {
+ Partition newPart = oldPart.deepCopy();
+ newPart.setTableName(newTable.getTableName());
+ newParts.add(newPart);
+ }
+ getHBase().replacePartitions(oldParts, newParts);
+ } catch (NoSuchObjectException e) {
+ LOG.debug("No partitions found for old table so not worrying about it");
+ }
+
+ }
        commit = true;
      } catch (IOException e) {
        LOG.error("Unable to alter table " + tableNameForErrorMsg(dbname, name), e);

Search Discussions

Related Discussions

Discussion Navigation
viewthread | post
Discussion Overview
groupcommits @
categorieshive, hadoop
postedJul 1, '15 at 10:47p
activeJul 1, '15 at 10:47p
posts1
users1
websitehive.apache.org

1 user in discussion

Gates: 1 post

People

Translate

site design / logo © 2021 Grokbase