FAQ
Repository: hive
Updated Branches:
   refs/heads/branch-1 0c16534b3 -> da0c08fd0


HIVE-11130 - Refactoring the code so that HiveTxnManager interface will support lock/unlock table/database object (Aihua Xu, reviewed by Alan Gates and Chao Sun)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/da0c08fd
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/da0c08fd
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/da0c08fd

Branch: refs/heads/branch-1
Commit: da0c08fd040c1ab46480a0bcd37b27973ac79c49
Parents: 0c16534
Author: Aihua Xu <aihuaxu@gmail.com>
Authored: Tue Jul 14 11:50:24 2015 -0700
Committer: Chao Sun <chao.sun@cloudera.com>
Committed: Tue Jul 14 11:50:24 2015 -0700

----------------------------------------------------------------------
  .../org/apache/hadoop/hive/ql/exec/DDLTask.java | 147 +------------------
  .../hadoop/hive/ql/lockmgr/HiveLockObject.java | 35 +++++
  .../hadoop/hive/ql/lockmgr/HiveTxnManager.java | 47 +++++-
  .../hive/ql/lockmgr/HiveTxnManagerImpl.java | 140 ++++++++++++++++++
  4 files changed, 227 insertions(+), 142 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/da0c08fd/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
index 049857b..a8c6aca 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
@@ -2444,8 +2444,8 @@ public class DDLTask extends Task<DDLWork> implements Serializable {
          locks = lockMgr.getLocks(false, isExt);
        }
        else {
- locks = lockMgr.getLocks(getHiveObject(showLocks.getTableName(),
- showLocks.getPartSpec()),
+ locks = lockMgr.getLocks(HiveLockObject.createFrom(db,
+ showLocks.getTableName(), showLocks.getPartSpec()),
              true, isExt);
        }

@@ -2705,46 +2705,7 @@ public class DDLTask extends Task<DDLWork> implements Serializable {
    private int lockTable(LockTableDesc lockTbl) throws HiveException {
      Context ctx = driverContext.getCtx();
      HiveTxnManager txnManager = ctx.getHiveTxnManager();
- if (!txnManager.supportsExplicitLock()) {
- throw new HiveException(ErrorMsg.LOCK_REQUEST_UNSUPPORTED,
- conf.getVar(HiveConf.ConfVars.HIVE_TXN_MANAGER));
- }
- HiveLockManager lockMgr = txnManager.getLockManager();
- if (lockMgr == null) {
- throw new HiveException("lock Table LockManager not specified");
- }
-
- HiveLockMode mode = HiveLockMode.valueOf(lockTbl.getMode());
- String tabName = lockTbl.getTableName();
- Table tbl = db.getTable(tabName);
- if (tbl == null) {
- throw new HiveException("Table " + tabName + " does not exist ");
- }
-
- Map<String, String> partSpec = lockTbl.getPartSpec();
- HiveLockObjectData lockData =
- new HiveLockObjectData(lockTbl.getQueryId(),
- String.valueOf(System.currentTimeMillis()),
- "EXPLICIT",
- lockTbl.getQueryStr());
-
- if (partSpec == null) {
- HiveLock lck = lockMgr.lock(new HiveLockObject(tbl, lockData), mode, true);
- if (lck == null) {
- return 1;
- }
- return 0;
- }
-
- Partition par = db.getPartition(tbl, partSpec, false);
- if (par == null) {
- throw new HiveException("Partition " + partSpec + " for table " + tabName + " does not exist");
- }
- HiveLock lck = lockMgr.lock(new HiveLockObject(par, lockData), mode, true);
- if (lck == null) {
- return 1;
- }
- return 0;
+ return txnManager.lockTable(db, lockTbl);
    }

    /**
@@ -2759,33 +2720,7 @@ public class DDLTask extends Task<DDLWork> implements Serializable {
    private int lockDatabase(LockDatabaseDesc lockDb) throws HiveException {
      Context ctx = driverContext.getCtx();
      HiveTxnManager txnManager = ctx.getHiveTxnManager();
- if (!txnManager.supportsExplicitLock()) {
- throw new HiveException(ErrorMsg.LOCK_REQUEST_UNSUPPORTED,
- conf.getVar(HiveConf.ConfVars.HIVE_TXN_MANAGER));
- }
- HiveLockManager lockMgr = txnManager.getLockManager();
- if (lockMgr == null) {
- throw new HiveException("lock Database LockManager not specified");
- }
-
- HiveLockMode mode = HiveLockMode.valueOf(lockDb.getMode());
- String dbName = lockDb.getDatabaseName();
-
- Database dbObj = db.getDatabase(dbName);
- if (dbObj == null) {
- throw new HiveException("Database " + dbName + " does not exist ");
- }
-
- HiveLockObjectData lockData =
- new HiveLockObjectData(lockDb.getQueryId(),
- String.valueOf(System.currentTimeMillis()),
- "EXPLICIT", lockDb.getQueryStr());
-
- HiveLock lck = lockMgr.lock(new HiveLockObject(dbObj.getName(), lockData), mode, true);
- if (lck == null) {
- return 1;
- }
- return 0;
+ return txnManager.lockDatabase(db, lockDb);
    }

    /**
@@ -2800,55 +2735,7 @@ public class DDLTask extends Task<DDLWork> implements Serializable {
    private int unlockDatabase(UnlockDatabaseDesc unlockDb) throws HiveException {
      Context ctx = driverContext.getCtx();
      HiveTxnManager txnManager = ctx.getHiveTxnManager();
- if (!txnManager.supportsExplicitLock()) {
- throw new HiveException(ErrorMsg.LOCK_REQUEST_UNSUPPORTED,
- conf.getVar(HiveConf.ConfVars.HIVE_TXN_MANAGER));
- }
- HiveLockManager lockMgr = txnManager.getLockManager();
- if (lockMgr == null) {
- throw new HiveException("unlock Database LockManager not specified");
- }
-
- String dbName = unlockDb.getDatabaseName();
-
- Database dbObj = db.getDatabase(dbName);
- if (dbObj == null) {
- throw new HiveException("Database " + dbName + " does not exist ");
- }
- HiveLockObject obj = new HiveLockObject(dbObj.getName(), null);
-
- List<HiveLock> locks = lockMgr.getLocks(obj, false, false);
- if ((locks == null) || (locks.isEmpty())) {
- throw new HiveException("Database " + dbName + " is not locked ");
- }
-
- for (HiveLock lock: locks) {
- lockMgr.unlock(lock);
-
- }
- return 0;
- }
-
- private HiveLockObject getHiveObject(String tabName,
- Map<String, String> partSpec) throws HiveException {
- Table tbl = db.getTable(tabName);
- if (tbl == null) {
- throw new HiveException("Table " + tabName + " does not exist ");
- }
-
- HiveLockObject obj = null;
-
- if (partSpec == null) {
- obj = new HiveLockObject(tbl, null);
- }
- else {
- Partition par = db.getPartition(tbl, partSpec, false);
- if (par == null) {
- throw new HiveException("Partition " + partSpec + " for table " + tabName + " does not exist");
- }
- obj = new HiveLockObject(par, null);
- }
- return obj;
+ return txnManager.unlockDatabase(db, unlockDb);
    }

    /**
@@ -2863,29 +2750,7 @@ public class DDLTask extends Task<DDLWork> implements Serializable {
    private int unlockTable(UnlockTableDesc unlockTbl) throws HiveException {
      Context ctx = driverContext.getCtx();
      HiveTxnManager txnManager = ctx.getHiveTxnManager();
- if (!txnManager.supportsExplicitLock()) {
- throw new HiveException(ErrorMsg.LOCK_REQUEST_UNSUPPORTED,
- conf.getVar(HiveConf.ConfVars.HIVE_TXN_MANAGER));
- }
- HiveLockManager lockMgr = txnManager.getLockManager();
- if (lockMgr == null) {
- throw new HiveException("unlock Table LockManager not specified");
- }
-
- String tabName = unlockTbl.getTableName();
- HiveLockObject obj = getHiveObject(tabName, unlockTbl.getPartSpec());
-
- List<HiveLock> locks = lockMgr.getLocks(obj, false, false);
- if ((locks == null) || (locks.isEmpty())) {
- throw new HiveException("Table " + tabName + " is not locked ");
- }
- Iterator<HiveLock> locksIter = locks.iterator();
- while (locksIter.hasNext()) {
- HiveLock lock = locksIter.next();
- lockMgr.unlock(lock);
- }
-
- return 0;
+ return txnManager.unlockTable(db, unlockTbl);
    }

    /**

http://git-wip-us.apache.org/repos/asf/hive/blob/da0c08fd/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/HiveLockObject.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/HiveLockObject.java b/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/HiveLockObject.java
index 7e93387..fadd074 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/HiveLockObject.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/HiveLockObject.java
@@ -19,9 +19,12 @@
  package org.apache.hadoop.hive.ql.lockmgr;

  import java.util.Arrays;
+import java.util.Map;

  import org.apache.commons.lang.builder.HashCodeBuilder;
  import org.apache.hadoop.hive.ql.metadata.DummyPartition;
+import org.apache.hadoop.hive.ql.metadata.Hive;
+import org.apache.hadoop.hive.ql.metadata.HiveException;
  import org.apache.hadoop.hive.ql.metadata.Partition;
  import org.apache.hadoop.hive.ql.metadata.Table;

@@ -196,6 +199,38 @@ public class HiveLockObject {
      this(new String[] {par.getName()}, lockData);
    }

+ /**
+ * Creates a locking object for a table (when partition spec is not provided)
+ * or a table partition
+ * @param hiveDB an object to communicate with the metastore
+ * @param tableName the table to create the locking object on
+ * @param partSpec the spec of a partition to create the locking object on
+ * @return the locking object
+ * @throws HiveException
+ */
+ public static HiveLockObject createFrom(Hive hiveDB, String tableName,
+ Map<String, String> partSpec) throws HiveException {
+ Table tbl = hiveDB.getTable(tableName);
+ if (tbl == null) {
+ throw new HiveException("Table " + tableName + " does not exist ");
+ }
+
+ HiveLockObject obj = null;
+
+ if (partSpec == null) {
+ obj = new HiveLockObject(tbl, null);
+ }
+ else {
+ Partition par = hiveDB.getPartition(tbl, partSpec, false);
+ if (par == null) {
+ throw new HiveException("Partition " + partSpec + " for table " +
+ tableName + " does not exist");
+ }
+ obj = new HiveLockObject(par, null);
+ }
+ return obj;
+ }
+
    public String[] getPaths() {
      return pathNames;
    }

http://git-wip-us.apache.org/repos/asf/hive/blob/da0c08fd/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/HiveTxnManager.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/HiveTxnManager.java b/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/HiveTxnManager.java
index 6c3dc33..c900548 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/HiveTxnManager.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/HiveTxnManager.java
@@ -18,9 +18,14 @@
  package org.apache.hadoop.hive.ql.lockmgr;

  import org.apache.hadoop.hive.common.ValidTxnList;
-import org.apache.hadoop.hive.metastore.IMetaStoreClient;
  import org.apache.hadoop.hive.ql.Context;
  import org.apache.hadoop.hive.ql.QueryPlan;
+import org.apache.hadoop.hive.ql.metadata.Hive;
+import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.ql.plan.LockDatabaseDesc;
+import org.apache.hadoop.hive.ql.plan.LockTableDesc;
+import org.apache.hadoop.hive.ql.plan.UnlockDatabaseDesc;
+import org.apache.hadoop.hive.ql.plan.UnlockTableDesc;

  /**
   * An interface that allows Hive to manage transactions. All classes
@@ -116,6 +121,46 @@ public interface HiveTxnManager {
    boolean supportsExplicitLock();

    /**
+ * This function is called to lock the table when explicit lock command is
+ * issued on a table.
+ * @param hiveDB an object to communicate with the metastore
+ * @param lockTbl table locking info, such as table name, locking mode
+ * @return 0 if the locking succeeds, 1 otherwise.
+ * @throws HiveException
+ */
+ int lockTable(Hive hiveDB, LockTableDesc lockTbl) throws HiveException;
+
+ /**
+ * This function is called to unlock the table when explicit unlock command is
+ * issued on a table.
+ * @param hiveDB an object to communicate with the metastore
+ * @param unlockTbl table unlocking info, such as table name
+ * @return 0 if the locking succeeds, 1 otherwise.
+ * @throws HiveException
+ */
+ int unlockTable(Hive hiveDB, UnlockTableDesc unlockTbl) throws HiveException;
+
+ /**
+ * This function is called to lock the database when explicit lock command is
+ * issued on a database.
+ * @param hiveDB an object to communicate with the metastore
+ * @param lockDb database locking info, such as database name, locking mode
+ * @return 0 if the locking succeeds, 1 otherwise.
+ * @throws HiveException
+ */
+ int lockDatabase(Hive hiveDB, LockDatabaseDesc lockDb) throws HiveException;
+
+ /**
+ * This function is called to unlock the database when explicit unlock command
+ * is issued on a database.
+ * @param hiveDB an object to communicate with the metastore
+ * @param unlockDb database unlocking info, such as database name
+ * @return 0 if the locking succeeds, 1 otherwise.
+ * @throws HiveException
+ */
+ int unlockDatabase(Hive hiveDB, UnlockDatabaseDesc unlockDb) throws HiveException;
+
+ /**
     * Indicate whether this transaction manager returns information about locks in the new format
     * for show locks or the old one.
     * @return true if the new format should be used.

http://git-wip-us.apache.org/repos/asf/hive/blob/da0c08fd/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/HiveTxnManagerImpl.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/HiveTxnManagerImpl.java b/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/HiveTxnManagerImpl.java
index eccb8d1..ceeae68 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/HiveTxnManagerImpl.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/HiveTxnManagerImpl.java
@@ -17,7 +17,22 @@
   */
  package org.apache.hadoop.hive.ql.lockmgr;

+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+
  import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.metastore.api.Database;
+import org.apache.hadoop.hive.ql.ErrorMsg;
+import org.apache.hadoop.hive.ql.lockmgr.HiveLockObject.HiveLockObjectData;
+import org.apache.hadoop.hive.ql.metadata.Hive;
+import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.ql.metadata.Partition;
+import org.apache.hadoop.hive.ql.metadata.Table;
+import org.apache.hadoop.hive.ql.plan.LockDatabaseDesc;
+import org.apache.hadoop.hive.ql.plan.LockTableDesc;
+import org.apache.hadoop.hive.ql.plan.UnlockDatabaseDesc;
+import org.apache.hadoop.hive.ql.plan.UnlockTableDesc;

  /**
   * An implementation HiveTxnManager that includes internal methods that all
@@ -44,4 +59,129 @@ abstract class HiveTxnManagerImpl implements HiveTxnManager {
      destruct();
    }

+ @Override
+ public int lockTable(Hive db, LockTableDesc lockTbl) throws HiveException {
+ HiveLockManager lockMgr = getAndCheckLockManager();
+
+ HiveLockMode mode = HiveLockMode.valueOf(lockTbl.getMode());
+ String tabName = lockTbl.getTableName();
+ Table tbl = db.getTable(tabName);
+ if (tbl == null) {
+ throw new HiveException("Table " + tabName + " does not exist ");
+ }
+
+ Map<String, String> partSpec = lockTbl.getPartSpec();
+ HiveLockObjectData lockData =
+ new HiveLockObjectData(lockTbl.getQueryId(),
+ String.valueOf(System.currentTimeMillis()),
+ "EXPLICIT",
+ lockTbl.getQueryStr());
+
+ if (partSpec == null) {
+ HiveLock lck = lockMgr.lock(new HiveLockObject(tbl, lockData), mode, true);
+ if (lck == null) {
+ return 1;
+ }
+ return 0;
+ }
+
+ Partition par = db.getPartition(tbl, partSpec, false);
+ if (par == null) {
+ throw new HiveException("Partition " + partSpec + " for table " +
+ tabName + " does not exist");
+ }
+ HiveLock lck = lockMgr.lock(new HiveLockObject(par, lockData), mode, true);
+ if (lck == null) {
+ return 1;
+ }
+ return 0;
+ }
+
+ @Override
+ public int unlockTable(Hive hiveDB, UnlockTableDesc unlockTbl) throws HiveException {
+ HiveLockManager lockMgr = getAndCheckLockManager();
+
+ String tabName = unlockTbl.getTableName();
+ HiveLockObject obj = HiveLockObject.createFrom(hiveDB, tabName,
+ unlockTbl.getPartSpec());
+
+ List<HiveLock> locks = lockMgr.getLocks(obj, false, false);
+ if ((locks == null) || (locks.isEmpty())) {
+ throw new HiveException("Table " + tabName + " is not locked ");
+ }
+ Iterator<HiveLock> locksIter = locks.iterator();
+ while (locksIter.hasNext()) {
+ HiveLock lock = locksIter.next();
+ lockMgr.unlock(lock);
+ }
+
+ return 0;
+ }
+
+ @Override
+ public int lockDatabase(Hive hiveDB, LockDatabaseDesc lockDb) throws HiveException {
+ HiveLockManager lockMgr = getAndCheckLockManager();
+
+ HiveLockMode mode = HiveLockMode.valueOf(lockDb.getMode());
+ String dbName = lockDb.getDatabaseName();
+
+ Database dbObj = hiveDB.getDatabase(dbName);
+ if (dbObj == null) {
+ throw new HiveException("Database " + dbName + " does not exist ");
+ }
+
+ HiveLockObjectData lockData =
+ new HiveLockObjectData(lockDb.getQueryId(),
+ String.valueOf(System.currentTimeMillis()),
+ "EXPLICIT", lockDb.getQueryStr());
+
+ HiveLock lck = lockMgr.lock(new HiveLockObject(dbObj.getName(), lockData), mode, true);
+ if (lck == null) {
+ return 1;
+ }
+ return 0;
+ }
+
+ @Override
+ public int unlockDatabase(Hive hiveDB, UnlockDatabaseDesc unlockDb) throws HiveException {
+ HiveLockManager lockMgr = getAndCheckLockManager();
+
+ String dbName = unlockDb.getDatabaseName();
+
+ Database dbObj = hiveDB.getDatabase(dbName);
+ if (dbObj == null) {
+ throw new HiveException("Database " + dbName + " does not exist ");
+ }
+ HiveLockObject obj = new HiveLockObject(dbObj.getName(), null);
+
+ List<HiveLock> locks = lockMgr.getLocks(obj, false, false);
+ if ((locks == null) || (locks.isEmpty())) {
+ throw new HiveException("Database " + dbName + " is not locked ");
+ }
+
+ for (HiveLock lock: locks) {
+ lockMgr.unlock(lock);
+
+ }
+ return 0;
+ }
+
+ /**
+ * Gets the lock manager and verifies if the explicit lock is supported
+ * @return the lock manager
+ * @throws HiveException
+ */
+ protected HiveLockManager getAndCheckLockManager() throws HiveException {
+ HiveLockManager lockMgr = getLockManager();
+ if (lockMgr == null) {
+ throw new HiveException("LockManager cannot be acquired");
+ }
+
+ if (!supportsExplicitLock()) {
+ throw new HiveException(ErrorMsg.LOCK_REQUEST_UNSUPPORTED,
+ conf.getVar(HiveConf.ConfVars.HIVE_TXN_MANAGER));
+ }
+
+ return lockMgr;
+ }
  }

Search Discussions

Discussion Posts

Previous

Related Discussions

Discussion Navigation
viewthread | post
posts ‹ prev | 2 of 2 | next ›
Discussion Overview
groupcommits @
categorieshive, hadoop
postedJul 14, '15 at 6:49p
activeJul 14, '15 at 6:51p
posts2
users1
websitehive.apache.org

1 user in discussion

Sunchao: 2 posts

People

Translate

site design / logo © 2021 Grokbase