FAQ
Repository: hive
Updated Branches:
   refs/heads/master 999e0e361 -> 46c76d6b8


HIVE-11130 - Refactoring the code so that HiveTxnManager interface will support lock/unlock table/database object (Aihua Xu, reviewed by Alan Gates and Chao Sun)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/46c76d6b
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/46c76d6b
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/46c76d6b

Branch: refs/heads/master
Commit: 46c76d6b8e1fb323d1f8ac75061b6fd3bb747d6d
Parents: 999e0e3
Author: Aihua Xu <aihuaxu@gmail.com>
Authored: Tue Jul 14 11:49:09 2015 -0700
Committer: Chao Sun <chao.sun@cloudera.com>
Committed: Tue Jul 14 11:49:09 2015 -0700

----------------------------------------------------------------------
  .../org/apache/hadoop/hive/ql/exec/DDLTask.java | 147 +------------------
  .../hadoop/hive/ql/lockmgr/HiveLockObject.java | 35 +++++
  .../hadoop/hive/ql/lockmgr/HiveTxnManager.java | 47 +++++-
  .../hive/ql/lockmgr/HiveTxnManagerImpl.java | 140 ++++++++++++++++++
  4 files changed, 227 insertions(+), 142 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/46c76d6b/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
index 049857b..a8c6aca 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
@@ -2444,8 +2444,8 @@ public class DDLTask extends Task<DDLWork> implements Serializable {
          locks = lockMgr.getLocks(false, isExt);
        }
        else {
- locks = lockMgr.getLocks(getHiveObject(showLocks.getTableName(),
- showLocks.getPartSpec()),
+ locks = lockMgr.getLocks(HiveLockObject.createFrom(db,
+ showLocks.getTableName(), showLocks.getPartSpec()),
              true, isExt);
        }

@@ -2705,46 +2705,7 @@ public class DDLTask extends Task<DDLWork> implements Serializable {
    private int lockTable(LockTableDesc lockTbl) throws HiveException {
      Context ctx = driverContext.getCtx();
      HiveTxnManager txnManager = ctx.getHiveTxnManager();
- if (!txnManager.supportsExplicitLock()) {
- throw new HiveException(ErrorMsg.LOCK_REQUEST_UNSUPPORTED,
- conf.getVar(HiveConf.ConfVars.HIVE_TXN_MANAGER));
- }
- HiveLockManager lockMgr = txnManager.getLockManager();
- if (lockMgr == null) {
- throw new HiveException("lock Table LockManager not specified");
- }
-
- HiveLockMode mode = HiveLockMode.valueOf(lockTbl.getMode());
- String tabName = lockTbl.getTableName();
- Table tbl = db.getTable(tabName);
- if (tbl == null) {
- throw new HiveException("Table " + tabName + " does not exist ");
- }
-
- Map<String, String> partSpec = lockTbl.getPartSpec();
- HiveLockObjectData lockData =
- new HiveLockObjectData(lockTbl.getQueryId(),
- String.valueOf(System.currentTimeMillis()),
- "EXPLICIT",
- lockTbl.getQueryStr());
-
- if (partSpec == null) {
- HiveLock lck = lockMgr.lock(new HiveLockObject(tbl, lockData), mode, true);
- if (lck == null) {
- return 1;
- }
- return 0;
- }
-
- Partition par = db.getPartition(tbl, partSpec, false);
- if (par == null) {
- throw new HiveException("Partition " + partSpec + " for table " + tabName + " does not exist");
- }
- HiveLock lck = lockMgr.lock(new HiveLockObject(par, lockData), mode, true);
- if (lck == null) {
- return 1;
- }
- return 0;
+ return txnManager.lockTable(db, lockTbl);
    }

    /**
@@ -2759,33 +2720,7 @@ public class DDLTask extends Task<DDLWork> implements Serializable {
    private int lockDatabase(LockDatabaseDesc lockDb) throws HiveException {
      Context ctx = driverContext.getCtx();
      HiveTxnManager txnManager = ctx.getHiveTxnManager();
- if (!txnManager.supportsExplicitLock()) {
- throw new HiveException(ErrorMsg.LOCK_REQUEST_UNSUPPORTED,
- conf.getVar(HiveConf.ConfVars.HIVE_TXN_MANAGER));
- }
- HiveLockManager lockMgr = txnManager.getLockManager();
- if (lockMgr == null) {
- throw new HiveException("lock Database LockManager not specified");
- }
-
- HiveLockMode mode = HiveLockMode.valueOf(lockDb.getMode());
- String dbName = lockDb.getDatabaseName();
-
- Database dbObj = db.getDatabase(dbName);
- if (dbObj == null) {
- throw new HiveException("Database " + dbName + " does not exist ");
- }
-
- HiveLockObjectData lockData =
- new HiveLockObjectData(lockDb.getQueryId(),
- String.valueOf(System.currentTimeMillis()),
- "EXPLICIT", lockDb.getQueryStr());
-
- HiveLock lck = lockMgr.lock(new HiveLockObject(dbObj.getName(), lockData), mode, true);
- if (lck == null) {
- return 1;
- }
- return 0;
+ return txnManager.lockDatabase(db, lockDb);
    }

    /**
@@ -2800,55 +2735,7 @@ public class DDLTask extends Task<DDLWork> implements Serializable {
    private int unlockDatabase(UnlockDatabaseDesc unlockDb) throws HiveException {
      Context ctx = driverContext.getCtx();
      HiveTxnManager txnManager = ctx.getHiveTxnManager();
- if (!txnManager.supportsExplicitLock()) {
- throw new HiveException(ErrorMsg.LOCK_REQUEST_UNSUPPORTED,
- conf.getVar(HiveConf.ConfVars.HIVE_TXN_MANAGER));
- }
- HiveLockManager lockMgr = txnManager.getLockManager();
- if (lockMgr == null) {
- throw new HiveException("unlock Database LockManager not specified");
- }
-
- String dbName = unlockDb.getDatabaseName();
-
- Database dbObj = db.getDatabase(dbName);
- if (dbObj == null) {
- throw new HiveException("Database " + dbName + " does not exist ");
- }
- HiveLockObject obj = new HiveLockObject(dbObj.getName(), null);
-
- List<HiveLock> locks = lockMgr.getLocks(obj, false, false);
- if ((locks == null) || (locks.isEmpty())) {
- throw new HiveException("Database " + dbName + " is not locked ");
- }
-
- for (HiveLock lock: locks) {
- lockMgr.unlock(lock);
-
- }
- return 0;
- }
-
- private HiveLockObject getHiveObject(String tabName,
- Map<String, String> partSpec) throws HiveException {
- Table tbl = db.getTable(tabName);
- if (tbl == null) {
- throw new HiveException("Table " + tabName + " does not exist ");
- }
-
- HiveLockObject obj = null;
-
- if (partSpec == null) {
- obj = new HiveLockObject(tbl, null);
- }
- else {
- Partition par = db.getPartition(tbl, partSpec, false);
- if (par == null) {
- throw new HiveException("Partition " + partSpec + " for table " + tabName + " does not exist");
- }
- obj = new HiveLockObject(par, null);
- }
- return obj;
+ return txnManager.unlockDatabase(db, unlockDb);
    }

    /**
@@ -2863,29 +2750,7 @@ public class DDLTask extends Task<DDLWork> implements Serializable {
    private int unlockTable(UnlockTableDesc unlockTbl) throws HiveException {
      Context ctx = driverContext.getCtx();
      HiveTxnManager txnManager = ctx.getHiveTxnManager();
- if (!txnManager.supportsExplicitLock()) {
- throw new HiveException(ErrorMsg.LOCK_REQUEST_UNSUPPORTED,
- conf.getVar(HiveConf.ConfVars.HIVE_TXN_MANAGER));
- }
- HiveLockManager lockMgr = txnManager.getLockManager();
- if (lockMgr == null) {
- throw new HiveException("unlock Table LockManager not specified");
- }
-
- String tabName = unlockTbl.getTableName();
- HiveLockObject obj = getHiveObject(tabName, unlockTbl.getPartSpec());
-
- List<HiveLock> locks = lockMgr.getLocks(obj, false, false);
- if ((locks == null) || (locks.isEmpty())) {
- throw new HiveException("Table " + tabName + " is not locked ");
- }
- Iterator<HiveLock> locksIter = locks.iterator();
- while (locksIter.hasNext()) {
- HiveLock lock = locksIter.next();
- lockMgr.unlock(lock);
- }
-
- return 0;
+ return txnManager.unlockTable(db, unlockTbl);
    }

    /**

http://git-wip-us.apache.org/repos/asf/hive/blob/46c76d6b/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/HiveLockObject.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/HiveLockObject.java b/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/HiveLockObject.java
index 7e93387..fadd074 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/HiveLockObject.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/HiveLockObject.java
@@ -19,9 +19,12 @@
  package org.apache.hadoop.hive.ql.lockmgr;

  import java.util.Arrays;
+import java.util.Map;

  import org.apache.commons.lang.builder.HashCodeBuilder;
  import org.apache.hadoop.hive.ql.metadata.DummyPartition;
+import org.apache.hadoop.hive.ql.metadata.Hive;
+import org.apache.hadoop.hive.ql.metadata.HiveException;
  import org.apache.hadoop.hive.ql.metadata.Partition;
  import org.apache.hadoop.hive.ql.metadata.Table;

@@ -196,6 +199,38 @@ public class HiveLockObject {
      this(new String[] {par.getName()}, lockData);
    }

+ /**
+ * Creates a locking object for a table (when partition spec is not provided)
+ * or a table partition
+ * @param hiveDB an object to communicate with the metastore
+ * @param tableName the table to create the locking object on
+ * @param partSpec the spec of a partition to create the locking object on
+ * @return the locking object
+ * @throws HiveException
+ */
+ public static HiveLockObject createFrom(Hive hiveDB, String tableName,
+ Map<String, String> partSpec) throws HiveException {
+ Table tbl = hiveDB.getTable(tableName);
+ if (tbl == null) {
+ throw new HiveException("Table " + tableName + " does not exist ");
+ }
+
+ HiveLockObject obj = null;
+
+ if (partSpec == null) {
+ obj = new HiveLockObject(tbl, null);
+ }
+ else {
+ Partition par = hiveDB.getPartition(tbl, partSpec, false);
+ if (par == null) {
+ throw new HiveException("Partition " + partSpec + " for table " +
+ tableName + " does not exist");
+ }
+ obj = new HiveLockObject(par, null);
+ }
+ return obj;
+ }
+
    public String[] getPaths() {
      return pathNames;
    }

http://git-wip-us.apache.org/repos/asf/hive/blob/46c76d6b/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/HiveTxnManager.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/HiveTxnManager.java b/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/HiveTxnManager.java
index 6c3dc33..c900548 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/HiveTxnManager.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/HiveTxnManager.java
@@ -18,9 +18,14 @@
  package org.apache.hadoop.hive.ql.lockmgr;

  import org.apache.hadoop.hive.common.ValidTxnList;
-import org.apache.hadoop.hive.metastore.IMetaStoreClient;
  import org.apache.hadoop.hive.ql.Context;
  import org.apache.hadoop.hive.ql.QueryPlan;
+import org.apache.hadoop.hive.ql.metadata.Hive;
+import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.ql.plan.LockDatabaseDesc;
+import org.apache.hadoop.hive.ql.plan.LockTableDesc;
+import org.apache.hadoop.hive.ql.plan.UnlockDatabaseDesc;
+import org.apache.hadoop.hive.ql.plan.UnlockTableDesc;

  /**
   * An interface that allows Hive to manage transactions. All classes
@@ -116,6 +121,46 @@ public interface HiveTxnManager {
    boolean supportsExplicitLock();

    /**
+ * This function is called to lock the table when explicit lock command is
+ * issued on a table.
+ * @param hiveDB an object to communicate with the metastore
+ * @param lockTbl table locking info, such as table name, locking mode
+ * @return 0 if the locking succeeds, 1 otherwise.
+ * @throws HiveException
+ */
+ int lockTable(Hive hiveDB, LockTableDesc lockTbl) throws HiveException;
+
+ /**
+ * This function is called to unlock the table when explicit unlock command is
+ * issued on a table.
+ * @param hiveDB an object to communicate with the metastore
+ * @param unlockTbl table unlocking info, such as table name
+ * @return 0 if the locking succeeds, 1 otherwise.
+ * @throws HiveException
+ */
+ int unlockTable(Hive hiveDB, UnlockTableDesc unlockTbl) throws HiveException;
+
+ /**
+ * This function is called to lock the database when explicit lock command is
+ * issued on a database.
+ * @param hiveDB an object to communicate with the metastore
+ * @param lockDb database locking info, such as database name, locking mode
+ * @return 0 if the locking succeeds, 1 otherwise.
+ * @throws HiveException
+ */
+ int lockDatabase(Hive hiveDB, LockDatabaseDesc lockDb) throws HiveException;
+
+ /**
+ * This function is called to unlock the database when explicit unlock command
+ * is issued on a database.
+ * @param hiveDB an object to communicate with the metastore
+ * @param unlockDb database unlocking info, such as database name
+ * @return 0 if the locking succeeds, 1 otherwise.
+ * @throws HiveException
+ */
+ int unlockDatabase(Hive hiveDB, UnlockDatabaseDesc unlockDb) throws HiveException;
+
+ /**
     * Indicate whether this transaction manager returns information about locks in the new format
     * for show locks or the old one.
     * @return true if the new format should be used.

http://git-wip-us.apache.org/repos/asf/hive/blob/46c76d6b/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/HiveTxnManagerImpl.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/HiveTxnManagerImpl.java b/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/HiveTxnManagerImpl.java
index eccb8d1..ceeae68 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/HiveTxnManagerImpl.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/HiveTxnManagerImpl.java
@@ -17,7 +17,22 @@
   */
  package org.apache.hadoop.hive.ql.lockmgr;

+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+
  import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.metastore.api.Database;
+import org.apache.hadoop.hive.ql.ErrorMsg;
+import org.apache.hadoop.hive.ql.lockmgr.HiveLockObject.HiveLockObjectData;
+import org.apache.hadoop.hive.ql.metadata.Hive;
+import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.ql.metadata.Partition;
+import org.apache.hadoop.hive.ql.metadata.Table;
+import org.apache.hadoop.hive.ql.plan.LockDatabaseDesc;
+import org.apache.hadoop.hive.ql.plan.LockTableDesc;
+import org.apache.hadoop.hive.ql.plan.UnlockDatabaseDesc;
+import org.apache.hadoop.hive.ql.plan.UnlockTableDesc;

  /**
   * An implementation HiveTxnManager that includes internal methods that all
@@ -44,4 +59,129 @@ abstract class HiveTxnManagerImpl implements HiveTxnManager {
      destruct();
    }

+ @Override
+ public int lockTable(Hive db, LockTableDesc lockTbl) throws HiveException {
+ HiveLockManager lockMgr = getAndCheckLockManager();
+
+ HiveLockMode mode = HiveLockMode.valueOf(lockTbl.getMode());
+ String tabName = lockTbl.getTableName();
+ Table tbl = db.getTable(tabName);
+ if (tbl == null) {
+ throw new HiveException("Table " + tabName + " does not exist ");
+ }
+
+ Map<String, String> partSpec = lockTbl.getPartSpec();
+ HiveLockObjectData lockData =
+ new HiveLockObjectData(lockTbl.getQueryId(),
+ String.valueOf(System.currentTimeMillis()),
+ "EXPLICIT",
+ lockTbl.getQueryStr());
+
+ if (partSpec == null) {
+ HiveLock lck = lockMgr.lock(new HiveLockObject(tbl, lockData), mode, true);
+ if (lck == null) {
+ return 1;
+ }
+ return 0;
+ }
+
+ Partition par = db.getPartition(tbl, partSpec, false);
+ if (par == null) {
+ throw new HiveException("Partition " + partSpec + " for table " +
+ tabName + " does not exist");
+ }
+ HiveLock lck = lockMgr.lock(new HiveLockObject(par, lockData), mode, true);
+ if (lck == null) {
+ return 1;
+ }
+ return 0;
+ }
+
+ @Override
+ public int unlockTable(Hive hiveDB, UnlockTableDesc unlockTbl) throws HiveException {
+ HiveLockManager lockMgr = getAndCheckLockManager();
+
+ String tabName = unlockTbl.getTableName();
+ HiveLockObject obj = HiveLockObject.createFrom(hiveDB, tabName,
+ unlockTbl.getPartSpec());
+
+ List<HiveLock> locks = lockMgr.getLocks(obj, false, false);
+ if ((locks == null) || (locks.isEmpty())) {
+ throw new HiveException("Table " + tabName + " is not locked ");
+ }
+ Iterator<HiveLock> locksIter = locks.iterator();
+ while (locksIter.hasNext()) {
+ HiveLock lock = locksIter.next();
+ lockMgr.unlock(lock);
+ }
+
+ return 0;
+ }
+
+ @Override
+ public int lockDatabase(Hive hiveDB, LockDatabaseDesc lockDb) throws HiveException {
+ HiveLockManager lockMgr = getAndCheckLockManager();
+
+ HiveLockMode mode = HiveLockMode.valueOf(lockDb.getMode());
+ String dbName = lockDb.getDatabaseName();
+
+ Database dbObj = hiveDB.getDatabase(dbName);
+ if (dbObj == null) {
+ throw new HiveException("Database " + dbName + " does not exist ");
+ }
+
+ HiveLockObjectData lockData =
+ new HiveLockObjectData(lockDb.getQueryId(),
+ String.valueOf(System.currentTimeMillis()),
+ "EXPLICIT", lockDb.getQueryStr());
+
+ HiveLock lck = lockMgr.lock(new HiveLockObject(dbObj.getName(), lockData), mode, true);
+ if (lck == null) {
+ return 1;
+ }
+ return 0;
+ }
+
+ @Override
+ public int unlockDatabase(Hive hiveDB, UnlockDatabaseDesc unlockDb) throws HiveException {
+ HiveLockManager lockMgr = getAndCheckLockManager();
+
+ String dbName = unlockDb.getDatabaseName();
+
+ Database dbObj = hiveDB.getDatabase(dbName);
+ if (dbObj == null) {
+ throw new HiveException("Database " + dbName + " does not exist ");
+ }
+ HiveLockObject obj = new HiveLockObject(dbObj.getName(), null);
+
+ List<HiveLock> locks = lockMgr.getLocks(obj, false, false);
+ if ((locks == null) || (locks.isEmpty())) {
+ throw new HiveException("Database " + dbName + " is not locked ");
+ }
+
+ for (HiveLock lock: locks) {
+ lockMgr.unlock(lock);
+
+ }
+ return 0;
+ }
+
+ /**
+ * Gets the lock manager and verifies if the explicit lock is supported
+ * @return the lock manager
+ * @throws HiveException
+ */
+ protected HiveLockManager getAndCheckLockManager() throws HiveException {
+ HiveLockManager lockMgr = getLockManager();
+ if (lockMgr == null) {
+ throw new HiveException("LockManager cannot be acquired");
+ }
+
+ if (!supportsExplicitLock()) {
+ throw new HiveException(ErrorMsg.LOCK_REQUEST_UNSUPPORTED,
+ conf.getVar(HiveConf.ConfVars.HIVE_TXN_MANAGER));
+ }
+
+ return lockMgr;
+ }
  }

Search Discussions

  • Sunchao at Jul 14, 2015 at 6:51 pm
    Repository: hive
    Updated Branches:
       refs/heads/branch-1 0c16534b3 -> da0c08fd0


    HIVE-11130 - Refactoring the code so that HiveTxnManager interface will support lock/unlock table/database object (Aihua Xu, reviewed by Alan Gates and Chao Sun)


    Project: http://git-wip-us.apache.org/repos/asf/hive/repo
    Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/da0c08fd
    Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/da0c08fd
    Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/da0c08fd

    Branch: refs/heads/branch-1
    Commit: da0c08fd040c1ab46480a0bcd37b27973ac79c49
    Parents: 0c16534
    Author: Aihua Xu <aihuaxu@gmail.com>
    Authored: Tue Jul 14 11:50:24 2015 -0700
    Committer: Chao Sun <chao.sun@cloudera.com>
    Committed: Tue Jul 14 11:50:24 2015 -0700

    ----------------------------------------------------------------------
      .../org/apache/hadoop/hive/ql/exec/DDLTask.java | 147 +------------------
      .../hadoop/hive/ql/lockmgr/HiveLockObject.java | 35 +++++
      .../hadoop/hive/ql/lockmgr/HiveTxnManager.java | 47 +++++-
      .../hive/ql/lockmgr/HiveTxnManagerImpl.java | 140 ++++++++++++++++++
      4 files changed, 227 insertions(+), 142 deletions(-)
    ----------------------------------------------------------------------


    http://git-wip-us.apache.org/repos/asf/hive/blob/da0c08fd/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
    ----------------------------------------------------------------------
    diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
    index 049857b..a8c6aca 100644
    --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
    +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
    @@ -2444,8 +2444,8 @@ public class DDLTask extends Task<DDLWork> implements Serializable {
              locks = lockMgr.getLocks(false, isExt);
            }
            else {
    - locks = lockMgr.getLocks(getHiveObject(showLocks.getTableName(),
    - showLocks.getPartSpec()),
    + locks = lockMgr.getLocks(HiveLockObject.createFrom(db,
    + showLocks.getTableName(), showLocks.getPartSpec()),
                  true, isExt);
            }

    @@ -2705,46 +2705,7 @@ public class DDLTask extends Task<DDLWork> implements Serializable {
        private int lockTable(LockTableDesc lockTbl) throws HiveException {
          Context ctx = driverContext.getCtx();
          HiveTxnManager txnManager = ctx.getHiveTxnManager();
    - if (!txnManager.supportsExplicitLock()) {
    - throw new HiveException(ErrorMsg.LOCK_REQUEST_UNSUPPORTED,
    - conf.getVar(HiveConf.ConfVars.HIVE_TXN_MANAGER));
    - }
    - HiveLockManager lockMgr = txnManager.getLockManager();
    - if (lockMgr == null) {
    - throw new HiveException("lock Table LockManager not specified");
    - }
    -
    - HiveLockMode mode = HiveLockMode.valueOf(lockTbl.getMode());
    - String tabName = lockTbl.getTableName();
    - Table tbl = db.getTable(tabName);
    - if (tbl == null) {
    - throw new HiveException("Table " + tabName + " does not exist ");
    - }
    -
    - Map<String, String> partSpec = lockTbl.getPartSpec();
    - HiveLockObjectData lockData =
    - new HiveLockObjectData(lockTbl.getQueryId(),
    - String.valueOf(System.currentTimeMillis()),
    - "EXPLICIT",
    - lockTbl.getQueryStr());
    -
    - if (partSpec == null) {
    - HiveLock lck = lockMgr.lock(new HiveLockObject(tbl, lockData), mode, true);
    - if (lck == null) {
    - return 1;
    - }
    - return 0;
    - }
    -
    - Partition par = db.getPartition(tbl, partSpec, false);
    - if (par == null) {
    - throw new HiveException("Partition " + partSpec + " for table " + tabName + " does not exist");
    - }
    - HiveLock lck = lockMgr.lock(new HiveLockObject(par, lockData), mode, true);
    - if (lck == null) {
    - return 1;
    - }
    - return 0;
    + return txnManager.lockTable(db, lockTbl);
        }

        /**
    @@ -2759,33 +2720,7 @@ public class DDLTask extends Task<DDLWork> implements Serializable {
        private int lockDatabase(LockDatabaseDesc lockDb) throws HiveException {
          Context ctx = driverContext.getCtx();
          HiveTxnManager txnManager = ctx.getHiveTxnManager();
    - if (!txnManager.supportsExplicitLock()) {
    - throw new HiveException(ErrorMsg.LOCK_REQUEST_UNSUPPORTED,
    - conf.getVar(HiveConf.ConfVars.HIVE_TXN_MANAGER));
    - }
    - HiveLockManager lockMgr = txnManager.getLockManager();
    - if (lockMgr == null) {
    - throw new HiveException("lock Database LockManager not specified");
    - }
    -
    - HiveLockMode mode = HiveLockMode.valueOf(lockDb.getMode());
    - String dbName = lockDb.getDatabaseName();
    -
    - Database dbObj = db.getDatabase(dbName);
    - if (dbObj == null) {
    - throw new HiveException("Database " + dbName + " does not exist ");
    - }
    -
    - HiveLockObjectData lockData =
    - new HiveLockObjectData(lockDb.getQueryId(),
    - String.valueOf(System.currentTimeMillis()),
    - "EXPLICIT", lockDb.getQueryStr());
    -
    - HiveLock lck = lockMgr.lock(new HiveLockObject(dbObj.getName(), lockData), mode, true);
    - if (lck == null) {
    - return 1;
    - }
    - return 0;
    + return txnManager.lockDatabase(db, lockDb);
        }

        /**
    @@ -2800,55 +2735,7 @@ public class DDLTask extends Task<DDLWork> implements Serializable {
        private int unlockDatabase(UnlockDatabaseDesc unlockDb) throws HiveException {
          Context ctx = driverContext.getCtx();
          HiveTxnManager txnManager = ctx.getHiveTxnManager();
    - if (!txnManager.supportsExplicitLock()) {
    - throw new HiveException(ErrorMsg.LOCK_REQUEST_UNSUPPORTED,
    - conf.getVar(HiveConf.ConfVars.HIVE_TXN_MANAGER));
    - }
    - HiveLockManager lockMgr = txnManager.getLockManager();
    - if (lockMgr == null) {
    - throw new HiveException("unlock Database LockManager not specified");
    - }
    -
    - String dbName = unlockDb.getDatabaseName();
    -
    - Database dbObj = db.getDatabase(dbName);
    - if (dbObj == null) {
    - throw new HiveException("Database " + dbName + " does not exist ");
    - }
    - HiveLockObject obj = new HiveLockObject(dbObj.getName(), null);
    -
    - List<HiveLock> locks = lockMgr.getLocks(obj, false, false);
    - if ((locks == null) || (locks.isEmpty())) {
    - throw new HiveException("Database " + dbName + " is not locked ");
    - }
    -
    - for (HiveLock lock: locks) {
    - lockMgr.unlock(lock);
    -
    - }
    - return 0;
    - }
    -
    - private HiveLockObject getHiveObject(String tabName,
    - Map<String, String> partSpec) throws HiveException {
    - Table tbl = db.getTable(tabName);
    - if (tbl == null) {
    - throw new HiveException("Table " + tabName + " does not exist ");
    - }
    -
    - HiveLockObject obj = null;
    -
    - if (partSpec == null) {
    - obj = new HiveLockObject(tbl, null);
    - }
    - else {
    - Partition par = db.getPartition(tbl, partSpec, false);
    - if (par == null) {
    - throw new HiveException("Partition " + partSpec + " for table " + tabName + " does not exist");
    - }
    - obj = new HiveLockObject(par, null);
    - }
    - return obj;
    + return txnManager.unlockDatabase(db, unlockDb);
        }

        /**
    @@ -2863,29 +2750,7 @@ public class DDLTask extends Task<DDLWork> implements Serializable {
        private int unlockTable(UnlockTableDesc unlockTbl) throws HiveException {
          Context ctx = driverContext.getCtx();
          HiveTxnManager txnManager = ctx.getHiveTxnManager();
    - if (!txnManager.supportsExplicitLock()) {
    - throw new HiveException(ErrorMsg.LOCK_REQUEST_UNSUPPORTED,
    - conf.getVar(HiveConf.ConfVars.HIVE_TXN_MANAGER));
    - }
    - HiveLockManager lockMgr = txnManager.getLockManager();
    - if (lockMgr == null) {
    - throw new HiveException("unlock Table LockManager not specified");
    - }
    -
    - String tabName = unlockTbl.getTableName();
    - HiveLockObject obj = getHiveObject(tabName, unlockTbl.getPartSpec());
    -
    - List<HiveLock> locks = lockMgr.getLocks(obj, false, false);
    - if ((locks == null) || (locks.isEmpty())) {
    - throw new HiveException("Table " + tabName + " is not locked ");
    - }
    - Iterator<HiveLock> locksIter = locks.iterator();
    - while (locksIter.hasNext()) {
    - HiveLock lock = locksIter.next();
    - lockMgr.unlock(lock);
    - }
    -
    - return 0;
    + return txnManager.unlockTable(db, unlockTbl);
        }

        /**

    http://git-wip-us.apache.org/repos/asf/hive/blob/da0c08fd/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/HiveLockObject.java
    ----------------------------------------------------------------------
    diff --git a/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/HiveLockObject.java b/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/HiveLockObject.java
    index 7e93387..fadd074 100644
    --- a/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/HiveLockObject.java
    +++ b/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/HiveLockObject.java
    @@ -19,9 +19,12 @@
      package org.apache.hadoop.hive.ql.lockmgr;

      import java.util.Arrays;
    +import java.util.Map;

      import org.apache.commons.lang.builder.HashCodeBuilder;
      import org.apache.hadoop.hive.ql.metadata.DummyPartition;
    +import org.apache.hadoop.hive.ql.metadata.Hive;
    +import org.apache.hadoop.hive.ql.metadata.HiveException;
      import org.apache.hadoop.hive.ql.metadata.Partition;
      import org.apache.hadoop.hive.ql.metadata.Table;

    @@ -196,6 +199,38 @@ public class HiveLockObject {
          this(new String[] {par.getName()}, lockData);
        }

    + /**
    + * Creates a locking object for a table (when partition spec is not provided)
    + * or a table partition
    + * @param hiveDB an object to communicate with the metastore
    + * @param tableName the table to create the locking object on
    + * @param partSpec the spec of a partition to create the locking object on
    + * @return the locking object
    + * @throws HiveException
    + */
    + public static HiveLockObject createFrom(Hive hiveDB, String tableName,
    + Map<String, String> partSpec) throws HiveException {
    + Table tbl = hiveDB.getTable(tableName);
    + if (tbl == null) {
    + throw new HiveException("Table " + tableName + " does not exist ");
    + }
    +
    + HiveLockObject obj = null;
    +
    + if (partSpec == null) {
    + obj = new HiveLockObject(tbl, null);
    + }
    + else {
    + Partition par = hiveDB.getPartition(tbl, partSpec, false);
    + if (par == null) {
    + throw new HiveException("Partition " + partSpec + " for table " +
    + tableName + " does not exist");
    + }
    + obj = new HiveLockObject(par, null);
    + }
    + return obj;
    + }
    +
        public String[] getPaths() {
          return pathNames;
        }

    http://git-wip-us.apache.org/repos/asf/hive/blob/da0c08fd/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/HiveTxnManager.java
    ----------------------------------------------------------------------
    diff --git a/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/HiveTxnManager.java b/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/HiveTxnManager.java
    index 6c3dc33..c900548 100644
    --- a/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/HiveTxnManager.java
    +++ b/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/HiveTxnManager.java
    @@ -18,9 +18,14 @@
      package org.apache.hadoop.hive.ql.lockmgr;

      import org.apache.hadoop.hive.common.ValidTxnList;
    -import org.apache.hadoop.hive.metastore.IMetaStoreClient;
      import org.apache.hadoop.hive.ql.Context;
      import org.apache.hadoop.hive.ql.QueryPlan;
    +import org.apache.hadoop.hive.ql.metadata.Hive;
    +import org.apache.hadoop.hive.ql.metadata.HiveException;
    +import org.apache.hadoop.hive.ql.plan.LockDatabaseDesc;
    +import org.apache.hadoop.hive.ql.plan.LockTableDesc;
    +import org.apache.hadoop.hive.ql.plan.UnlockDatabaseDesc;
    +import org.apache.hadoop.hive.ql.plan.UnlockTableDesc;

      /**
       * An interface that allows Hive to manage transactions. All classes
    @@ -116,6 +121,46 @@ public interface HiveTxnManager {
        boolean supportsExplicitLock();

        /**
    + * This function is called to lock the table when explicit lock command is
    + * issued on a table.
    + * @param hiveDB an object to communicate with the metastore
    + * @param lockTbl table locking info, such as table name, locking mode
    + * @return 0 if the locking succeeds, 1 otherwise.
    + * @throws HiveException
    + */
    + int lockTable(Hive hiveDB, LockTableDesc lockTbl) throws HiveException;
    +
    + /**
    + * This function is called to unlock the table when explicit unlock command is
    + * issued on a table.
    + * @param hiveDB an object to communicate with the metastore
    + * @param unlockTbl table unlocking info, such as table name
    + * @return 0 if the locking succeeds, 1 otherwise.
    + * @throws HiveException
    + */
    + int unlockTable(Hive hiveDB, UnlockTableDesc unlockTbl) throws HiveException;
    +
    + /**
    + * This function is called to lock the database when explicit lock command is
    + * issued on a database.
    + * @param hiveDB an object to communicate with the metastore
    + * @param lockDb database locking info, such as database name, locking mode
    + * @return 0 if the locking succeeds, 1 otherwise.
    + * @throws HiveException
    + */
    + int lockDatabase(Hive hiveDB, LockDatabaseDesc lockDb) throws HiveException;
    +
    + /**
    + * This function is called to unlock the database when explicit unlock command
    + * is issued on a database.
    + * @param hiveDB an object to communicate with the metastore
    + * @param unlockDb database unlocking info, such as database name
    + * @return 0 if the locking succeeds, 1 otherwise.
    + * @throws HiveException
    + */
    + int unlockDatabase(Hive hiveDB, UnlockDatabaseDesc unlockDb) throws HiveException;
    +
    + /**
         * Indicate whether this transaction manager returns information about locks in the new format
         * for show locks or the old one.
         * @return true if the new format should be used.

    http://git-wip-us.apache.org/repos/asf/hive/blob/da0c08fd/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/HiveTxnManagerImpl.java
    ----------------------------------------------------------------------
    diff --git a/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/HiveTxnManagerImpl.java b/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/HiveTxnManagerImpl.java
    index eccb8d1..ceeae68 100644
    --- a/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/HiveTxnManagerImpl.java
    +++ b/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/HiveTxnManagerImpl.java
    @@ -17,7 +17,22 @@
       */
      package org.apache.hadoop.hive.ql.lockmgr;

    +import java.util.Iterator;
    +import java.util.List;
    +import java.util.Map;
    +
      import org.apache.hadoop.hive.conf.HiveConf;
    +import org.apache.hadoop.hive.metastore.api.Database;
    +import org.apache.hadoop.hive.ql.ErrorMsg;
    +import org.apache.hadoop.hive.ql.lockmgr.HiveLockObject.HiveLockObjectData;
    +import org.apache.hadoop.hive.ql.metadata.Hive;
    +import org.apache.hadoop.hive.ql.metadata.HiveException;
    +import org.apache.hadoop.hive.ql.metadata.Partition;
    +import org.apache.hadoop.hive.ql.metadata.Table;
    +import org.apache.hadoop.hive.ql.plan.LockDatabaseDesc;
    +import org.apache.hadoop.hive.ql.plan.LockTableDesc;
    +import org.apache.hadoop.hive.ql.plan.UnlockDatabaseDesc;
    +import org.apache.hadoop.hive.ql.plan.UnlockTableDesc;

      /**
       * An implementation HiveTxnManager that includes internal methods that all
    @@ -44,4 +59,129 @@ abstract class HiveTxnManagerImpl implements HiveTxnManager {
          destruct();
        }

    + @Override
    + public int lockTable(Hive db, LockTableDesc lockTbl) throws HiveException {
    + HiveLockManager lockMgr = getAndCheckLockManager();
    +
    + HiveLockMode mode = HiveLockMode.valueOf(lockTbl.getMode());
    + String tabName = lockTbl.getTableName();
    + Table tbl = db.getTable(tabName);
    + if (tbl == null) {
    + throw new HiveException("Table " + tabName + " does not exist ");
    + }
    +
    + Map<String, String> partSpec = lockTbl.getPartSpec();
    + HiveLockObjectData lockData =
    + new HiveLockObjectData(lockTbl.getQueryId(),
    + String.valueOf(System.currentTimeMillis()),
    + "EXPLICIT",
    + lockTbl.getQueryStr());
    +
    + if (partSpec == null) {
    + HiveLock lck = lockMgr.lock(new HiveLockObject(tbl, lockData), mode, true);
    + if (lck == null) {
    + return 1;
    + }
    + return 0;
    + }
    +
    + Partition par = db.getPartition(tbl, partSpec, false);
    + if (par == null) {
    + throw new HiveException("Partition " + partSpec + " for table " +
    + tabName + " does not exist");
    + }
    + HiveLock lck = lockMgr.lock(new HiveLockObject(par, lockData), mode, true);
    + if (lck == null) {
    + return 1;
    + }
    + return 0;
    + }
    +
    + @Override
    + public int unlockTable(Hive hiveDB, UnlockTableDesc unlockTbl) throws HiveException {
    + HiveLockManager lockMgr = getAndCheckLockManager();
    +
    + String tabName = unlockTbl.getTableName();
    + HiveLockObject obj = HiveLockObject.createFrom(hiveDB, tabName,
    + unlockTbl.getPartSpec());
    +
    + List<HiveLock> locks = lockMgr.getLocks(obj, false, false);
    + if ((locks == null) || (locks.isEmpty())) {
    + throw new HiveException("Table " + tabName + " is not locked ");
    + }
    + Iterator<HiveLock> locksIter = locks.iterator();
    + while (locksIter.hasNext()) {
    + HiveLock lock = locksIter.next();
    + lockMgr.unlock(lock);
    + }
    +
    + return 0;
    + }
    +
    + @Override
    + public int lockDatabase(Hive hiveDB, LockDatabaseDesc lockDb) throws HiveException {
    + HiveLockManager lockMgr = getAndCheckLockManager();
    +
    + HiveLockMode mode = HiveLockMode.valueOf(lockDb.getMode());
    + String dbName = lockDb.getDatabaseName();
    +
    + Database dbObj = hiveDB.getDatabase(dbName);
    + if (dbObj == null) {
    + throw new HiveException("Database " + dbName + " does not exist ");
    + }
    +
    + HiveLockObjectData lockData =
    + new HiveLockObjectData(lockDb.getQueryId(),
    + String.valueOf(System.currentTimeMillis()),
    + "EXPLICIT", lockDb.getQueryStr());
    +
    + HiveLock lck = lockMgr.lock(new HiveLockObject(dbObj.getName(), lockData), mode, true);
    + if (lck == null) {
    + return 1;
    + }
    + return 0;
    + }
    +
    + @Override
    + public int unlockDatabase(Hive hiveDB, UnlockDatabaseDesc unlockDb) throws HiveException {
    + HiveLockManager lockMgr = getAndCheckLockManager();
    +
    + String dbName = unlockDb.getDatabaseName();
    +
    + Database dbObj = hiveDB.getDatabase(dbName);
    + if (dbObj == null) {
    + throw new HiveException("Database " + dbName + " does not exist ");
    + }
    + HiveLockObject obj = new HiveLockObject(dbObj.getName(), null);
    +
    + List<HiveLock> locks = lockMgr.getLocks(obj, false, false);
    + if ((locks == null) || (locks.isEmpty())) {
    + throw new HiveException("Database " + dbName + " is not locked ");
    + }
    +
    + for (HiveLock lock: locks) {
    + lockMgr.unlock(lock);
    +
    + }
    + return 0;
    + }
    +
    + /**
    + * Gets the lock manager and verifies if the explicit lock is supported
    + * @return the lock manager
    + * @throws HiveException
    + */
    + protected HiveLockManager getAndCheckLockManager() throws HiveException {
    + HiveLockManager lockMgr = getLockManager();
    + if (lockMgr == null) {
    + throw new HiveException("LockManager cannot be acquired");
    + }
    +
    + if (!supportsExplicitLock()) {
    + throw new HiveException(ErrorMsg.LOCK_REQUEST_UNSUPPORTED,
    + conf.getVar(HiveConf.ConfVars.HIVE_TXN_MANAGER));
    + }
    +
    + return lockMgr;
    + }
      }

Related Discussions

Discussion Navigation
viewthread | post
Discussion Overview
groupcommits @
categorieshive, hadoop
postedJul 14, '15 at 6:49p
activeJul 14, '15 at 6:51p
posts2
users1
websitehive.apache.org

1 user in discussion

Sunchao: 2 posts

People

Translate

site design / logo © 2021 Grokbase