HIVE-11130 - Refactoring the code so that HiveTxnManager interface will support lock/unlock table/database object (Aihua Xu, reviewed by Alan Gates and Chao Sun)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/46c76d6b Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/46c76d6b Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/46c76d6b Branch: refs/heads/beeline-cli Commit: 46c76d6b8e1fb323d1f8ac75061b6fd3bb747d6d Parents: 999e0e3 Author: Aihua Xu <aihu...@gmail.com> Authored: Tue Jul 14 11:49:09 2015 -0700 Committer: Chao Sun <chao....@cloudera.com> Committed: Tue Jul 14 11:49:09 2015 -0700 ---------------------------------------------------------------------- .../org/apache/hadoop/hive/ql/exec/DDLTask.java | 147 +------------------ .../hadoop/hive/ql/lockmgr/HiveLockObject.java | 35 +++++ .../hadoop/hive/ql/lockmgr/HiveTxnManager.java | 47 +++++- .../hive/ql/lockmgr/HiveTxnManagerImpl.java | 140 ++++++++++++++++++ 4 files changed, 227 insertions(+), 142 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/46c76d6b/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java index 049857b..a8c6aca 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java @@ -2444,8 +2444,8 @@ public class DDLTask extends Task<DDLWork> implements Serializable { locks = lockMgr.getLocks(false, isExt); } else { - locks = lockMgr.getLocks(getHiveObject(showLocks.getTableName(), - showLocks.getPartSpec()), + locks = lockMgr.getLocks(HiveLockObject.createFrom(db, + showLocks.getTableName(), showLocks.getPartSpec()), true, isExt); } @@ -2705,46 +2705,7 @@ public class DDLTask extends Task<DDLWork> implements Serializable { private int lockTable(LockTableDesc lockTbl) throws HiveException { Context ctx = driverContext.getCtx(); HiveTxnManager txnManager = ctx.getHiveTxnManager(); - if (!txnManager.supportsExplicitLock()) { - throw new HiveException(ErrorMsg.LOCK_REQUEST_UNSUPPORTED, - conf.getVar(HiveConf.ConfVars.HIVE_TXN_MANAGER)); - } - HiveLockManager lockMgr = txnManager.getLockManager(); - if (lockMgr == null) { - throw new HiveException("lock Table LockManager not specified"); - } - - HiveLockMode mode = HiveLockMode.valueOf(lockTbl.getMode()); - String tabName = lockTbl.getTableName(); - Table tbl = db.getTable(tabName); - if (tbl == null) { - throw new HiveException("Table " + tabName + " does not exist "); - } - - Map<String, String> partSpec = lockTbl.getPartSpec(); - HiveLockObjectData lockData = - new HiveLockObjectData(lockTbl.getQueryId(), - String.valueOf(System.currentTimeMillis()), - "EXPLICIT", - lockTbl.getQueryStr()); - - if (partSpec == null) { - HiveLock lck = lockMgr.lock(new HiveLockObject(tbl, lockData), mode, true); - if (lck == null) { - return 1; - } - return 0; - } - - Partition par = db.getPartition(tbl, partSpec, false); - if (par == null) { - throw new HiveException("Partition " + partSpec + " for table " + tabName + " does not exist"); - } - HiveLock lck = lockMgr.lock(new HiveLockObject(par, lockData), mode, true); - if (lck == null) { - return 1; - } - return 0; + return txnManager.lockTable(db, lockTbl); } /** @@ -2759,33 +2720,7 @@ public class DDLTask extends Task<DDLWork> implements Serializable { private int lockDatabase(LockDatabaseDesc lockDb) throws HiveException { Context ctx = driverContext.getCtx(); HiveTxnManager txnManager = ctx.getHiveTxnManager(); - if (!txnManager.supportsExplicitLock()) { - throw new HiveException(ErrorMsg.LOCK_REQUEST_UNSUPPORTED, - conf.getVar(HiveConf.ConfVars.HIVE_TXN_MANAGER)); - } - HiveLockManager lockMgr = txnManager.getLockManager(); - if (lockMgr == null) { - throw new HiveException("lock Database LockManager not specified"); - } - - HiveLockMode mode = HiveLockMode.valueOf(lockDb.getMode()); - String dbName = lockDb.getDatabaseName(); - - Database dbObj = db.getDatabase(dbName); - if (dbObj == null) { - throw new HiveException("Database " + dbName + " does not exist "); - } - - HiveLockObjectData lockData = - new HiveLockObjectData(lockDb.getQueryId(), - String.valueOf(System.currentTimeMillis()), - "EXPLICIT", lockDb.getQueryStr()); - - HiveLock lck = lockMgr.lock(new HiveLockObject(dbObj.getName(), lockData), mode, true); - if (lck == null) { - return 1; - } - return 0; + return txnManager.lockDatabase(db, lockDb); } /** @@ -2800,55 +2735,7 @@ public class DDLTask extends Task<DDLWork> implements Serializable { private int unlockDatabase(UnlockDatabaseDesc unlockDb) throws HiveException { Context ctx = driverContext.getCtx(); HiveTxnManager txnManager = ctx.getHiveTxnManager(); - if (!txnManager.supportsExplicitLock()) { - throw new HiveException(ErrorMsg.LOCK_REQUEST_UNSUPPORTED, - conf.getVar(HiveConf.ConfVars.HIVE_TXN_MANAGER)); - } - HiveLockManager lockMgr = txnManager.getLockManager(); - if (lockMgr == null) { - throw new HiveException("unlock Database LockManager not specified"); - } - - String dbName = unlockDb.getDatabaseName(); - - Database dbObj = db.getDatabase(dbName); - if (dbObj == null) { - throw new HiveException("Database " + dbName + " does not exist "); - } - HiveLockObject obj = new HiveLockObject(dbObj.getName(), null); - - List<HiveLock> locks = lockMgr.getLocks(obj, false, false); - if ((locks == null) || (locks.isEmpty())) { - throw new HiveException("Database " + dbName + " is not locked "); - } - - for (HiveLock lock: locks) { - lockMgr.unlock(lock); - - } - return 0; - } - - private HiveLockObject getHiveObject(String tabName, - Map<String, String> partSpec) throws HiveException { - Table tbl = db.getTable(tabName); - if (tbl == null) { - throw new HiveException("Table " + tabName + " does not exist "); - } - - HiveLockObject obj = null; - - if (partSpec == null) { - obj = new HiveLockObject(tbl, null); - } - else { - Partition par = db.getPartition(tbl, partSpec, false); - if (par == null) { - throw new HiveException("Partition " + partSpec + " for table " + tabName + " does not exist"); - } - obj = new HiveLockObject(par, null); - } - return obj; + return txnManager.unlockDatabase(db, unlockDb); } /** @@ -2863,29 +2750,7 @@ public class DDLTask extends Task<DDLWork> implements Serializable { private int unlockTable(UnlockTableDesc unlockTbl) throws HiveException { Context ctx = driverContext.getCtx(); HiveTxnManager txnManager = ctx.getHiveTxnManager(); - if (!txnManager.supportsExplicitLock()) { - throw new HiveException(ErrorMsg.LOCK_REQUEST_UNSUPPORTED, - conf.getVar(HiveConf.ConfVars.HIVE_TXN_MANAGER)); - } - HiveLockManager lockMgr = txnManager.getLockManager(); - if (lockMgr == null) { - throw new HiveException("unlock Table LockManager not specified"); - } - - String tabName = unlockTbl.getTableName(); - HiveLockObject obj = getHiveObject(tabName, unlockTbl.getPartSpec()); - - List<HiveLock> locks = lockMgr.getLocks(obj, false, false); - if ((locks == null) || (locks.isEmpty())) { - throw new HiveException("Table " + tabName + " is not locked "); - } - Iterator<HiveLock> locksIter = locks.iterator(); - while (locksIter.hasNext()) { - HiveLock lock = locksIter.next(); - lockMgr.unlock(lock); - } - - return 0; + return txnManager.unlockTable(db, unlockTbl); } /** http://git-wip-us.apache.org/repos/asf/hive/blob/46c76d6b/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/HiveLockObject.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/HiveLockObject.java b/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/HiveLockObject.java index 7e93387..fadd074 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/HiveLockObject.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/HiveLockObject.java @@ -19,9 +19,12 @@ package org.apache.hadoop.hive.ql.lockmgr; import java.util.Arrays; +import java.util.Map; import org.apache.commons.lang.builder.HashCodeBuilder; import org.apache.hadoop.hive.ql.metadata.DummyPartition; +import org.apache.hadoop.hive.ql.metadata.Hive; +import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.ql.metadata.Partition; import org.apache.hadoop.hive.ql.metadata.Table; @@ -196,6 +199,38 @@ public class HiveLockObject { this(new String[] {par.getName()}, lockData); } + /** + * Creates a locking object for a table (when partition spec is not provided) + * or a table partition + * @param hiveDB an object to communicate with the metastore + * @param tableName the table to create the locking object on + * @param partSpec the spec of a partition to create the locking object on + * @return the locking object + * @throws HiveException + */ + public static HiveLockObject createFrom(Hive hiveDB, String tableName, + Map<String, String> partSpec) throws HiveException { + Table tbl = hiveDB.getTable(tableName); + if (tbl == null) { + throw new HiveException("Table " + tableName + " does not exist "); + } + + HiveLockObject obj = null; + + if (partSpec == null) { + obj = new HiveLockObject(tbl, null); + } + else { + Partition par = hiveDB.getPartition(tbl, partSpec, false); + if (par == null) { + throw new HiveException("Partition " + partSpec + " for table " + + tableName + " does not exist"); + } + obj = new HiveLockObject(par, null); + } + return obj; + } + public String[] getPaths() { return pathNames; } http://git-wip-us.apache.org/repos/asf/hive/blob/46c76d6b/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/HiveTxnManager.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/HiveTxnManager.java b/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/HiveTxnManager.java index 6c3dc33..c900548 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/HiveTxnManager.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/HiveTxnManager.java @@ -18,9 +18,14 @@ package org.apache.hadoop.hive.ql.lockmgr; import org.apache.hadoop.hive.common.ValidTxnList; -import org.apache.hadoop.hive.metastore.IMetaStoreClient; import org.apache.hadoop.hive.ql.Context; import org.apache.hadoop.hive.ql.QueryPlan; +import org.apache.hadoop.hive.ql.metadata.Hive; +import org.apache.hadoop.hive.ql.metadata.HiveException; +import org.apache.hadoop.hive.ql.plan.LockDatabaseDesc; +import org.apache.hadoop.hive.ql.plan.LockTableDesc; +import org.apache.hadoop.hive.ql.plan.UnlockDatabaseDesc; +import org.apache.hadoop.hive.ql.plan.UnlockTableDesc; /** * An interface that allows Hive to manage transactions. All classes @@ -116,6 +121,46 @@ public interface HiveTxnManager { boolean supportsExplicitLock(); /** + * This function is called to lock the table when explicit lock command is + * issued on a table. + * @param hiveDB an object to communicate with the metastore + * @param lockTbl table locking info, such as table name, locking mode + * @return 0 if the locking succeeds, 1 otherwise. + * @throws HiveException + */ + int lockTable(Hive hiveDB, LockTableDesc lockTbl) throws HiveException; + + /** + * This function is called to unlock the table when explicit unlock command is + * issued on a table. + * @param hiveDB an object to communicate with the metastore + * @param unlockTbl table unlocking info, such as table name + * @return 0 if the locking succeeds, 1 otherwise. + * @throws HiveException + */ + int unlockTable(Hive hiveDB, UnlockTableDesc unlockTbl) throws HiveException; + + /** + * This function is called to lock the database when explicit lock command is + * issued on a database. + * @param hiveDB an object to communicate with the metastore + * @param lockDb database locking info, such as database name, locking mode + * @return 0 if the locking succeeds, 1 otherwise. + * @throws HiveException + */ + int lockDatabase(Hive hiveDB, LockDatabaseDesc lockDb) throws HiveException; + + /** + * This function is called to unlock the database when explicit unlock command + * is issued on a database. + * @param hiveDB an object to communicate with the metastore + * @param unlockDb database unlocking info, such as database name + * @return 0 if the locking succeeds, 1 otherwise. + * @throws HiveException + */ + int unlockDatabase(Hive hiveDB, UnlockDatabaseDesc unlockDb) throws HiveException; + + /** * Indicate whether this transaction manager returns information about locks in the new format * for show locks or the old one. * @return true if the new format should be used. http://git-wip-us.apache.org/repos/asf/hive/blob/46c76d6b/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/HiveTxnManagerImpl.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/HiveTxnManagerImpl.java b/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/HiveTxnManagerImpl.java index eccb8d1..ceeae68 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/HiveTxnManagerImpl.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/HiveTxnManagerImpl.java @@ -17,7 +17,22 @@ */ package org.apache.hadoop.hive.ql.lockmgr; +import java.util.Iterator; +import java.util.List; +import java.util.Map; + import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.metastore.api.Database; +import org.apache.hadoop.hive.ql.ErrorMsg; +import org.apache.hadoop.hive.ql.lockmgr.HiveLockObject.HiveLockObjectData; +import org.apache.hadoop.hive.ql.metadata.Hive; +import org.apache.hadoop.hive.ql.metadata.HiveException; +import org.apache.hadoop.hive.ql.metadata.Partition; +import org.apache.hadoop.hive.ql.metadata.Table; +import org.apache.hadoop.hive.ql.plan.LockDatabaseDesc; +import org.apache.hadoop.hive.ql.plan.LockTableDesc; +import org.apache.hadoop.hive.ql.plan.UnlockDatabaseDesc; +import org.apache.hadoop.hive.ql.plan.UnlockTableDesc; /** * An implementation HiveTxnManager that includes internal methods that all @@ -44,4 +59,129 @@ abstract class HiveTxnManagerImpl implements HiveTxnManager { destruct(); } + @Override + public int lockTable(Hive db, LockTableDesc lockTbl) throws HiveException { + HiveLockManager lockMgr = getAndCheckLockManager(); + + HiveLockMode mode = HiveLockMode.valueOf(lockTbl.getMode()); + String tabName = lockTbl.getTableName(); + Table tbl = db.getTable(tabName); + if (tbl == null) { + throw new HiveException("Table " + tabName + " does not exist "); + } + + Map<String, String> partSpec = lockTbl.getPartSpec(); + HiveLockObjectData lockData = + new HiveLockObjectData(lockTbl.getQueryId(), + String.valueOf(System.currentTimeMillis()), + "EXPLICIT", + lockTbl.getQueryStr()); + + if (partSpec == null) { + HiveLock lck = lockMgr.lock(new HiveLockObject(tbl, lockData), mode, true); + if (lck == null) { + return 1; + } + return 0; + } + + Partition par = db.getPartition(tbl, partSpec, false); + if (par == null) { + throw new HiveException("Partition " + partSpec + " for table " + + tabName + " does not exist"); + } + HiveLock lck = lockMgr.lock(new HiveLockObject(par, lockData), mode, true); + if (lck == null) { + return 1; + } + return 0; + } + + @Override + public int unlockTable(Hive hiveDB, UnlockTableDesc unlockTbl) throws HiveException { + HiveLockManager lockMgr = getAndCheckLockManager(); + + String tabName = unlockTbl.getTableName(); + HiveLockObject obj = HiveLockObject.createFrom(hiveDB, tabName, + unlockTbl.getPartSpec()); + + List<HiveLock> locks = lockMgr.getLocks(obj, false, false); + if ((locks == null) || (locks.isEmpty())) { + throw new HiveException("Table " + tabName + " is not locked "); + } + Iterator<HiveLock> locksIter = locks.iterator(); + while (locksIter.hasNext()) { + HiveLock lock = locksIter.next(); + lockMgr.unlock(lock); + } + + return 0; + } + + @Override + public int lockDatabase(Hive hiveDB, LockDatabaseDesc lockDb) throws HiveException { + HiveLockManager lockMgr = getAndCheckLockManager(); + + HiveLockMode mode = HiveLockMode.valueOf(lockDb.getMode()); + String dbName = lockDb.getDatabaseName(); + + Database dbObj = hiveDB.getDatabase(dbName); + if (dbObj == null) { + throw new HiveException("Database " + dbName + " does not exist "); + } + + HiveLockObjectData lockData = + new HiveLockObjectData(lockDb.getQueryId(), + String.valueOf(System.currentTimeMillis()), + "EXPLICIT", lockDb.getQueryStr()); + + HiveLock lck = lockMgr.lock(new HiveLockObject(dbObj.getName(), lockData), mode, true); + if (lck == null) { + return 1; + } + return 0; + } + + @Override + public int unlockDatabase(Hive hiveDB, UnlockDatabaseDesc unlockDb) throws HiveException { + HiveLockManager lockMgr = getAndCheckLockManager(); + + String dbName = unlockDb.getDatabaseName(); + + Database dbObj = hiveDB.getDatabase(dbName); + if (dbObj == null) { + throw new HiveException("Database " + dbName + " does not exist "); + } + HiveLockObject obj = new HiveLockObject(dbObj.getName(), null); + + List<HiveLock> locks = lockMgr.getLocks(obj, false, false); + if ((locks == null) || (locks.isEmpty())) { + throw new HiveException("Database " + dbName + " is not locked "); + } + + for (HiveLock lock: locks) { + lockMgr.unlock(lock); + + } + return 0; + } + + /** + * Gets the lock manager and verifies if the explicit lock is supported + * @return the lock manager + * @throws HiveException + */ + protected HiveLockManager getAndCheckLockManager() throws HiveException { + HiveLockManager lockMgr = getLockManager(); + if (lockMgr == null) { + throw new HiveException("LockManager cannot be acquired"); + } + + if (!supportsExplicitLock()) { + throw new HiveException(ErrorMsg.LOCK_REQUEST_UNSUPPORTED, + conf.getVar(HiveConf.ConfVars.HIVE_TXN_MANAGER)); + } + + return lockMgr; + } }