hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From sunc...@apache.org
Subject hive git commit: HIVE-11130 - Refactoring the code so that HiveTxnManager interface will support lock/unlock table/database object (Aihua Xu, reviewed by Alan Gates and Chao Sun)
Date Tue, 14 Jul 2015 18:51:00 GMT
Repository: hive
Updated Branches:
  refs/heads/branch-1 0c16534b3 -> da0c08fd0


HIVE-11130 - Refactoring the code so that HiveTxnManager interface will support lock/unlock
table/database object (Aihua Xu, reviewed by Alan Gates and Chao Sun)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/da0c08fd
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/da0c08fd
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/da0c08fd

Branch: refs/heads/branch-1
Commit: da0c08fd040c1ab46480a0bcd37b27973ac79c49
Parents: 0c16534
Author: Aihua Xu <aihuaxu@gmail.com>
Authored: Tue Jul 14 11:50:24 2015 -0700
Committer: Chao Sun <chao.sun@cloudera.com>
Committed: Tue Jul 14 11:50:24 2015 -0700

----------------------------------------------------------------------
 .../org/apache/hadoop/hive/ql/exec/DDLTask.java | 147 +------------------
 .../hadoop/hive/ql/lockmgr/HiveLockObject.java  |  35 +++++
 .../hadoop/hive/ql/lockmgr/HiveTxnManager.java  |  47 +++++-
 .../hive/ql/lockmgr/HiveTxnManagerImpl.java     | 140 ++++++++++++++++++
 4 files changed, 227 insertions(+), 142 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/da0c08fd/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
index 049857b..a8c6aca 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
@@ -2444,8 +2444,8 @@ public class DDLTask extends Task<DDLWork> implements Serializable
{
         locks = lockMgr.getLocks(false, isExt);
       }
       else {
-        locks = lockMgr.getLocks(getHiveObject(showLocks.getTableName(),
-            showLocks.getPartSpec()),
+        locks = lockMgr.getLocks(HiveLockObject.createFrom(db,
+            showLocks.getTableName(), showLocks.getPartSpec()),
             true, isExt);
       }
 
@@ -2705,46 +2705,7 @@ public class DDLTask extends Task<DDLWork> implements Serializable
{
   private int lockTable(LockTableDesc lockTbl) throws HiveException {
     Context ctx = driverContext.getCtx();
     HiveTxnManager txnManager = ctx.getHiveTxnManager();
-    if (!txnManager.supportsExplicitLock()) {
-      throw new HiveException(ErrorMsg.LOCK_REQUEST_UNSUPPORTED,
-          conf.getVar(HiveConf.ConfVars.HIVE_TXN_MANAGER));
-    }
-    HiveLockManager lockMgr = txnManager.getLockManager();
-    if (lockMgr == null) {
-      throw new HiveException("lock Table LockManager not specified");
-    }
-
-    HiveLockMode mode = HiveLockMode.valueOf(lockTbl.getMode());
-    String tabName = lockTbl.getTableName();
-    Table  tbl = db.getTable(tabName);
-    if (tbl == null) {
-      throw new HiveException("Table " + tabName + " does not exist ");
-    }
-
-    Map<String, String> partSpec = lockTbl.getPartSpec();
-    HiveLockObjectData lockData =
-        new HiveLockObjectData(lockTbl.getQueryId(),
-            String.valueOf(System.currentTimeMillis()),
-            "EXPLICIT",
-            lockTbl.getQueryStr());
-
-    if (partSpec == null) {
-      HiveLock lck = lockMgr.lock(new HiveLockObject(tbl, lockData), mode, true);
-      if (lck == null) {
-        return 1;
-      }
-      return 0;
-    }
-
-    Partition par = db.getPartition(tbl, partSpec, false);
-    if (par == null) {
-      throw new HiveException("Partition " + partSpec + " for table " + tabName + " does
not exist");
-    }
-    HiveLock lck = lockMgr.lock(new HiveLockObject(par, lockData), mode, true);
-    if (lck == null) {
-      return 1;
-    }
-    return 0;
+    return txnManager.lockTable(db, lockTbl);
   }
 
   /**
@@ -2759,33 +2720,7 @@ public class DDLTask extends Task<DDLWork> implements Serializable
{
   private int lockDatabase(LockDatabaseDesc lockDb) throws HiveException {
     Context ctx = driverContext.getCtx();
     HiveTxnManager txnManager = ctx.getHiveTxnManager();
-    if (!txnManager.supportsExplicitLock()) {
-      throw new HiveException(ErrorMsg.LOCK_REQUEST_UNSUPPORTED,
-          conf.getVar(HiveConf.ConfVars.HIVE_TXN_MANAGER));
-    }
-    HiveLockManager lockMgr = txnManager.getLockManager();
-    if (lockMgr == null) {
-      throw new HiveException("lock Database LockManager not specified");
-    }
-
-    HiveLockMode mode = HiveLockMode.valueOf(lockDb.getMode());
-    String dbName = lockDb.getDatabaseName();
-
-    Database dbObj = db.getDatabase(dbName);
-    if (dbObj == null) {
-      throw new HiveException("Database " + dbName + " does not exist ");
-    }
-
-    HiveLockObjectData lockData =
-        new HiveLockObjectData(lockDb.getQueryId(),
-            String.valueOf(System.currentTimeMillis()),
-            "EXPLICIT", lockDb.getQueryStr());
-
-    HiveLock lck = lockMgr.lock(new HiveLockObject(dbObj.getName(), lockData), mode, true);
-    if (lck == null) {
-      return 1;
-    }
-    return 0;
+    return txnManager.lockDatabase(db, lockDb);
   }
 
   /**
@@ -2800,55 +2735,7 @@ public class DDLTask extends Task<DDLWork> implements Serializable
{
   private int unlockDatabase(UnlockDatabaseDesc unlockDb) throws HiveException {
     Context ctx = driverContext.getCtx();
     HiveTxnManager txnManager = ctx.getHiveTxnManager();
-    if (!txnManager.supportsExplicitLock()) {
-      throw new HiveException(ErrorMsg.LOCK_REQUEST_UNSUPPORTED,
-          conf.getVar(HiveConf.ConfVars.HIVE_TXN_MANAGER));
-    }
-    HiveLockManager lockMgr = txnManager.getLockManager();
-    if (lockMgr == null) {
-      throw new HiveException("unlock Database LockManager not specified");
-    }
-
-    String dbName = unlockDb.getDatabaseName();
-
-    Database dbObj = db.getDatabase(dbName);
-    if (dbObj == null) {
-      throw new HiveException("Database " + dbName + " does not exist ");
-    }
-    HiveLockObject obj = new HiveLockObject(dbObj.getName(), null);
-
-    List<HiveLock> locks = lockMgr.getLocks(obj, false, false);
-    if ((locks == null) || (locks.isEmpty())) {
-      throw new HiveException("Database " + dbName + " is not locked ");
-    }
-
-    for (HiveLock lock: locks) {
-      lockMgr.unlock(lock);
-
-    }
-    return 0;
-  }
-
-  private HiveLockObject getHiveObject(String tabName,
-      Map<String, String> partSpec) throws HiveException {
-    Table  tbl = db.getTable(tabName);
-    if (tbl == null) {
-      throw new HiveException("Table " + tabName + " does not exist ");
-    }
-
-    HiveLockObject obj = null;
-
-    if  (partSpec == null) {
-      obj = new HiveLockObject(tbl, null);
-    }
-    else {
-      Partition par = db.getPartition(tbl, partSpec, false);
-      if (par == null) {
-        throw new HiveException("Partition " + partSpec + " for table " + tabName + " does
not exist");
-      }
-      obj = new HiveLockObject(par, null);
-    }
-    return obj;
+    return txnManager.unlockDatabase(db, unlockDb);
   }
 
   /**
@@ -2863,29 +2750,7 @@ public class DDLTask extends Task<DDLWork> implements Serializable
{
   private int unlockTable(UnlockTableDesc unlockTbl) throws HiveException {
     Context ctx = driverContext.getCtx();
     HiveTxnManager txnManager = ctx.getHiveTxnManager();
-    if (!txnManager.supportsExplicitLock()) {
-      throw new HiveException(ErrorMsg.LOCK_REQUEST_UNSUPPORTED,
-          conf.getVar(HiveConf.ConfVars.HIVE_TXN_MANAGER));
-    }
-    HiveLockManager lockMgr = txnManager.getLockManager();
-    if (lockMgr == null) {
-      throw new HiveException("unlock Table LockManager not specified");
-    }
-
-    String tabName = unlockTbl.getTableName();
-    HiveLockObject obj = getHiveObject(tabName, unlockTbl.getPartSpec());
-
-    List<HiveLock> locks = lockMgr.getLocks(obj, false, false);
-    if ((locks == null) || (locks.isEmpty())) {
-      throw new HiveException("Table " + tabName + " is not locked ");
-    }
-    Iterator<HiveLock> locksIter = locks.iterator();
-    while (locksIter.hasNext()) {
-      HiveLock lock = locksIter.next();
-      lockMgr.unlock(lock);
-    }
-
-    return 0;
+    return txnManager.unlockTable(db, unlockTbl);
   }
 
   /**

http://git-wip-us.apache.org/repos/asf/hive/blob/da0c08fd/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/HiveLockObject.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/HiveLockObject.java b/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/HiveLockObject.java
index 7e93387..fadd074 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/HiveLockObject.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/HiveLockObject.java
@@ -19,9 +19,12 @@
 package org.apache.hadoop.hive.ql.lockmgr;
 
 import java.util.Arrays;
+import java.util.Map;
 
 import org.apache.commons.lang.builder.HashCodeBuilder;
 import org.apache.hadoop.hive.ql.metadata.DummyPartition;
+import org.apache.hadoop.hive.ql.metadata.Hive;
+import org.apache.hadoop.hive.ql.metadata.HiveException;
 import org.apache.hadoop.hive.ql.metadata.Partition;
 import org.apache.hadoop.hive.ql.metadata.Table;
 
@@ -196,6 +199,38 @@ public class HiveLockObject {
     this(new String[] {par.getName()}, lockData);
   }
 
+  /**
+   * Creates a locking object for a table (when partition spec is not provided)
+   * or a table partition
+   * @param hiveDB    an object to communicate with the metastore
+   * @param tableName the table to create the locking object on
+   * @param partSpec  the spec of a partition to create the locking object on
+   * @return  the locking object
+   * @throws HiveException
+   */
+  public static HiveLockObject createFrom(Hive hiveDB, String tableName,
+      Map<String, String> partSpec) throws HiveException {
+    Table  tbl = hiveDB.getTable(tableName);
+    if (tbl == null) {
+      throw new HiveException("Table " + tableName + " does not exist ");
+    }
+
+    HiveLockObject obj = null;
+
+    if  (partSpec == null) {
+      obj = new HiveLockObject(tbl, null);
+    }
+    else {
+      Partition par = hiveDB.getPartition(tbl, partSpec, false);
+      if (par == null) {
+        throw new HiveException("Partition " + partSpec + " for table " +
+            tableName + " does not exist");
+      }
+      obj = new HiveLockObject(par, null);
+    }
+    return obj;
+  }
+
   public String[] getPaths() {
     return pathNames;
   }

http://git-wip-us.apache.org/repos/asf/hive/blob/da0c08fd/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/HiveTxnManager.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/HiveTxnManager.java b/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/HiveTxnManager.java
index 6c3dc33..c900548 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/HiveTxnManager.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/HiveTxnManager.java
@@ -18,9 +18,14 @@
 package org.apache.hadoop.hive.ql.lockmgr;
 
 import org.apache.hadoop.hive.common.ValidTxnList;
-import org.apache.hadoop.hive.metastore.IMetaStoreClient;
 import org.apache.hadoop.hive.ql.Context;
 import org.apache.hadoop.hive.ql.QueryPlan;
+import org.apache.hadoop.hive.ql.metadata.Hive;
+import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.ql.plan.LockDatabaseDesc;
+import org.apache.hadoop.hive.ql.plan.LockTableDesc;
+import org.apache.hadoop.hive.ql.plan.UnlockDatabaseDesc;
+import org.apache.hadoop.hive.ql.plan.UnlockTableDesc;
 
 /**
  * An interface that allows Hive to manage transactions.  All classes
@@ -116,6 +121,46 @@ public interface HiveTxnManager {
   boolean supportsExplicitLock();
 
   /**
+   * This function is called to lock the table when explicit lock command is
+   * issued on a table.
+   * @param hiveDB    an object to communicate with the metastore
+   * @param lockTbl   table locking info, such as table name, locking mode
+   * @return 0 if the locking succeeds, 1 otherwise.
+   * @throws HiveException
+   */
+  int lockTable(Hive hiveDB, LockTableDesc lockTbl) throws HiveException;
+
+  /**
+   * This function is called to unlock the table when explicit unlock command is
+   * issued on a table.
+   * @param hiveDB    an object to communicate with the metastore
+   * @param unlockTbl table unlocking info, such as table name
+   * @return 0 if the locking succeeds, 1 otherwise.
+   * @throws HiveException
+   */
+  int unlockTable(Hive hiveDB, UnlockTableDesc unlockTbl) throws HiveException;
+
+  /**
+   * This function is called to lock the database when explicit lock command is
+   * issued on a database.
+   * @param hiveDB    an object to communicate with the metastore
+   * @param lockDb    database locking info, such as database name, locking mode
+   * @return 0 if the locking succeeds, 1 otherwise.
+   * @throws HiveException
+   */
+  int lockDatabase(Hive hiveDB, LockDatabaseDesc lockDb) throws HiveException;
+
+  /**
+   * This function is called to unlock the database when explicit unlock command
+   * is issued on a database.
+   * @param hiveDB    an object to communicate with the metastore
+   * @param unlockDb  database unlocking info, such as database name
+   * @return 0 if the locking succeeds, 1 otherwise.
+   * @throws HiveException
+   */
+  int unlockDatabase(Hive hiveDB, UnlockDatabaseDesc unlockDb) throws HiveException;
+
+  /**
    * Indicate whether this transaction manager returns information about locks in the new
format
    * for show locks or the old one.
    * @return true if the new format should be used.

http://git-wip-us.apache.org/repos/asf/hive/blob/da0c08fd/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/HiveTxnManagerImpl.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/HiveTxnManagerImpl.java b/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/HiveTxnManagerImpl.java
index eccb8d1..ceeae68 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/HiveTxnManagerImpl.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/HiveTxnManagerImpl.java
@@ -17,7 +17,22 @@
  */
 package org.apache.hadoop.hive.ql.lockmgr;
 
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+
 import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.metastore.api.Database;
+import org.apache.hadoop.hive.ql.ErrorMsg;
+import org.apache.hadoop.hive.ql.lockmgr.HiveLockObject.HiveLockObjectData;
+import org.apache.hadoop.hive.ql.metadata.Hive;
+import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.ql.metadata.Partition;
+import org.apache.hadoop.hive.ql.metadata.Table;
+import org.apache.hadoop.hive.ql.plan.LockDatabaseDesc;
+import org.apache.hadoop.hive.ql.plan.LockTableDesc;
+import org.apache.hadoop.hive.ql.plan.UnlockDatabaseDesc;
+import org.apache.hadoop.hive.ql.plan.UnlockTableDesc;
 
 /**
  * An implementation HiveTxnManager that includes internal methods that all
@@ -44,4 +59,129 @@ abstract class HiveTxnManagerImpl implements HiveTxnManager {
     destruct();
   }
 
+  @Override
+  public int lockTable(Hive db, LockTableDesc lockTbl) throws HiveException {
+    HiveLockManager lockMgr = getAndCheckLockManager();
+
+    HiveLockMode mode = HiveLockMode.valueOf(lockTbl.getMode());
+    String tabName = lockTbl.getTableName();
+    Table  tbl = db.getTable(tabName);
+    if (tbl == null) {
+      throw new HiveException("Table " + tabName + " does not exist ");
+    }
+
+    Map<String, String> partSpec = lockTbl.getPartSpec();
+    HiveLockObjectData lockData =
+        new HiveLockObjectData(lockTbl.getQueryId(),
+            String.valueOf(System.currentTimeMillis()),
+            "EXPLICIT",
+            lockTbl.getQueryStr());
+
+    if (partSpec == null) {
+      HiveLock lck = lockMgr.lock(new HiveLockObject(tbl, lockData), mode, true);
+      if (lck == null) {
+        return 1;
+      }
+      return 0;
+    }
+
+    Partition par = db.getPartition(tbl, partSpec, false);
+    if (par == null) {
+      throw new HiveException("Partition " + partSpec + " for table " +
+          tabName + " does not exist");
+    }
+    HiveLock lck = lockMgr.lock(new HiveLockObject(par, lockData), mode, true);
+    if (lck == null) {
+      return 1;
+    }
+    return 0;
+  }
+
+  @Override
+  public int unlockTable(Hive hiveDB, UnlockTableDesc unlockTbl) throws HiveException {
+    HiveLockManager lockMgr = getAndCheckLockManager();
+
+    String tabName = unlockTbl.getTableName();
+    HiveLockObject obj = HiveLockObject.createFrom(hiveDB, tabName,
+        unlockTbl.getPartSpec());
+
+    List<HiveLock> locks = lockMgr.getLocks(obj, false, false);
+    if ((locks == null) || (locks.isEmpty())) {
+      throw new HiveException("Table " + tabName + " is not locked ");
+    }
+    Iterator<HiveLock> locksIter = locks.iterator();
+    while (locksIter.hasNext()) {
+      HiveLock lock = locksIter.next();
+      lockMgr.unlock(lock);
+    }
+
+    return 0;
+  }
+
+  @Override
+  public int lockDatabase(Hive hiveDB, LockDatabaseDesc lockDb) throws HiveException {
+    HiveLockManager lockMgr = getAndCheckLockManager();
+
+    HiveLockMode mode = HiveLockMode.valueOf(lockDb.getMode());
+    String dbName = lockDb.getDatabaseName();
+
+    Database dbObj = hiveDB.getDatabase(dbName);
+    if (dbObj == null) {
+      throw new HiveException("Database " + dbName + " does not exist ");
+    }
+
+    HiveLockObjectData lockData =
+        new HiveLockObjectData(lockDb.getQueryId(),
+            String.valueOf(System.currentTimeMillis()),
+            "EXPLICIT", lockDb.getQueryStr());
+
+    HiveLock lck = lockMgr.lock(new HiveLockObject(dbObj.getName(), lockData), mode, true);
+    if (lck == null) {
+      return 1;
+    }
+    return 0;
+  }
+
+  @Override
+  public int unlockDatabase(Hive hiveDB, UnlockDatabaseDesc unlockDb) throws HiveException
{
+    HiveLockManager lockMgr = getAndCheckLockManager();
+
+    String dbName = unlockDb.getDatabaseName();
+
+    Database dbObj = hiveDB.getDatabase(dbName);
+    if (dbObj == null) {
+      throw new HiveException("Database " + dbName + " does not exist ");
+    }
+    HiveLockObject obj = new HiveLockObject(dbObj.getName(), null);
+
+    List<HiveLock> locks = lockMgr.getLocks(obj, false, false);
+    if ((locks == null) || (locks.isEmpty())) {
+      throw new HiveException("Database " + dbName + " is not locked ");
+    }
+
+    for (HiveLock lock: locks) {
+      lockMgr.unlock(lock);
+
+    }
+    return 0;
+  }
+
+  /**
+   * Gets the lock manager and verifies if the explicit lock is supported
+   * @return  the lock manager
+   * @throws HiveException
+   */
+  protected HiveLockManager getAndCheckLockManager() throws HiveException {
+    HiveLockManager lockMgr = getLockManager();
+    if (lockMgr == null) {
+      throw new HiveException("LockManager cannot be acquired");
+    }
+
+    if (!supportsExplicitLock()) {
+      throw new HiveException(ErrorMsg.LOCK_REQUEST_UNSUPPORTED,
+          conf.getVar(HiveConf.ConfVars.HIVE_TXN_MANAGER));
+    }
+
+    return lockMgr;
+  }
 }


Mime
View raw message