hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From w...@apache.org
Subject hive git commit: HIVE-15774 : Ensure DbLockManager backward compatibility for non-ACID resources (Wei Zheng, reviewed by Eugene Koifman)
Date Thu, 02 Feb 2017 22:07:40 GMT
Repository: hive
Updated Branches:
  refs/heads/master ed3b05185 -> aa20b777e


HIVE-15774 : Ensure DbLockManager backward compatibility for non-ACID resources (Wei Zheng,
reviewed by Eugene Koifman)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/aa20b777
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/aa20b777
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/aa20b777

Branch: refs/heads/master
Commit: aa20b777e87dd1e5e4fadb26680aac200fb10060
Parents: ed3b051
Author: Wei Zheng <weiz@apache.org>
Authored: Thu Feb 2 14:07:26 2017 -0800
Committer: Wei Zheng <weiz@apache.org>
Committed: Thu Feb 2 14:07:26 2017 -0800

----------------------------------------------------------------------
 .../org/apache/hadoop/hive/conf/HiveConf.java    |  5 +++++
 .../hadoop/hive/ql/lockmgr/DbTxnManager.java     |  8 ++++++--
 .../hive/ql/lockmgr/TestDbTxnManager2.java       | 19 +++++++++++++++++++
 3 files changed, 30 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/aa20b777/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
----------------------------------------------------------------------
diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
index 29de605..53b9b0c 100644
--- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
+++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
@@ -1832,6 +1832,11 @@ public class HiveConf extends Configuration {
         "(true), and hive.exec.dynamic.partition.mode (nonstrict).\n" +
         "The default DummyTxnManager replicates pre-Hive-0.13 behavior and provides\n" +
         "no transactions."),
+    HIVE_TXN_STRICT_LOCKING_MODE("hive.txn.strict.locking.mode", true, "In strict mode non-ACID\n"
+
+        "resources use standard R/W lock semantics, e.g. INSERT will acquire exclusive lock.\n"
+
+        "In nonstrict mode, for non-ACID resources, INSERT will only acquire shared lock,
which\n" +
+        "allows two concurrent writes to the same partition but still lets lock manager prevent\n"
+
+        "DROP TABLE etc. when the table is being written to"),
     HIVE_TXN_TIMEOUT("hive.txn.timeout", "300s", new TimeValidator(TimeUnit.SECONDS),
         "time after which transactions are declared aborted if the client has not sent a
heartbeat."),
     HIVE_TXN_HEARTBEAT_THREADPOOL_SIZE("hive.txn.heartbeat.threadpool.size", 5, "The number
of " +

http://git-wip-us.apache.org/repos/asf/hive/blob/aa20b777/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/DbTxnManager.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/DbTxnManager.java b/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/DbTxnManager.java
index 28ca77c..a985eb1 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/DbTxnManager.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/DbTxnManager.java
@@ -250,6 +250,7 @@ public class DbTxnManager extends HiveTxnManagerImpl {
     // overwrite) than we need a shared.  If it's update or delete then we
     // need a SEMI-SHARED.
     for (WriteEntity output : plan.getOutputs()) {
+      LOG.debug("output is null " + (output == null));
       if (output.getType() == Entity.Type.DFS_DIR || output.getType() == Entity.Type.LOCAL_DIR
||
           (output.getType() == Entity.Type.TABLE && output.getTable().isTemporary()))
{
         // We don't lock files or directories. We also skip locking temp tables.
@@ -257,7 +258,6 @@ public class DbTxnManager extends HiveTxnManagerImpl {
       }
       LockComponentBuilder compBuilder = new LockComponentBuilder();
       Table t = null;
-      LOG.debug("output is null " + (output == null));
       switch (output.getWriteType()) {
         case DDL_EXCLUSIVE:
         case INSERT_OVERWRITE:
@@ -272,7 +272,11 @@ public class DbTxnManager extends HiveTxnManagerImpl {
             compBuilder.setIsAcid(true);
           }
           else {
-            compBuilder.setExclusive();
+            if (conf.getBoolVar(HiveConf.ConfVars.HIVE_TXN_STRICT_LOCKING_MODE)) {
+              compBuilder.setExclusive();
+            } else {  // this is backward compatible for non-ACID resources, w/o ACID semantics
+              compBuilder.setShared();
+            }
             compBuilder.setIsAcid(false);
           }
           compBuilder.setOperationType(DataOperationType.INSERT);

http://git-wip-us.apache.org/repos/asf/hive/blob/aa20b777/ql/src/test/org/apache/hadoop/hive/ql/lockmgr/TestDbTxnManager2.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/lockmgr/TestDbTxnManager2.java b/ql/src/test/org/apache/hadoop/hive/ql/lockmgr/TestDbTxnManager2.java
index 384f6eb..180eea6 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/lockmgr/TestDbTxnManager2.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/lockmgr/TestDbTxnManager2.java
@@ -687,6 +687,25 @@ public class TestDbTxnManager2 {
     checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "tab_not_acid", "np=blah",
locks);
     checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "tab_not_acid", "np=doh",
locks);
     checkLock(LockType.EXCLUSIVE, LockState.WAITING, "default", "tab_not_acid", "np=doh",
locks);
+
+    // Test strict locking mode, i.e. backward compatible locking mode for non-ACID resources.
+    // With non-strict mode, INSERT got SHARED_READ lock, instead of EXCLUSIVE with ACID
semantics
+    conf.setBoolVar(HiveConf.ConfVars.HIVE_TXN_STRICT_LOCKING_MODE, false);
+    HiveTxnManager txnMgr3 = TxnManagerFactory.getTxnManagerFactory().getTxnManager(conf);
+    txnMgr3.openTxn(ctx, "T3");
+    checkCmdOnDriver(driver.compileAndRespond("insert into tab_not_acid partition(np='blah')
values(7,8)"));
+    ((DbTxnManager)txnMgr3).acquireLocks(driver.getPlan(), ctx, "T3", false);
+    locks = getLocks(txnMgr3);
+    Assert.assertEquals("Unexpected lock count", 8, locks.size());
+    checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "tab_acid", null, locks);
+    checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "tab_acid", "p=bar", locks);
+    checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "tab_acid", "p=foo", locks);
+    checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "tab_not_acid", null,
locks);
+    checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "tab_not_acid", "np=blah",
locks);
+    checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "tab_not_acid", "np=doh",
locks);
+    checkLock(LockType.EXCLUSIVE, LockState.WAITING, "default", "tab_not_acid", "np=doh",
locks);
+    checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "tab_not_acid", "np=blah",
locks);
+    conf.setBoolVar(HiveConf.ConfVars.HIVE_TXN_STRICT_LOCKING_MODE, true);
   }
 
   /** The list is small, and the object is generated, so we don't use sets/equals/etc. */


Mime
View raw message