hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From hashut...@apache.org
Subject hive git commit: HIVE-12367 : Lock/unlock database should add current database to inputs and outputs of authz hook (Dapeng Sun via Ashutosh Chauhan)
Date Fri, 25 Mar 2016 02:23:01 GMT
Repository: hive
Updated Branches:
  refs/heads/master a71edcf6a -> 6bfec2e97


HIVE-12367 : Lock/unlock database should add current database to inputs and outputs of authz
hook (Dapeng Sun via Ashutosh Chauhan)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/6bfec2e9
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/6bfec2e9
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/6bfec2e9

Branch: refs/heads/master
Commit: 6bfec2e97c4e434646aa9aeffd98c9939313fa6e
Parents: a71edcf
Author: Dapeng Sun <sdp@apache.org>
Authored: Tue Dec 15 19:39:00 2015 -0800
Committer: Ashutosh Chauhan <hashutosh@apache.org>
Committed: Thu Mar 24 19:22:28 2016 -0700

----------------------------------------------------------------------
 .../java/org/apache/hadoop/hive/ql/Driver.java  | 20 ++++++++++++++++++++
 .../hive/ql/parse/DDLSemanticAnalyzer.java      | 15 +++++++++++++--
 .../clientnegative/dbtxnmgr_nodblock.q.out      |  2 ++
 .../clientnegative/dbtxnmgr_nodbunlock.q.out    |  2 ++
 .../lockneg_query_tbl_in_locked_db.q.out        |  6 ++++++
 .../lockneg_try_db_lock_conflict.q.out          |  6 ++++++
 .../lockneg_try_drop_locked_db.q.out            |  4 ++++
 .../lockneg_try_lock_db_in_use.q.out            |  6 ++++++
 8 files changed, 59 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/6bfec2e9/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/Driver.java b/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
index d7e4ac7..7276e31 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
@@ -1416,6 +1416,10 @@ public class Driver implements CommandProcessor {
     if (!checkConcurrency()) {
       return false;
     }
+    // Lock operations themselves don't require the lock.
+    if (isExplicitLockOperation()){
+      return false;
+    }
     if (!HiveConf.getBoolVar(conf, ConfVars.HIVE_LOCK_MAPRED_ONLY)) {
       return true;
     }
@@ -1438,6 +1442,22 @@ public class Driver implements CommandProcessor {
     return false;
   }
 
+  private boolean isExplicitLockOperation() {
+    HiveOperation currentOpt = plan.getOperation();
+    if (currentOpt != null) {
+      switch (currentOpt) {
+      case LOCKDB:
+      case UNLOCKDB:
+      case LOCKTABLE:
+      case UNLOCKTABLE:
+        return true;
+      default:
+        return false;
+      }
+    }
+    return false;
+  }
+
   private CommandProcessorResponse createProcessorResponse(int ret) {
     queryDisplay.setErrorMessage(errorMessage);
     return new CommandProcessorResponse(ret, errorMessage, SQLState, downstreamError);

http://git-wip-us.apache.org/repos/asf/hive/blob/6bfec2e9/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java
index 0c087ed..fe9b8cc 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java
@@ -2444,8 +2444,12 @@ public class DDLSemanticAnalyzer extends BaseSemanticAnalyzer {
     String dbName = unescapeIdentifier(ast.getChild(0).getText());
     String mode  = unescapeIdentifier(ast.getChild(1).getText().toUpperCase());
 
-    //inputs.add(new ReadEntity(dbName));
-    //outputs.add(new WriteEntity(dbName));
+    inputs.add(new ReadEntity(getDatabase(dbName)));
+    // Lock database operation is to acquire the lock explicitly, the operation
+    // itself doesn't need to be locked. Set the WriteEntity as WriteType:
+    // DDL_NO_LOCK here, otherwise it will conflict with Hive's transaction.
+    outputs.add(new WriteEntity(getDatabase(dbName), WriteType.DDL_NO_LOCK));
+
     LockDatabaseDesc lockDatabaseDesc = new LockDatabaseDesc(dbName, mode,
                         HiveConf.getVar(conf, ConfVars.HIVEQUERYID));
     lockDatabaseDesc.setQueryStr(ctx.getCmd());
@@ -2457,6 +2461,13 @@ public class DDLSemanticAnalyzer extends BaseSemanticAnalyzer {
   private void analyzeUnlockDatabase(ASTNode ast) throws SemanticException {
     String dbName = unescapeIdentifier(ast.getChild(0).getText());
 
+    inputs.add(new ReadEntity(getDatabase(dbName)));
+    // Unlock database operation is to release the lock explicitly, the
+    // operation itself don't need to be locked. Set the WriteEntity as
+    // WriteType: DDL_NO_LOCK here, otherwise it will conflict with
+    // Hive's transaction.
+    outputs.add(new WriteEntity(getDatabase(dbName), WriteType.DDL_NO_LOCK));
+
     UnlockDatabaseDesc unlockDatabaseDesc = new UnlockDatabaseDesc(dbName);
     DDLWork work = new DDLWork(getInputs(), getOutputs(), unlockDatabaseDesc);
     rootTasks.add(TaskFactory.get(work, conf));

http://git-wip-us.apache.org/repos/asf/hive/blob/6bfec2e9/ql/src/test/results/clientnegative/dbtxnmgr_nodblock.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientnegative/dbtxnmgr_nodblock.q.out b/ql/src/test/results/clientnegative/dbtxnmgr_nodblock.q.out
index b8bbe93..e783251 100644
--- a/ql/src/test/results/clientnegative/dbtxnmgr_nodblock.q.out
+++ b/ql/src/test/results/clientnegative/dbtxnmgr_nodblock.q.out
@@ -10,4 +10,6 @@ POSTHOOK: type: CREATEDATABASE
 POSTHOOK: Output: database:drop_nodblock
 PREHOOK: query: lock database drop_nodblock shared
 PREHOOK: type: LOCKDATABASE
+PREHOOK: Input: database:drop_nodblock
+PREHOOK: Output: database:drop_nodblock
 FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.exec.DDLTask. Current
transaction manager does not support explicit lock requests.  Transaction manager:   org.apache.hadoop.hive.ql.lockmgr.DbTxnManager

http://git-wip-us.apache.org/repos/asf/hive/blob/6bfec2e9/ql/src/test/results/clientnegative/dbtxnmgr_nodbunlock.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientnegative/dbtxnmgr_nodbunlock.q.out b/ql/src/test/results/clientnegative/dbtxnmgr_nodbunlock.q.out
index ea7a3eb..d7a39f0 100644
--- a/ql/src/test/results/clientnegative/dbtxnmgr_nodbunlock.q.out
+++ b/ql/src/test/results/clientnegative/dbtxnmgr_nodbunlock.q.out
@@ -10,4 +10,6 @@ POSTHOOK: type: CREATEDATABASE
 POSTHOOK: Output: database:drop_nodbunlock
 PREHOOK: query: unlock database drop_nodbunlock
 PREHOOK: type: UNLOCKDATABASE
+PREHOOK: Input: database:drop_nodbunlock
+PREHOOK: Output: database:drop_nodbunlock
 FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.exec.DDLTask. Current
transaction manager does not support explicit lock requests.  Transaction manager:   org.apache.hadoop.hive.ql.lockmgr.DbTxnManager

http://git-wip-us.apache.org/repos/asf/hive/blob/6bfec2e9/ql/src/test/results/clientnegative/lockneg_query_tbl_in_locked_db.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientnegative/lockneg_query_tbl_in_locked_db.q.out b/ql/src/test/results/clientnegative/lockneg_query_tbl_in_locked_db.q.out
index 134a57b..2c46159 100644
--- a/ql/src/test/results/clientnegative/lockneg_query_tbl_in_locked_db.q.out
+++ b/ql/src/test/results/clientnegative/lockneg_query_tbl_in_locked_db.q.out
@@ -34,8 +34,12 @@ POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).key SIMPLE
[(srcpar
 POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value,
type:string, comment:default), ]
 PREHOOK: query: lock database lockneg1 shared
 PREHOOK: type: LOCKDATABASE
+PREHOOK: Input: database:lockneg1
+PREHOOK: Output: database:lockneg1
 POSTHOOK: query: lock database lockneg1 shared
 POSTHOOK: type: LOCKDATABASE
+POSTHOOK: Input: database:lockneg1
+POSTHOOK: Output: database:lockneg1
 PREHOOK: query: show locks database lockneg1
 PREHOOK: type: SHOWLOCKS
 POSTHOOK: query: show locks database lockneg1
@@ -53,4 +57,6 @@ POSTHOOK: Input: lockneg1@tstsrcpart@ds=2008-04-08/hr=11
 500
 PREHOOK: query: unlock database lockneg1
 PREHOOK: type: UNLOCKDATABASE
+PREHOOK: Input: database:lockneg1
+PREHOOK: Output: database:lockneg1
 FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.exec.DDLTask. Database
lockneg1 is not locked 

http://git-wip-us.apache.org/repos/asf/hive/blob/6bfec2e9/ql/src/test/results/clientnegative/lockneg_try_db_lock_conflict.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientnegative/lockneg_try_db_lock_conflict.q.out b/ql/src/test/results/clientnegative/lockneg_try_db_lock_conflict.q.out
index 0a5d98a..fe12d83 100644
--- a/ql/src/test/results/clientnegative/lockneg_try_db_lock_conflict.q.out
+++ b/ql/src/test/results/clientnegative/lockneg_try_db_lock_conflict.q.out
@@ -6,9 +6,15 @@ POSTHOOK: type: CREATEDATABASE
 POSTHOOK: Output: database:lockneg4
 PREHOOK: query: lock database lockneg4 exclusive
 PREHOOK: type: LOCKDATABASE
+PREHOOK: Input: database:lockneg4
+PREHOOK: Output: database:lockneg4
 POSTHOOK: query: lock database lockneg4 exclusive
 POSTHOOK: type: LOCKDATABASE
+POSTHOOK: Input: database:lockneg4
+POSTHOOK: Output: database:lockneg4
 PREHOOK: query: lock database lockneg4 shared
 PREHOOK: type: LOCKDATABASE
+PREHOOK: Input: database:lockneg4
+PREHOOK: Output: database:lockneg4
 Unable to acquire EXPLICIT, SHARED lock lockneg4 after 1 attempts.
 FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.exec.DDLTask

http://git-wip-us.apache.org/repos/asf/hive/blob/6bfec2e9/ql/src/test/results/clientnegative/lockneg_try_drop_locked_db.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientnegative/lockneg_try_drop_locked_db.q.out b/ql/src/test/results/clientnegative/lockneg_try_drop_locked_db.q.out
index 5ac4e3c..e66965e 100644
--- a/ql/src/test/results/clientnegative/lockneg_try_drop_locked_db.q.out
+++ b/ql/src/test/results/clientnegative/lockneg_try_drop_locked_db.q.out
@@ -6,8 +6,12 @@ POSTHOOK: type: CREATEDATABASE
 POSTHOOK: Output: database:lockneg9
 PREHOOK: query: lock database lockneg9 shared
 PREHOOK: type: LOCKDATABASE
+PREHOOK: Input: database:lockneg9
+PREHOOK: Output: database:lockneg9
 POSTHOOK: query: lock database lockneg9 shared
 POSTHOOK: type: LOCKDATABASE
+POSTHOOK: Input: database:lockneg9
+POSTHOOK: Output: database:lockneg9
 PREHOOK: query: show locks database lockneg9
 PREHOOK: type: SHOWLOCKS
 POSTHOOK: query: show locks database lockneg9

http://git-wip-us.apache.org/repos/asf/hive/blob/6bfec2e9/ql/src/test/results/clientnegative/lockneg_try_lock_db_in_use.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientnegative/lockneg_try_lock_db_in_use.q.out b/ql/src/test/results/clientnegative/lockneg_try_lock_db_in_use.q.out
index 5486151..e5c8f3e 100644
--- a/ql/src/test/results/clientnegative/lockneg_try_lock_db_in_use.q.out
+++ b/ql/src/test/results/clientnegative/lockneg_try_lock_db_in_use.q.out
@@ -34,13 +34,19 @@ POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).key SIMPLE
[(srcpar
 POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value,
type:string, comment:default), ]
 PREHOOK: query: lock database lockneg2 shared
 PREHOOK: type: LOCKDATABASE
+PREHOOK: Input: database:lockneg2
+PREHOOK: Output: database:lockneg2
 POSTHOOK: query: lock database lockneg2 shared
 POSTHOOK: type: LOCKDATABASE
+POSTHOOK: Input: database:lockneg2
+POSTHOOK: Output: database:lockneg2
 PREHOOK: query: show locks
 PREHOOK: type: SHOWLOCKS
 POSTHOOK: query: show locks
 POSTHOOK: type: SHOWLOCKS
 PREHOOK: query: lock database lockneg2 exclusive
 PREHOOK: type: LOCKDATABASE
+PREHOOK: Input: database:lockneg2
+PREHOOK: Output: database:lockneg2
 Unable to acquire EXPLICIT, EXCLUSIVE lock lockneg2 after 1 attempts.
 FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.exec.DDLTask


Mime
View raw message