hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From da...@apache.org
Subject hive git commit: HIVE-12441: Driver.acquireLocksAndOpenTxn() should only call recordValidTxns() when needed (Wei Zheng, reviewed by Alan Gates)
Date Thu, 11 Feb 2016 18:51:07 GMT
Repository: hive
Updated Branches:
  refs/heads/branch-1 dce033be5 -> ad7c129e9


HIVE-12441: Driver.acquireLocksAndOpenTxn() should only call recordValidTxns() when needed
(Wei Zheng, reviewed by Alan Gates)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/ad7c129e
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/ad7c129e
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/ad7c129e

Branch: refs/heads/branch-1
Commit: ad7c129e92c17609c2ac491f5bd7aecea68baa41
Parents: dce033b
Author: Daniel Dai <daijy@hortonworks.com>
Authored: Thu Feb 11 10:50:39 2016 -0800
Committer: Daniel Dai <daijy@hortonworks.com>
Committed: Thu Feb 11 10:50:39 2016 -0800

----------------------------------------------------------------------
 ql/src/java/org/apache/hadoop/hive/ql/Driver.java     |  7 +++++--
 .../hadoop/hive/ql/parse/BaseSemanticAnalyzer.java    |  7 +++++++
 .../apache/hadoop/hive/ql/parse/SemanticAnalyzer.java |  2 ++
 .../org/apache/hadoop/hive/ql/TestTxnCommands2.java   | 14 ++++++++++++++
 4 files changed, 28 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/ad7c129e/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/Driver.java b/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
index fe780ce..b2bc71e 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
@@ -147,6 +147,8 @@ public class Driver implements CommandProcessor {
 
   // A list of FileSinkOperators writing in an ACID compliant manner
   private Set<FileSinkDesc> acidSinks;
+  // whether any ACID table is involved in a query
+  private boolean acidInQuery;
 
   // A limit on the number of threads that can be launched
   private int maxthreads;
@@ -462,6 +464,7 @@ public class Driver implements CommandProcessor {
 
       // validate the plan
       sem.validate();
+      acidInQuery = sem.hasAcidInQuery();
       perfLogger.PerfLogEnd(CLASS_NAME, PerfLogger.ANALYZE);
 
       // get the output schema
@@ -1052,12 +1055,12 @@ public class Driver implements CommandProcessor {
       For multi-stmt txns this is not sufficient and will be managed via WriteSet tracking
       in the lock manager.*/
       txnMgr.acquireLocks(plan, ctx, userFromUGI);
-      if(initiatingTransaction || readOnlyQueryInAutoCommit) {
+      if(initiatingTransaction || (readOnlyQueryInAutoCommit && acidInQuery)) {
         //For multi-stmt txns we should record the snapshot when txn starts but
         // don't update it after that until txn completes.  Thus the check for {@code initiatingTransaction}
         //For autoCommit=true, Read-only statements, txn is implicit, i.e. lock in the snapshot
         //for each statement.
-        recordValidTxns();//todo: we should only need to do this for RO query if it has ACID
resources in it.
+        recordValidTxns();
       }
 
       return 0;

http://git-wip-us.apache.org/repos/asf/hive/blob/ad7c129e/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java
index fbe93f9..7730031 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java
@@ -103,6 +103,9 @@ public abstract class BaseSemanticAnalyzer {
    * back and set it once we actually start running the query.
    */
   protected Set<FileSinkDesc> acidFileSinks = new HashSet<FileSinkDesc>();
+  
+  // whether any ACID table is involved in a query
+  protected boolean acidInQuery;
 
   public static int HIVE_COLUMN_ORDER_ASC = 1;
   public static int HIVE_COLUMN_ORDER_DESC = 0;
@@ -1079,6 +1082,10 @@ public abstract class BaseSemanticAnalyzer {
   public Set<FileSinkDesc> getAcidFileSinks() {
     return acidFileSinks;
   }
+  
+  public boolean hasAcidInQuery() {
+    return acidInQuery;
+  }
 
   /**
    * Construct list bucketing context.

http://git-wip-us.apache.org/repos/asf/hive/blob/ad7c129e/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
index a6c4bdc..36b0c2c 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
@@ -10631,6 +10631,7 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
         tbl = p.getTable();
       }
       if (tbl != null && AcidUtils.isAcidTable(tbl)) {
+        acidInQuery = true;
         checkAcidTxnManager(tbl);
       }
     }
@@ -10707,6 +10708,7 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
       }
 
       if (tbl != null && AcidUtils.isAcidTable(tbl)) {
+        acidInQuery = true;
         checkAcidTxnManager(tbl);
       }
     }

http://git-wip-us.apache.org/repos/asf/hive/blob/ad7c129e/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands2.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands2.java b/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands2.java
index 1b28877..a16a788 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands2.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands2.java
@@ -23,6 +23,7 @@ import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.FileUtil;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hive.common.FileUtils;
+import org.apache.hadoop.hive.common.ValidTxnList;
 import org.apache.hadoop.hive.conf.HiveConf;
 import org.apache.hadoop.hive.metastore.HouseKeeperService;
 import org.apache.hadoop.hive.metastore.api.CompactionRequest;
@@ -401,6 +402,19 @@ public class TestTxnCommands2 {
   }
 
   @Test
+  public void testValidTxnsBookkeeping() throws Exception {
+    // 1. Run a query against a non-ACID table, and we shouldn't have txn logged in conf
+    runStatementOnDriver("select * from " + Table.NONACIDORCTBL);
+    String value = hiveConf.get(ValidTxnList.VALID_TXNS_KEY);
+    Assert.assertNull("The entry should be null for query that doesn't involve ACID tables",
value);
+
+    // 2. Run a query against an ACID table, and we should have txn logged in conf
+    runStatementOnDriver("select * from " + Table.ACIDTBL);
+    value = hiveConf.get(ValidTxnList.VALID_TXNS_KEY);
+    Assert.assertNotNull("The entry shouldn't be null for query that involves ACID tables",
value);
+  }
+
+  @Test
   public void testUpdateMixedCase() throws Exception {
     int[][] tableData = {{1,2},{3,3},{5,3}};
     runStatementOnDriver("insert into " + Table.ACIDTBL + "(a,b) " + makeValuesClause(tableData));


Mime
View raw message