hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ga...@apache.org
Subject svn commit: r1627914 - in /hive/trunk: itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/ metastore/src/java/org/apache/hadoop/hive/metastore/ metastore/src/java/org/apache/hadoop/hive/metastore/txn/ ql/src/java/org/apache/hadoop/hive/ql/...
Date Sat, 27 Sep 2014 00:37:01 GMT
Author: gates
Date: Sat Sep 27 00:37:01 2014
New Revision: 1627914

URL: http://svn.apache.org/r1627914
Log:
HIVE-8203 ACID operations result in NPE when run through HS2 (Alan Gates, reviewed by Eugene
Koifman)

Modified:
    hive/trunk/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStoreTxns.java
    hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java
    hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/IMetaStoreClient.java
    hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/DbTxnManager.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Initiator.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Worker.java

Modified: hive/trunk/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStoreTxns.java
URL: http://svn.apache.org/viewvc/hive/trunk/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStoreTxns.java?rev=1627914&r1=1627913&r2=1627914&view=diff
==============================================================================
--- hive/trunk/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStoreTxns.java
(original)
+++ hive/trunk/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStoreTxns.java
Sat Sep 27 00:37:01 2014
@@ -76,6 +76,21 @@ public class TestHiveMetaStoreTxns {
   }
 
   @Test
+  public void testOpenTxnNotExcluded() throws Exception {
+    List<Long> tids = client.openTxns("me", 3).getTxn_ids();
+    Assert.assertEquals(1L, (long) tids.get(0));
+    Assert.assertEquals(2L, (long) tids.get(1));
+    Assert.assertEquals(3L, (long) tids.get(2));
+    client.rollbackTxn(1);
+    client.commitTxn(2);
+    ValidTxnList validTxns = client.getValidTxns(3);
+    Assert.assertFalse(validTxns.isTxnCommitted(1));
+    Assert.assertTrue(validTxns.isTxnCommitted(2));
+    Assert.assertTrue(validTxns.isTxnCommitted(3));
+    Assert.assertFalse(validTxns.isTxnCommitted(4));
+  }
+
+  @Test
   public void testTxnRange() throws Exception {
     ValidTxnList validTxns = client.getValidTxns();
     Assert.assertEquals(ValidTxnList.RangeResponse.NONE,

Modified: hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java
URL: http://svn.apache.org/viewvc/hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java?rev=1627914&r1=1627913&r2=1627914&view=diff
==============================================================================
--- hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java
(original)
+++ hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java
Sat Sep 27 00:37:01 2014
@@ -1689,7 +1689,12 @@ public class HiveMetaStoreClient impleme
 
   @Override
   public ValidTxnList getValidTxns() throws TException {
-    return TxnHandler.createValidTxnList(client.get_open_txns());
+    return TxnHandler.createValidTxnList(client.get_open_txns(), 0);
+  }
+
+  @Override
+  public ValidTxnList getValidTxns(long currentTxn) throws TException {
+    return TxnHandler.createValidTxnList(client.get_open_txns(), currentTxn);
   }
 
   @Override

Modified: hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/IMetaStoreClient.java
URL: http://svn.apache.org/viewvc/hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/IMetaStoreClient.java?rev=1627914&r1=1627913&r2=1627914&view=diff
==============================================================================
--- hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/IMetaStoreClient.java (original)
+++ hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/IMetaStoreClient.java Sat
Sep 27 00:37:01 2014
@@ -1086,6 +1086,15 @@ public interface IMetaStoreClient {
   ValidTxnList getValidTxns() throws TException;
 
   /**
+   * Get a structure that details valid transactions.
+   * @param currentTxn The current transaction of the caller.  This will be removed from
the
+   *                   exceptions list so that the caller sees records from his own transaction.
+   * @return list of valid transactions
+   * @throws TException
+   */
+  ValidTxnList getValidTxns(long currentTxn) throws TException;
+
+  /**
    * Initiate a transaction.
    * @param user User who is opening this transaction.  This is the Hive user,
    *             not necessarily the OS user.  It is assumed that this user has already been

Modified: hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java
URL: http://svn.apache.org/viewvc/hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java?rev=1627914&r1=1627913&r2=1627914&view=diff
==============================================================================
--- hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java (original)
+++ hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java Sat
Sep 27 00:37:01 2014
@@ -233,12 +233,22 @@ public class TxnHandler {
     }
   }
 
-  public static ValidTxnList createValidTxnList(GetOpenTxnsResponse txns) {
+  /**
+   * Transform a {@link org.apache.hadoop.hive.metastore.api.GetOpenTxnsResponse} to a
+   * {@link org.apache.hadoop.hive.common.ValidTxnList}.
+   * @param txns txn list from the metastore
+   * @param currentTxn Current transaction that the user has open.  If this is greater than
0 it
+   *                   will be removed from the exceptions list so that the user sees his
own
+   *                   transaction as valid.
+   * @return a valid txn list.
+   */
+  public static ValidTxnList createValidTxnList(GetOpenTxnsResponse txns, long currentTxn)
{
     long highWater = txns.getTxn_high_water_mark();
     Set<Long> open = txns.getOpen_txns();
-    long[] exceptions = new long[open.size()];
+    long[] exceptions = new long[open.size() - (currentTxn > 0 ? 1 : 0)];
     int i = 0;
     for(long txn: open) {
+      if (currentTxn > 0 && currentTxn == txn) continue;
       exceptions[i++] = txn;
     }
     return new ValidTxnListImpl(exceptions, highWater);

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/Driver.java?rev=1627914&r1=1627913&r2=1627914&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/Driver.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/Driver.java Sat Sep 27 00:37:01 2014
@@ -390,6 +390,9 @@ public class Driver implements CommandPr
       tree = ParseUtils.findRootNonNullToken(tree);
       perfLogger.PerfLogEnd(CLASS_NAME, PerfLogger.PARSE);
 
+      // Initialize the transaction manager.  This must be done before analyze is called
+      SessionState.get().initTxnMgr(conf);
+
       perfLogger.PerfLogBegin(CLASS_NAME, PerfLogger.ANALYZE);
       BaseSemanticAnalyzer sem = SemanticAnalyzerFactory.get(conf, tree);
       List<HiveSemanticAnalyzerHook> saHooks =
@@ -889,9 +892,12 @@ public class Driver implements CommandPr
 
   /**
    * Acquire read and write locks needed by the statement. The list of objects to be locked
are
-   * obtained from he inputs and outputs populated by the compiler. The lock acuisition scheme
is
+   * obtained from the inputs and outputs populated by the compiler. The lock acuisition
scheme is
    * pretty simple. If all the locks cannot be obtained, error out. Deadlock is avoided by
making
    * sure that the locks are lexicographically sorted.
+   *
+   * This method also records the list of valid transactions.  This must be done after any
+   * transactions have been opened and locks acquired.
    **/
   private int acquireLocksAndOpenTxn() {
     PerfLogger perfLogger = PerfLogger.getPerfLogger();
@@ -931,7 +937,7 @@ public class Driver implements CommandPr
 
       txnMgr.acquireLocks(plan, ctx, userFromUGI);
 
-      return 0;
+      return recordValidTxns();
     } catch (LockException e) {
       errorMessage = "FAILED: Error in acquiring locks: " + e.getMessage();
       SQLState = ErrorMsg.findSQLState(e.getMessage());
@@ -1108,11 +1114,6 @@ public class Driver implements CommandPr
     SessionState ss = SessionState.get();
     try {
       ckLock = checkConcurrency();
-      try {
-        ss.initTxnMgr(conf);
-      } catch (LockException e) {
-        throw new SemanticException(e.getMessage(), e);
-      }
     } catch (SemanticException e) {
       errorMessage = "FAILED: Error in semantic analysis: " + e.getMessage();
       SQLState = ErrorMsg.findSQLState(e.getMessage());
@@ -1121,11 +1122,8 @@ public class Driver implements CommandPr
           + org.apache.hadoop.util.StringUtils.stringifyException(e));
       return createProcessorResponse(10);
     }
-    int ret = recordValidTxns();
-    if (ret != 0) {
-      return createProcessorResponse(ret);
-    }
 
+    int ret;
     if (!alreadyCompiled) {
       ret = compileInternal(command);
       if (ret != 0) {

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/DbTxnManager.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/DbTxnManager.java?rev=1627914&r1=1627913&r2=1627914&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/DbTxnManager.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/DbTxnManager.java Sat Sep 27
00:37:01 2014
@@ -286,7 +286,7 @@ public class DbTxnManager extends HiveTx
   public ValidTxnList getValidTxns() throws LockException {
     init();
     try {
-      return client.getValidTxns();
+      return client.getValidTxns(txnId);
     } catch (TException e) {
       throw new LockException(ErrorMsg.METASTORE_COMMUNICATION_FAILED.getMsg(),
           e);

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Initiator.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Initiator.java?rev=1627914&r1=1627913&r2=1627914&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Initiator.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Initiator.java Sat Sep
27 00:37:01 2014
@@ -76,7 +76,7 @@ public class Initiator extends Compactor
         // don't doom the entire thread.
         try {
           ShowCompactResponse currentCompactions = txnHandler.showCompact(new ShowCompactRequest());
-          ValidTxnList txns = TxnHandler.createValidTxnList(txnHandler.getOpenTxns());
+          ValidTxnList txns = TxnHandler.createValidTxnList(txnHandler.getOpenTxns(), 0);
           Set<CompactionInfo> potentials = txnHandler.findPotentialCompactions(abortedThreshold);
           LOG.debug("Found " + potentials.size() + " potential compactions, " +
               "checking to see if we should compact any of them");

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Worker.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Worker.java?rev=1627914&r1=1627913&r2=1627914&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Worker.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Worker.java Sat Sep 27
00:37:01 2014
@@ -120,7 +120,7 @@ public class Worker extends CompactorThr
 
         final boolean isMajor = ci.isMajorCompaction();
         final ValidTxnList txns =
-            TxnHandler.createValidTxnList(txnHandler.getOpenTxns());
+            TxnHandler.createValidTxnList(txnHandler.getOpenTxns(), 0);
         final StringBuffer jobName = new StringBuffer(name);
         jobName.append("-compactor-");
         jobName.append(ci.getFullPartitionName());



Mime
View raw message