hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ekoif...@apache.org
Subject [2/3] hive git commit: HIVE-12636 Ensure that all queries (with DbTxnManager) run in a transaction (Eugene Koifman, reviewed by Wei Zheng)
Date Mon, 01 May 2017 16:43:35 GMT
http://git-wip-us.apache.org/repos/asf/hive/blob/21909601/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands.java b/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands.java
index c8bc119..7c66955 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands.java
@@ -137,7 +137,6 @@ public class TestTxnCommands {
   public void tearDown() throws Exception {
     try {
       if (d != null) {
-        runStatementOnDriver("set autocommit true");
         dropTables();
         d.destroy();
         d.close();
@@ -194,7 +193,6 @@ public class TestTxnCommands {
     runStatementOnDriver("insert into " + Table.ACIDTBL + "(a,b) " + makeValuesClause(rows1));
     //List<String> rs = runStatementOnDriver("select a,b from " + Table.ACIDTBL + "
order by a,b");
     //Assert.assertEquals("Data didn't match in autocommit=true (rs)", stringifyValues(rows1),
rs);
-    runStatementOnDriver("set autocommit false");
     runStatementOnDriver("START TRANSACTION");
     int[][] rows2 = {{5,6},{7,8}};
     runStatementOnDriver("insert into " + Table.ACIDTBL + "(a,b) " + makeValuesClause(rows2));
@@ -206,8 +204,8 @@ public class TestTxnCommands {
     dumpTableData(Table.ACIDTBL, 1, 0);
     dumpTableData(Table.ACIDTBL, 2, 0);
     runStatementOnDriver("select a,b from " + Table.ACIDTBL + " order by a,b");
-    runStatementOnDriver("COMMIT");//txn started implicitly by previous statement
-    runStatementOnDriver("set autocommit true");
+    CommandProcessorResponse cpr = runStatementOnDriverNegative("COMMIT");//txn started implicitly
by previous statement
+    Assert.assertEquals("Error didn't match: " + cpr, ErrorMsg.OP_NOT_ALLOWED_WITHOUT_TXN.getErrorCode(),
cpr.getErrorCode());
     List<String> rs1 = runStatementOnDriver("select a,b from " + Table.ACIDTBL + "
order by a,b");
     Assert.assertEquals("Data didn't match inside tx (rs0)", allData, rs1);
   }
@@ -218,43 +216,35 @@ public class TestTxnCommands {
    */
   @Test
   public void testErrors() throws Exception {
-    runStatementOnDriver("set autocommit true");
-    CommandProcessorResponse cpr = runStatementOnDriverNegative("start transaction");
-    Assert.assertEquals("Error didn't match: " + cpr, ErrorMsg.OP_NOT_ALLOWED_IN_AUTOCOMMIT.getErrorCode(),
cpr.getErrorCode());
-    runStatementOnDriver("set autocommit false");
     runStatementOnDriver("start transaction");
     CommandProcessorResponse cpr2 = runStatementOnDriverNegative("create table foo(x int,
y int)");
     Assert.assertEquals("Expected DDL to fail in an open txn", ErrorMsg.OP_NOT_ALLOWED_IN_TXN.getErrorCode(),
cpr2.getErrorCode());
-    runStatementOnDriver("set autocommit true");
     CommandProcessorResponse cpr3 = runStatementOnDriverNegative("update " + Table.ACIDTBL
+ " set a = 1 where b != 1");
     Assert.assertEquals("Expected update of bucket column to fail",
       "FAILED: SemanticException [Error 10302]: Updating values of bucketing columns is not
supported.  Column a.",
       cpr3.getErrorMessage());
-    //line below should in principle work but Driver doesn't propagate errorCode properly
-    //Assert.assertEquals("Expected update of bucket column to fail", ErrorMsg.UPDATE_CANNOT_UPDATE_BUCKET_VALUE.getErrorCode(),
cpr3.getErrorCode());
-    cpr3 = runStatementOnDriverNegative("commit work");//not allowed in AC=true
-    Assert.assertEquals("Error didn't match: " + cpr3, ErrorMsg.OP_NOT_ALLOWED_IN_AUTOCOMMIT.getErrorCode(),
cpr.getErrorCode());
-    cpr3 = runStatementOnDriverNegative("rollback work");//not allowed in AC=true
-    Assert.assertEquals("Error didn't match: " + cpr3, ErrorMsg.OP_NOT_ALLOWED_IN_AUTOCOMMIT.getErrorCode(),
cpr.getErrorCode());
-    runStatementOnDriver("set autocommit false");
+    Assert.assertEquals("Expected update of bucket column to fail",
+      ErrorMsg.UPDATE_CANNOT_UPDATE_BUCKET_VALUE.getErrorCode(), cpr3.getErrorCode());
     cpr3 = runStatementOnDriverNegative("commit");//not allowed in w/o tx
-    Assert.assertEquals("Error didn't match: " + cpr3, ErrorMsg.OP_NOT_ALLOWED_IN_AUTOCOMMIT.getErrorCode(),
cpr.getErrorCode());
+    Assert.assertEquals("Error didn't match: " + cpr3,
+      ErrorMsg.OP_NOT_ALLOWED_WITHOUT_TXN.getErrorCode(), cpr3.getErrorCode());
     cpr3 = runStatementOnDriverNegative("rollback");//not allowed in w/o tx
-    Assert.assertEquals("Error didn't match: " + cpr3, ErrorMsg.OP_NOT_ALLOWED_IN_AUTOCOMMIT.getErrorCode(),
cpr.getErrorCode());
+    Assert.assertEquals("Error didn't match: " + cpr3,
+      ErrorMsg.OP_NOT_ALLOWED_WITHOUT_TXN.getErrorCode(), cpr3.getErrorCode());
     runStatementOnDriver("start transaction");
     cpr3 = runStatementOnDriverNegative("start transaction");//not allowed in a tx
-    Assert.assertEquals("Expected start transaction to fail", ErrorMsg.OP_NOT_ALLOWED_IN_TXN.getErrorCode(),
cpr3.getErrorCode());
+    Assert.assertEquals("Expected start transaction to fail",
+      ErrorMsg.OP_NOT_ALLOWED_IN_TXN.getErrorCode(), cpr3.getErrorCode());
     runStatementOnDriver("start transaction");//ok since previously opened txn was killed
     runStatementOnDriver("insert into " + Table.ACIDTBL + "(a,b) values(1,2)");
     List<String> rs0 = runStatementOnDriver("select a,b from " + Table.ACIDTBL + "
order by a,b");
     Assert.assertEquals("Can't see my own write", 1, rs0.size());
-    runStatementOnDriver("set autocommit true");//this should commit previous txn
+    runStatementOnDriver("commit work");
     rs0 = runStatementOnDriver("select a,b from " + Table.ACIDTBL + " order by a,b");
     Assert.assertEquals("Can't see my own write", 1, rs0.size());
   }
   @Test
   public void testReadMyOwnInsert() throws Exception {
-    runStatementOnDriver("set autocommit false");
     runStatementOnDriver("START TRANSACTION");
     List<String> rs = runStatementOnDriver("select * from " + Table.ACIDTBL);
     Assert.assertEquals("Expected empty " + Table.ACIDTBL, 0, rs.size());
@@ -269,7 +259,6 @@ public class TestTxnCommands {
   }
   @Test
   public void testImplicitRollback() throws Exception {
-    runStatementOnDriver("set autocommit false");
     runStatementOnDriver("START TRANSACTION");
     runStatementOnDriver("insert into " + Table.ACIDTBL + "(a,b) values(1,2)");
     List<String> rs0 = runStatementOnDriver("select a,b from " + Table.ACIDTBL + "
order by a,b");
@@ -286,18 +275,15 @@ public class TestTxnCommands {
   }
   @Test
   public void testExplicitRollback() throws Exception {
-    runStatementOnDriver("set autocommit false");
     runStatementOnDriver("START TRANSACTION");
     runStatementOnDriver("insert into " + Table.ACIDTBL + "(a,b) values(1,2)");
     runStatementOnDriver("ROLLBACK");
-    runStatementOnDriver("set autocommit true");
     List<String> rs = runStatementOnDriver("select a,b from " + Table.ACIDTBL + " order
by a,b");
     Assert.assertEquals("Rollback didn't rollback", 0, rs.size());
   }
 
   @Test
   public void testMultipleInserts() throws Exception {
-    runStatementOnDriver("set autocommit false");
     runStatementOnDriver("START TRANSACTION");
     int[][] rows1 = {{1,2},{3,4}};
     runStatementOnDriver("insert into " + Table.ACIDTBL + "(a,b) " + makeValuesClause(rows1));
@@ -310,7 +296,6 @@ public class TestTxnCommands {
     runStatementOnDriver("commit");
     dumpTableData(Table.ACIDTBL, 1, 0);
     dumpTableData(Table.ACIDTBL, 1, 1);
-    runStatementOnDriver("set autocommit true");
     List<String> rs1 = runStatementOnDriver("select a,b from " + Table.ACIDTBL + "
order by a,b");
     Assert.assertEquals("Content didn't match after commit rs1", allData, rs1);
   }
@@ -320,14 +305,12 @@ public class TestTxnCommands {
     runStatementOnDriver("insert into " + Table.ACIDTBL + "(a,b) " + makeValuesClause(rows1));
     List<String> rs0 = runStatementOnDriver("select a,b from " + Table.ACIDTBL + "
order by a,b");
     Assert.assertEquals("Content didn't match rs0", stringifyValues(rows1), rs0);
-    runStatementOnDriver("set autocommit false");
     runStatementOnDriver("START TRANSACTION");
     runStatementOnDriver("delete from " + Table.ACIDTBL + " where b = 4");
     int[][] updatedData2 = {{1,2}};
     List<String> rs3 = runStatementOnDriver("select a,b from " + Table.ACIDTBL + "
order by a,b");
     Assert.assertEquals("Wrong data after delete", stringifyValues(updatedData2), rs3);
     runStatementOnDriver("commit");
-    runStatementOnDriver("set autocommit true");
     List<String> rs4 = runStatementOnDriver("select a,b from " + Table.ACIDTBL + "
order by a,b");
     Assert.assertEquals("Wrong data after commit", stringifyValues(updatedData2), rs4);
   }
@@ -338,7 +321,6 @@ public class TestTxnCommands {
     runStatementOnDriver("insert into " + Table.ACIDTBL + "(a,b) " + makeValuesClause(rows1));
     List<String> rs0 = runStatementOnDriver("select a,b from " + Table.ACIDTBL + "
order by a,b");
     Assert.assertEquals("Content didn't match rs0", stringifyValues(rows1), rs0);
-    runStatementOnDriver("set autocommit false");
     runStatementOnDriver("START TRANSACTION");
     int[][] rows2 = {{5,6},{7,8}};
     runStatementOnDriver("insert into " + Table.ACIDTBL + "(a,b) " + makeValuesClause(rows2));
@@ -351,7 +333,6 @@ public class TestTxnCommands {
     List<String> rs2 = runStatementOnDriver("select a,b from " + Table.ACIDTBL + "
order by a,b");
     Assert.assertEquals("Wrong data after update", stringifyValues(updatedData), rs2);
     runStatementOnDriver("commit");
-    runStatementOnDriver("set autocommit true");
     List<String> rs4 = runStatementOnDriver("select a,b from " + Table.ACIDTBL + "
order by a,b");
     Assert.assertEquals("Wrong data after commit", stringifyValues(updatedData), rs4);
   }
@@ -361,7 +342,6 @@ public class TestTxnCommands {
     runStatementOnDriver("insert into " + Table.ACIDTBL + "(a,b) " + makeValuesClause(rows1));
     List<String> rs0 = runStatementOnDriver("select a,b from " + Table.ACIDTBL + "
order by a,b");
     Assert.assertEquals("Content didn't match rs0", stringifyValues(rows1), rs0);
-    runStatementOnDriver("set autocommit false");
     runStatementOnDriver("START TRANSACTION");
     int[][] rows2 = {{5,6},{7,8}};
     runStatementOnDriver("insert into " + Table.ACIDTBL + "(a,b) " + makeValuesClause(rows2));
@@ -382,7 +362,6 @@ public class TestTxnCommands {
     List<String> rs3 = runStatementOnDriver("select a,b from " + Table.ACIDTBL + "
order by a,b");
     Assert.assertEquals("Wrong data after delete", stringifyValues(updatedData2), rs3);
     runStatementOnDriver("commit");
-    runStatementOnDriver("set autocommit true");
     List<String> rs4 = runStatementOnDriver("select a,b from " + Table.ACIDTBL + "
order by a,b");
     Assert.assertEquals("Wrong data after commit", stringifyValues(updatedData2), rs4);
   }
@@ -392,7 +371,6 @@ public class TestTxnCommands {
     runStatementOnDriver("insert into " + Table.ACIDTBL + "(a,b) " + makeValuesClause(rows1));
     List<String> rs0 = runStatementOnDriver("select a,b from " + Table.ACIDTBL + "
order by a,b");
     Assert.assertEquals("Content didn't match rs0", stringifyValues(rows1), rs0);
-    runStatementOnDriver("set autocommit false");
     runStatementOnDriver("START TRANSACTION");
     runStatementOnDriver("delete from " + Table.ACIDTBL + " where b = 8");
     int[][] updatedData2 = {{1,2},{3,4},{5,6}};
@@ -412,7 +390,6 @@ public class TestTxnCommands {
     int [][] updatedData4 = {{1,3},{5,3}};
     Assert.assertEquals("Wrong data after delete", stringifyValues(updatedData4), rs5);
     runStatementOnDriver("commit");
-    runStatementOnDriver("set autocommit true");
     List<String> rs4 = runStatementOnDriver("select a,b from " + Table.ACIDTBL + "
order by a,b");
     Assert.assertEquals("Wrong data after commit", stringifyValues(updatedData4), rs4);
   }
@@ -433,7 +410,6 @@ public class TestTxnCommands {
   }
   @Test
   public void testTimeOutReaper() throws Exception {
-    runStatementOnDriver("set autocommit false");
     runStatementOnDriver("start transaction");
     runStatementOnDriver("delete from " + Table.ACIDTBL + " where a = 5");
     //make sure currently running txn is considered aborted by housekeeper
@@ -467,7 +443,7 @@ public class TestTxnCommands {
       }
     }
     Assert.assertNotNull(txnInfo);
-    Assert.assertEquals(2, txnInfo.getId());
+    Assert.assertEquals(12, txnInfo.getId());
     Assert.assertEquals(TxnState.OPEN, txnInfo.getState());
     String s =TxnDbUtil.queryToString("select TXN_STARTED, TXN_LAST_HEARTBEAT from TXNS where
TXN_ID = " + txnInfo.getId(), false);
     String[] vals = s.split("\\s+");

http://git-wip-us.apache.org/repos/asf/hive/blob/21909601/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands2.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands2.java b/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands2.java
index e2db5b7..5786c4f 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands2.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands2.java
@@ -662,11 +662,11 @@ public class TestTxnCommands2 {
         FileStatus[] buckets = fs.listStatus(status[i].getPath(), FileUtils.STAGING_DIR_PATH_FILTER);
         Arrays.sort(buckets);
         if (numDelta == 1) {
-          Assert.assertEquals("delta_0000001_0000001_0000", status[i].getPath().getName());
+          Assert.assertEquals("delta_0000022_0000022_0000", status[i].getPath().getName());
           Assert.assertEquals(BUCKET_COUNT - 1, buckets.length);
           Assert.assertEquals("bucket_00001", buckets[0].getPath().getName());
         } else if (numDelta == 2) {
-          Assert.assertEquals("delta_0000002_0000002_0000", status[i].getPath().getName());
+          Assert.assertEquals("delta_0000023_0000023_0000", status[i].getPath().getName());
           Assert.assertEquals(BUCKET_COUNT, buckets.length);
           Assert.assertEquals("bucket_00000", buckets[0].getPath().getName());
           Assert.assertEquals("bucket_00001", buckets[1].getPath().getName());
@@ -711,7 +711,7 @@ public class TestTxnCommands2 {
           Assert.assertEquals("bucket_00001", buckets[0].getPath().getName());
         } else if (numBase == 2) {
           // The new base dir now has two bucket files, since the delta dir has two bucket
files
-          Assert.assertEquals("base_0000002", status[i].getPath().getName());
+          Assert.assertEquals("base_0000023", status[i].getPath().getName());
           Assert.assertEquals(BUCKET_COUNT, buckets.length);
           Assert.assertEquals("bucket_00000", buckets[0].getPath().getName());
           Assert.assertEquals("bucket_00001", buckets[1].getPath().getName());
@@ -738,7 +738,7 @@ public class TestTxnCommands2 {
     status = fs.listStatus(new Path(TEST_WAREHOUSE_DIR + "/" +
         (Table.NONACIDORCTBL).toString().toLowerCase()), FileUtils.STAGING_DIR_PATH_FILTER);
     Assert.assertEquals(1, status.length);
-    Assert.assertEquals("base_0000002", status[0].getPath().getName());
+    Assert.assertEquals("base_0000023", status[0].getPath().getName());
     FileStatus[] buckets = fs.listStatus(status[0].getPath(), FileUtils.STAGING_DIR_PATH_FILTER);
     Arrays.sort(buckets);
     Assert.assertEquals(BUCKET_COUNT, buckets.length);
@@ -760,11 +760,6 @@ public class TestTxnCommands2 {
     runStatementOnDriver("select * from " + Table.NONACIDORCTBL);
     String value = hiveConf.get(ValidTxnList.VALID_TXNS_KEY);
     Assert.assertNull("The entry should be null for query that doesn't involve ACID tables",
value);
-
-    // 2. Run a query against an ACID table, and we should have txn logged in conf
-    runStatementOnDriver("select * from " + Table.ACIDTBL);
-    value = hiveConf.get(ValidTxnList.VALID_TXNS_KEY);
-    Assert.assertNotNull("The entry shouldn't be null for query that involves ACID tables",
value);
   }
 
   @Test
@@ -773,10 +768,14 @@ public class TestTxnCommands2 {
     int[][] tableData = {{1,2},{3,3}};
     runStatementOnDriver("insert into " + Table.ACIDTBL + " " + makeValuesClause(tableData));
     int[][] tableData2 = {{5,3}};
+    //this will cause next txn to be marked aborted but the data is still written to disk
+    hiveConf.setBoolVar(HiveConf.ConfVars.HIVETESTMODEROLLBACKTXN, true);
     runStatementOnDriver("insert into " + Table.ACIDTBL + " " + makeValuesClause(tableData2));
-    hiveConf.set(ValidTxnList.VALID_TXNS_KEY, "0:");
+    assert hiveConf.get(ValidTxnList.VALID_TXNS_KEY) == null : "previous txn should've cleaned
it";
+    //so now if HIVEFETCHTASKCONVERSION were to use a stale value, it would use a
+    //ValidTxnList with HWM=MAX_LONG, i.e. include the data for aborted txn
     List<String> rs = runStatementOnDriver("select * from " + Table.ACIDTBL);
-    Assert.assertEquals("Missing data", 3, rs.size());
+    Assert.assertEquals("Extra data", 2, rs.size());
   }
   @Test
   public void testUpdateMixedCase() throws Exception {

http://git-wip-us.apache.org/repos/asf/hive/blob/21909601/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands2WithSplitUpdate.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands2WithSplitUpdate.java b/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands2WithSplitUpdate.java
index e516f18..ea5ecbc 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands2WithSplitUpdate.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands2WithSplitUpdate.java
@@ -84,7 +84,6 @@ public class TestTxnCommands2WithSplitUpdate extends TestTxnCommands2 {
     runStatementOnDriver("create table acidTblLegacy (a int, b int) clustered by (a) into
" + BUCKET_COUNT + " buckets stored as orc TBLPROPERTIES ('transactional'='true')");
     runStatementOnDriver("alter table acidTblLegacy SET TBLPROPERTIES ('transactional_properties'
= 'default')");
   }
-
   /**
    * Test the query correctness and directory layout for ACID table conversion with split-update
    * enabled.
@@ -96,7 +95,8 @@ public class TestTxnCommands2WithSplitUpdate extends TestTxnCommands2 {
    * @throws Exception
    */
   @Test
-  public void testNonAcidToAcidSplitUpdateConversion1() throws Exception {
+  @Override
+  public void testNonAcidToAcidConversion1() throws Exception {
     FileSystem fs = FileSystem.get(hiveConf);
     FileStatus[] status;
 
@@ -226,7 +226,8 @@ public class TestTxnCommands2WithSplitUpdate extends TestTxnCommands2
{
    * @throws Exception
    */
   @Test
-  public void testNonAcidToAcidSplitUpdateConversion2() throws Exception {
+  @Override
+  public void testNonAcidToAcidConversion2() throws Exception {
     FileSystem fs = FileSystem.get(hiveConf);
     FileStatus[] status;
 
@@ -360,7 +361,8 @@ public class TestTxnCommands2WithSplitUpdate extends TestTxnCommands2
{
    * @throws Exception
    */
   @Test
-  public void testNonAcidToAcidSplitUpdateConversion3() throws Exception {
+  @Override
+  public void testNonAcidToAcidConversion3() throws Exception {
     FileSystem fs = FileSystem.get(hiveConf);
     FileStatus[] status;
 
@@ -442,11 +444,11 @@ public class TestTxnCommands2WithSplitUpdate extends TestTxnCommands2
{
         FileStatus[] buckets = fs.listStatus(status[i].getPath(), FileUtils.STAGING_DIR_PATH_FILTER);
         Arrays.sort(buckets);
         if (numDelta == 1) {
-          Assert.assertEquals("delta_0000001_0000001_0000", status[i].getPath().getName());
+          Assert.assertEquals("delta_0000022_0000022_0000", status[i].getPath().getName());
           Assert.assertEquals(BUCKET_COUNT - 1, buckets.length);
           Assert.assertEquals("bucket_00001", buckets[0].getPath().getName());
         } else if (numDelta == 2) {
-          Assert.assertEquals("delta_0000002_0000002_0000", status[i].getPath().getName());
+          Assert.assertEquals("delta_0000023_0000023_0000", status[i].getPath().getName());
           Assert.assertEquals(1, buckets.length);
           Assert.assertEquals("bucket_00001", buckets[0].getPath().getName());
         }
@@ -455,7 +457,7 @@ public class TestTxnCommands2WithSplitUpdate extends TestTxnCommands2
{
         FileStatus[] buckets = fs.listStatus(status[i].getPath(), FileUtils.STAGING_DIR_PATH_FILTER);
         Arrays.sort(buckets);
         if (numDeleteDelta == 1) {
-          Assert.assertEquals("delete_delta_0000001_0000001_0000", status[i].getPath().getName());
+          Assert.assertEquals("delete_delta_0000022_0000022_0000", status[i].getPath().getName());
           Assert.assertEquals(BUCKET_COUNT - 1, buckets.length);
           Assert.assertEquals("bucket_00001", buckets[0].getPath().getName());
         }
@@ -502,7 +504,7 @@ public class TestTxnCommands2WithSplitUpdate extends TestTxnCommands2
{
           Assert.assertEquals("bucket_00001", buckets[0].getPath().getName());
         } else if (numBase == 2) {
           // The new base dir now has two bucket files, since the delta dir has two bucket
files
-          Assert.assertEquals("base_0000002", status[i].getPath().getName());
+          Assert.assertEquals("base_0000023", status[i].getPath().getName());
           Assert.assertEquals(1, buckets.length);
           Assert.assertEquals("bucket_00001", buckets[0].getPath().getName());
         }
@@ -528,7 +530,7 @@ public class TestTxnCommands2WithSplitUpdate extends TestTxnCommands2
{
     status = fs.listStatus(new Path(TEST_WAREHOUSE_DIR + "/" +
         (Table.NONACIDORCTBL).toString().toLowerCase()), FileUtils.STAGING_DIR_PATH_FILTER);
     Assert.assertEquals(1, status.length);
-    Assert.assertEquals("base_0000002", status[0].getPath().getName());
+    Assert.assertEquals("base_0000023", status[0].getPath().getName());
     FileStatus[] buckets = fs.listStatus(status[0].getPath(), FileUtils.STAGING_DIR_PATH_FILTER);
     Arrays.sort(buckets);
     Assert.assertEquals(1, buckets.length);

http://git-wip-us.apache.org/repos/asf/hive/blob/21909601/ql/src/test/org/apache/hadoop/hive/ql/lockmgr/TestDbTxnManager.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/lockmgr/TestDbTxnManager.java b/ql/src/test/org/apache/hadoop/hive/ql/lockmgr/TestDbTxnManager.java
index 9bfc7d1..14ff58e 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/lockmgr/TestDbTxnManager.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/lockmgr/TestDbTxnManager.java
@@ -36,11 +36,13 @@ import org.apache.hadoop.hive.ql.hooks.WriteEntity;
 import org.apache.hadoop.hive.ql.metadata.DummyPartition;
 import org.apache.hadoop.hive.ql.metadata.Partition;
 import org.apache.hadoop.hive.ql.metadata.Table;
+import org.apache.hadoop.hive.ql.plan.HiveOperation;
 import org.apache.hadoop.hive.ql.session.SessionState;
 import org.apache.hadoop.hive.ql.txn.AcidHouseKeeperService;
 import org.junit.After;
 import org.junit.Assert;
 import org.junit.Before;
+import org.junit.Ignore;
 import org.junit.Test;
 
 import java.util.ArrayList;
@@ -79,13 +81,14 @@ public class TestDbTxnManager {
   @Test
   public void testSingleReadTable() throws Exception {
     addTableInput();
-    QueryPlan qp = new MockQueryPlan(this);
+    QueryPlan qp = new MockQueryPlan(this, HiveOperation.QUERY);
+    txnMgr.openTxn(ctx, "fred");
     txnMgr.acquireLocks(qp, ctx, "fred");
     List<HiveLock> locks = ctx.getHiveLocks();
     Assert.assertEquals(1, locks.size());
     Assert.assertEquals(1,
         TxnDbUtil.countLockComponents(((DbLockManager.DbHiveLock) locks.get(0)).lockId));
-    txnMgr.getLockManager().unlock(locks.get(0));
+    txnMgr.commitTxn();
     locks = txnMgr.getLockManager().getLocks(false, false);
     Assert.assertEquals(0, locks.size());
   }
@@ -93,13 +96,14 @@ public class TestDbTxnManager {
   @Test
   public void testSingleReadPartition() throws Exception {
     addPartitionInput(newTable(true));
-    QueryPlan qp = new MockQueryPlan(this);
+    QueryPlan qp = new MockQueryPlan(this, HiveOperation.QUERY);
+    txnMgr.openTxn(ctx, null);
     txnMgr.acquireLocks(qp, ctx, null);
     List<HiveLock> locks = ctx.getHiveLocks();
     Assert.assertEquals(1, locks.size());
     Assert.assertEquals(1,
         TxnDbUtil.countLockComponents(((DbLockManager.DbHiveLock) locks.get(0)).lockId));
-    txnMgr.getLockManager().unlock(locks.get(0));
+    txnMgr.commitTxn();
     locks = txnMgr.getLockManager().getLocks(false, false);
     Assert.assertEquals(0, locks.size());
 
@@ -111,13 +115,14 @@ public class TestDbTxnManager {
     addPartitionInput(t);
     addPartitionInput(t);
     addPartitionInput(t);
-    QueryPlan qp = new MockQueryPlan(this);
+    QueryPlan qp = new MockQueryPlan(this, HiveOperation.QUERY);
+    txnMgr.openTxn(ctx, "fred");
     txnMgr.acquireLocks(qp, ctx, "fred");
     List<HiveLock> locks = ctx.getHiveLocks();
     Assert.assertEquals(1, locks.size());
     Assert.assertEquals(3,
         TxnDbUtil.countLockComponents(((DbLockManager.DbHiveLock) locks.get(0)).lockId));
-    txnMgr.getLockManager().unlock(locks.get(0));
+    txnMgr.commitTxn();
     locks = txnMgr.getLockManager().getLocks(false, false);
     Assert.assertEquals(0, locks.size());
   }
@@ -129,13 +134,14 @@ public class TestDbTxnManager {
     addPartitionInput(t);
     addPartitionInput(t);
     addTableInput();
-    QueryPlan qp = new MockQueryPlan(this);
+    QueryPlan qp = new MockQueryPlan(this, HiveOperation.QUERY);
+    txnMgr.openTxn(ctx, "fred");
     txnMgr.acquireLocks(qp, ctx, "fred");
     List<HiveLock> locks = ctx.getHiveLocks();
     Assert.assertEquals(1, locks.size());
     Assert.assertEquals(4,
         TxnDbUtil.countLockComponents(((DbLockManager.DbHiveLock) locks.get(0)).lockId));
-    txnMgr.getLockManager().unlock(locks.get(0));
+    txnMgr.commitTxn();
     locks = txnMgr.getLockManager().getLocks(false, false);
     Assert.assertEquals(0, locks.size());
   }
@@ -143,7 +149,7 @@ public class TestDbTxnManager {
   @Test
   public void testSingleWriteTable() throws Exception {
     WriteEntity we = addTableOutput(WriteEntity.WriteType.INSERT);
-    QueryPlan qp = new MockQueryPlan(this);
+    QueryPlan qp = new MockQueryPlan(this, HiveOperation.QUERY);
     txnMgr.openTxn(ctx, "fred");
     txnMgr.acquireLocks(qp, ctx, "fred");
     List<HiveLock> locks = ctx.getHiveLocks();
@@ -159,7 +165,7 @@ public class TestDbTxnManager {
   @Test
   public void testSingleWritePartition() throws Exception {
     WriteEntity we = addPartitionOutput(newTable(true), WriteEntity.WriteType.INSERT);
-    QueryPlan qp = new MockQueryPlan(this);
+    QueryPlan qp = new MockQueryPlan(this, HiveOperation.QUERY);
     txnMgr.openTxn(ctx, "fred");
     txnMgr.acquireLocks(qp, ctx, "fred");
     List<HiveLock> locks = ctx.getHiveLocks();
@@ -174,7 +180,7 @@ public class TestDbTxnManager {
   @Test
   public void testWriteDynamicPartition() throws Exception {
     WriteEntity we = addDynamicPartitionedOutput(newTable(true), WriteEntity.WriteType.INSERT);
-    QueryPlan qp = new MockQueryPlan(this);
+    QueryPlan qp = new MockQueryPlan(this, HiveOperation.QUERY);
     txnMgr.openTxn(ctx, "fred");
     txnMgr.acquireLocks(qp, ctx, "fred");
     List<HiveLock> locks = ctx.getHiveLocks();
@@ -217,7 +223,7 @@ public class TestDbTxnManager {
   @Test
   public void testExceptions() throws Exception {
     addPartitionOutput(newTable(true), WriteEntity.WriteType.INSERT);
-    QueryPlan qp = new MockQueryPlan(this);
+    QueryPlan qp = new MockQueryPlan(this, HiveOperation.QUERY);
     ((DbTxnManager) txnMgr).openTxn(ctx, "NicholasII", HiveConf.getTimeVar(conf, HiveConf.ConfVars.HIVE_TXN_TIMEOUT,
TimeUnit.MILLISECONDS) * 2);
     Thread.sleep(HiveConf.getTimeVar(conf, HiveConf.ConfVars.HIVE_TXN_TIMEOUT, TimeUnit.MILLISECONDS));
     runReaper();
@@ -243,10 +249,11 @@ public class TestDbTxnManager {
     txnMgr.rollbackTxn();//this is idempotent
   }
 
+  @Ignore("This seems useless now that we have a txn for everything")
   @Test
   public void testLockTimeout() throws Exception {
     addPartitionInput(newTable(true));
-    QueryPlan qp = new MockQueryPlan(this);
+    QueryPlan qp = new MockQueryPlan(this, HiveOperation.QUERY);
     //make sure it works with nothing to expire
     testLockExpiration(txnMgr, 0, true);
 
@@ -294,7 +301,7 @@ public class TestDbTxnManager {
     addPartitionInput(t);
     addPartitionInput(t);
     WriteEntity we = addTableOutput(WriteEntity.WriteType.INSERT);
-    QueryPlan qp = new MockQueryPlan(this);
+    QueryPlan qp = new MockQueryPlan(this, HiveOperation.QUERY);
     txnMgr.openTxn(ctx, "fred");
     txnMgr.acquireLocks(qp, ctx, "fred");
     List<HiveLock> locks = ctx.getHiveLocks();
@@ -309,7 +316,7 @@ public class TestDbTxnManager {
   @Test
   public void testUpdate() throws Exception {
     WriteEntity we = addTableOutput(WriteEntity.WriteType.UPDATE);
-    QueryPlan qp = new MockQueryPlan(this);
+    QueryPlan qp = new MockQueryPlan(this, HiveOperation.QUERY);
     txnMgr.openTxn(ctx, "fred");
     txnMgr.acquireLocks(qp, ctx, "fred");
     List<HiveLock> locks = ctx.getHiveLocks();
@@ -324,7 +331,7 @@ public class TestDbTxnManager {
   @Test
   public void testDelete() throws Exception {
     WriteEntity we = addTableOutput(WriteEntity.WriteType.DELETE);
-    QueryPlan qp = new MockQueryPlan(this);
+    QueryPlan qp = new MockQueryPlan(this, HiveOperation.QUERY);
     txnMgr.openTxn(ctx, "fred");
     txnMgr.acquireLocks(qp, ctx, "fred");
     List<HiveLock> locks = ctx.getHiveLocks();
@@ -339,7 +346,7 @@ public class TestDbTxnManager {
   @Test
   public void testRollback() throws Exception {
     WriteEntity we = addTableOutput(WriteEntity.WriteType.DELETE);
-    QueryPlan qp = new MockQueryPlan(this);
+    QueryPlan qp = new MockQueryPlan(this, HiveOperation.QUERY);
     txnMgr.openTxn(ctx, "fred");
     txnMgr.acquireLocks(qp, ctx, "fred");
     List<HiveLock> locks = ctx.getHiveLocks();
@@ -354,13 +361,14 @@ public class TestDbTxnManager {
   @Test
   public void testDDLExclusive() throws Exception {
     WriteEntity we = addTableOutput(WriteEntity.WriteType.DDL_EXCLUSIVE);
-    QueryPlan qp = new MockQueryPlan(this);
+    QueryPlan qp = new MockQueryPlan(this, HiveOperation.DROPTABLE);
+    txnMgr.openTxn(ctx, "fred");
     txnMgr.acquireLocks(qp, ctx, "fred");
     List<HiveLock> locks = ctx.getHiveLocks();
     Assert.assertEquals(1, locks.size());
     Assert.assertEquals(1,
         TxnDbUtil.countLockComponents(((DbLockManager.DbHiveLock) locks.get(0)).lockId));
-    txnMgr.getLockManager().unlock(locks.get(0));
+    txnMgr.rollbackTxn();
     locks = txnMgr.getLockManager().getLocks(false, false);
     Assert.assertEquals(0, locks.size());
   }
@@ -368,13 +376,14 @@ public class TestDbTxnManager {
   @Test
   public void testDDLShared() throws Exception {
     WriteEntity we = addTableOutput(WriteEntity.WriteType.DDL_SHARED);
-    QueryPlan qp = new MockQueryPlan(this);
+    QueryPlan qp = new MockQueryPlan(this, HiveOperation.ALTERTABLE_EXCHANGEPARTITION);
+    txnMgr.openTxn(ctx, "fred");
     txnMgr.acquireLocks(qp, ctx, "fred");
     List<HiveLock> locks = ctx.getHiveLocks();
     Assert.assertEquals(1, locks.size());
     Assert.assertEquals(1,
         TxnDbUtil.countLockComponents(((DbLockManager.DbHiveLock) locks.get(0)).lockId));
-    txnMgr.getLockManager().unlock(locks.get(0));
+    txnMgr.commitTxn();
     locks = txnMgr.getLockManager().getLocks(false, false);
     Assert.assertEquals(0, locks.size());
   }
@@ -382,10 +391,12 @@ public class TestDbTxnManager {
   @Test
   public void testDDLNoLock() throws Exception {
     WriteEntity we = addTableOutput(WriteEntity.WriteType.DDL_NO_LOCK);
-    QueryPlan qp = new MockQueryPlan(this);
+    QueryPlan qp = new MockQueryPlan(this, HiveOperation.CREATEDATABASE);
+    txnMgr.openTxn(ctx, "fred");
     txnMgr.acquireLocks(qp, ctx, "fred");
     List<HiveLock> locks = ctx.getHiveLocks();
     Assert.assertNull(locks);
+    txnMgr.rollbackTxn();
   }
 
   @Test
@@ -406,11 +417,12 @@ public class TestDbTxnManager {
   @Test
   public void testLockAcquisitionAndRelease() throws Exception {
     addTableInput();
-    QueryPlan qp = new MockQueryPlan(this);
+    QueryPlan qp = new MockQueryPlan(this, HiveOperation.QUERY);
+    txnMgr.openTxn(ctx, "fred");
     txnMgr.acquireLocks(qp, ctx, "fred");
     List<HiveLock> locks = ctx.getHiveLocks();
     Assert.assertEquals(1, locks.size());
-    txnMgr.releaseLocks(locks);
+    txnMgr.commitTxn();
     locks = txnMgr.getLockManager().getLocks(false, false);
     Assert.assertEquals(0, locks.size());
   }
@@ -421,7 +433,7 @@ public class TestDbTxnManager {
 
     addTableInput();
     LockException exception = null;
-    QueryPlan qp = new MockQueryPlan(this);
+    QueryPlan qp = new MockQueryPlan(this, HiveOperation.QUERY);
 
     // Case 1: If there's no delay for the heartbeat, txn should be able to commit
     txnMgr.openTxn(ctx, "fred");
@@ -493,7 +505,8 @@ public class TestDbTxnManager {
     private final HashSet<WriteEntity> outputs = new HashSet<>();
     private final String queryId;
     
-    MockQueryPlan(TestDbTxnManager test) {
+    MockQueryPlan(TestDbTxnManager test, HiveOperation operation) {
+      super(operation);
       inputs.addAll(test.readEntities);
       outputs.addAll(test.writeEntities);
       queryId = makeQueryId();


Mime
View raw message