hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From prasan...@apache.org
Subject svn commit: r1631841 [18/42] - in /hive/branches/llap: ./ accumulo-handler/ accumulo-handler/src/java/org/apache/hadoop/hive/accumulo/columns/ accumulo-handler/src/java/org/apache/hadoop/hive/accumulo/mr/ accumulo-handler/src/java/org/apache/hadoop/hiv...
Date Tue, 14 Oct 2014 19:07:05 GMT
Modified: hive/branches/llap/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestNewIntegerEncoding.java
URL: http://svn.apache.org/viewvc/hive/branches/llap/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestNewIntegerEncoding.java?rev=1631841&r1=1631840&r2=1631841&view=diff
==============================================================================
--- hive/branches/llap/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestNewIntegerEncoding.java (original)
+++ hive/branches/llap/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestNewIntegerEncoding.java Tue Oct 14 19:06:45 2014
@@ -335,6 +335,104 @@ public class TestNewIntegerEncoding {
   }
 
   @Test
+  public void testDeltaOverflow() throws Exception {
+    ObjectInspector inspector;
+    synchronized (TestOrcFile.class) {
+      inspector = ObjectInspectorFactory
+          .getReflectionObjectInspector(Long.class,
+              ObjectInspectorFactory.ObjectInspectorOptions.JAVA);
+    }
+
+    long[] inp = new long[]{4513343538618202719l, 4513343538618202711l,
+        2911390882471569739l,
+        -9181829309989854913l};
+    List<Long> input = Lists.newArrayList(Longs.asList(inp));
+
+    Writer writer = OrcFile.createWriter(
+        testFilePath,
+        OrcFile.writerOptions(conf).inspector(inspector).stripeSize(100000)
+            .compress(CompressionKind.NONE).bufferSize(10000));
+    for (Long l : input) {
+      writer.addRow(l);
+    }
+    writer.close();
+
+    Reader reader = OrcFile
+        .createReader(testFilePath, OrcFile.readerOptions(conf).filesystem(fs));
+    RecordReader rows = reader.rows();
+    int idx = 0;
+    while (rows.hasNext()) {
+      Object row = rows.next(null);
+      assertEquals(input.get(idx++).longValue(), ((LongWritable) row).get());
+    }
+  }
+
+  @Test
+  public void testDeltaOverflow2() throws Exception {
+    ObjectInspector inspector;
+    synchronized (TestOrcFile.class) {
+      inspector = ObjectInspectorFactory
+          .getReflectionObjectInspector(Long.class,
+              ObjectInspectorFactory.ObjectInspectorOptions.JAVA);
+    }
+
+    long[] inp = new long[]{Long.MAX_VALUE, 4513343538618202711l,
+        2911390882471569739l,
+        Long.MIN_VALUE};
+    List<Long> input = Lists.newArrayList(Longs.asList(inp));
+
+    Writer writer = OrcFile.createWriter(
+        testFilePath,
+        OrcFile.writerOptions(conf).inspector(inspector).stripeSize(100000)
+            .compress(CompressionKind.NONE).bufferSize(10000));
+    for (Long l : input) {
+      writer.addRow(l);
+    }
+    writer.close();
+
+    Reader reader = OrcFile
+        .createReader(testFilePath, OrcFile.readerOptions(conf).filesystem(fs));
+    RecordReader rows = reader.rows();
+    int idx = 0;
+    while (rows.hasNext()) {
+      Object row = rows.next(null);
+      assertEquals(input.get(idx++).longValue(), ((LongWritable) row).get());
+    }
+  }
+
+  @Test
+  public void testDeltaOverflow3() throws Exception {
+    ObjectInspector inspector;
+    synchronized (TestOrcFile.class) {
+      inspector = ObjectInspectorFactory
+          .getReflectionObjectInspector(Long.class,
+              ObjectInspectorFactory.ObjectInspectorOptions.JAVA);
+    }
+
+    long[] inp = new long[]{-4513343538618202711l, -2911390882471569739l, -2,
+        Long.MAX_VALUE};
+    List<Long> input = Lists.newArrayList(Longs.asList(inp));
+
+    Writer writer = OrcFile.createWriter(
+        testFilePath,
+        OrcFile.writerOptions(conf).inspector(inspector).stripeSize(100000)
+            .compress(CompressionKind.NONE).bufferSize(10000));
+    for (Long l : input) {
+      writer.addRow(l);
+    }
+    writer.close();
+
+    Reader reader = OrcFile
+        .createReader(testFilePath, OrcFile.readerOptions(conf).filesystem(fs));
+    RecordReader rows = reader.rows();
+    int idx = 0;
+    while (rows.hasNext()) {
+      Object row = rows.next(null);
+      assertEquals(input.get(idx++).longValue(), ((LongWritable) row).get());
+    }
+  }
+
+  @Test
   public void testIntegerMin() throws Exception {
     ObjectInspector inspector;
     synchronized (TestOrcFile.class) {

Modified: hive/branches/llap/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcFile.java
URL: http://svn.apache.org/viewvc/hive/branches/llap/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcFile.java?rev=1631841&r1=1631840&r2=1631841&view=diff
==============================================================================
--- hive/branches/llap/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcFile.java (original)
+++ hive/branches/llap/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcFile.java Tue Oct 14 19:06:45 2014
@@ -1754,9 +1754,9 @@ public class TestOrcFile {
           stripe.getDataLength() < 5000);
     }
     // with HIVE-7832, the dictionaries will be disabled after writing the first
-    // stripe as there are too many distinct values. Hence only 3 stripes as
+    // stripe as there are too many distinct values. Hence only 4 stripes as
     // compared to 25 stripes in version 0.11 (above test case)
-    assertEquals(3, i);
+    assertEquals(4, i);
     assertEquals(2500, reader.getNumberOfRows());
   }
 

Modified: hive/branches/llap/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcRawRecordMerger.java
URL: http://svn.apache.org/viewvc/hive/branches/llap/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcRawRecordMerger.java?rev=1631841&r1=1631840&r2=1631841&view=diff
==============================================================================
--- hive/branches/llap/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcRawRecordMerger.java (original)
+++ hive/branches/llap/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestOrcRawRecordMerger.java Tue Oct 14 19:06:45 2014
@@ -56,6 +56,7 @@ import java.util.List;
 
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertNull;
 
 public class TestOrcRawRecordMerger {
@@ -574,12 +575,14 @@ public class TestOrcRawRecordMerger {
         OrcRecordUpdater.getOperation(event));
     assertEquals(new ReaderKey(0, BUCKET, 0, 200), id);
     assertEquals("update 1", getValue(event));
+    assertFalse(merger.isDelete(event));
 
     assertEquals(true, merger.next(id, event));
     assertEquals(OrcRecordUpdater.INSERT_OPERATION,
         OrcRecordUpdater.getOperation(event));
     assertEquals(new ReaderKey(0, BUCKET, 1, 0), id);
     assertEquals("second", getValue(event));
+    assertFalse(merger.isDelete(event));
 
     assertEquals(true, merger.next(id, event));
     assertEquals(OrcRecordUpdater.UPDATE_OPERATION,
@@ -616,6 +619,7 @@ public class TestOrcRawRecordMerger {
         OrcRecordUpdater.getOperation(event));
     assertEquals(new ReaderKey(0, BUCKET, 7, 200), id);
     assertNull(OrcRecordUpdater.getRow(event));
+    assertTrue(merger.isDelete(event));
 
     assertEquals(true, merger.next(id, event));
     assertEquals(OrcRecordUpdater.DELETE_OPERATION,

Modified: hive/branches/llap/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestSerializationUtils.java
URL: http://svn.apache.org/viewvc/hive/branches/llap/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestSerializationUtils.java?rev=1631841&r1=1631840&r2=1631841&view=diff
==============================================================================
--- hive/branches/llap/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestSerializationUtils.java (original)
+++ hive/branches/llap/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestSerializationUtils.java Tue Oct 14 19:06:45 2014
@@ -17,15 +17,18 @@
  */
 package org.apache.hadoop.hive.ql.io.orc;
 
-import org.junit.Test;
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
 
 import java.io.ByteArrayInputStream;
 import java.io.ByteArrayOutputStream;
 import java.io.InputStream;
 import java.math.BigInteger;
 
-import static org.junit.Assert.assertArrayEquals;
-import static org.junit.Assert.assertEquals;
+import org.junit.Test;
+
+import com.google.common.math.LongMath;
 
 public class TestSerializationUtils {
 
@@ -112,6 +115,47 @@ public class TestSerializationUtils {
         SerializationUtils.readBigInteger(fromBuffer(buffer)));
   }
 
+  @Test
+  public void testSubtractionOverflow() {
+    // cross check results with Guava results below
+    SerializationUtils utils = new SerializationUtils();
+    assertEquals(false, utils.isSafeSubtract(22222222222L, Long.MIN_VALUE));
+    assertEquals(false, utils.isSafeSubtract(-22222222222L, Long.MAX_VALUE));
+    assertEquals(false, utils.isSafeSubtract(Long.MIN_VALUE, Long.MAX_VALUE));
+    assertEquals(true, utils.isSafeSubtract(-1553103058346370095L, 6553103058346370095L));
+    assertEquals(true, utils.isSafeSubtract(0, Long.MAX_VALUE));
+    assertEquals(true, utils.isSafeSubtract(Long.MIN_VALUE, 0));
+  }
+
+  @Test
+  public void testSubtractionOverflowGuava() {
+    try {
+      LongMath.checkedSubtract(22222222222L, Long.MIN_VALUE);
+      fail("expected ArithmeticException for overflow");
+    } catch (ArithmeticException ex) {
+      assertEquals(ex.getMessage(), "overflow");
+    }
+
+    try {
+      LongMath.checkedSubtract(-22222222222L, Long.MAX_VALUE);
+      fail("expected ArithmeticException for overflow");
+    } catch (ArithmeticException ex) {
+      assertEquals(ex.getMessage(), "overflow");
+    }
+
+    try {
+      LongMath.checkedSubtract(Long.MIN_VALUE, Long.MAX_VALUE);
+      fail("expected ArithmeticException for overflow");
+    } catch (ArithmeticException ex) {
+      assertEquals(ex.getMessage(), "overflow");
+    }
+
+    assertEquals(-8106206116692740190L,
+        LongMath.checkedSubtract(-1553103058346370095L, 6553103058346370095L));
+    assertEquals(-Long.MAX_VALUE, LongMath.checkedSubtract(0, Long.MAX_VALUE));
+    assertEquals(Long.MIN_VALUE, LongMath.checkedSubtract(Long.MIN_VALUE, 0));
+  }
+
   public static void main(String[] args) throws Exception {
     TestSerializationUtils test = new TestSerializationUtils();
     test.testDoubles();

Modified: hive/branches/llap/ql/src/test/org/apache/hadoop/hive/ql/lockmgr/zookeeper/TestZookeeperLockManager.java
URL: http://svn.apache.org/viewvc/hive/branches/llap/ql/src/test/org/apache/hadoop/hive/ql/lockmgr/zookeeper/TestZookeeperLockManager.java?rev=1631841&r1=1631840&r2=1631841&view=diff
==============================================================================
--- hive/branches/llap/ql/src/test/org/apache/hadoop/hive/ql/lockmgr/zookeeper/TestZookeeperLockManager.java (original)
+++ hive/branches/llap/ql/src/test/org/apache/hadoop/hive/ql/lockmgr/zookeeper/TestZookeeperLockManager.java Tue Oct 14 19:06:45 2014
@@ -25,6 +25,7 @@ import java.util.Collections;
 import org.apache.hadoop.hive.conf.HiveConf;
 import org.apache.hadoop.hive.ql.lockmgr.HiveLockMode;
 import org.apache.hadoop.hive.ql.lockmgr.HiveLockObject;
+import org.apache.hadoop.hive.ql.util.ZooKeeperHiveHelper;
 import org.apache.zookeeper.KeeperException;
 import org.apache.zookeeper.ZooKeeper;
 import org.junit.Assert;
@@ -87,14 +88,14 @@ public class TestZookeeperLockManager {
   public void testGetQuorumServers() {
     conf.setVar(HiveConf.ConfVars.HIVE_ZOOKEEPER_QUORUM, "node1");
     conf.setVar(HiveConf.ConfVars.HIVE_ZOOKEEPER_CLIENT_PORT, "9999");
-    Assert.assertEquals("node1:9999", ZooKeeperHiveLockManager.getQuorumServers(conf));
+    Assert.assertEquals("node1:9999", ZooKeeperHiveHelper.getQuorumServers(conf));
 
     conf.setVar(HiveConf.ConfVars.HIVE_ZOOKEEPER_QUORUM, "node1,node2,node3");
     conf.setVar(HiveConf.ConfVars.HIVE_ZOOKEEPER_CLIENT_PORT, "9999");
-    Assert.assertEquals("node1:9999,node2:9999,node3:9999", ZooKeeperHiveLockManager.getQuorumServers(conf));
+    Assert.assertEquals("node1:9999,node2:9999,node3:9999", ZooKeeperHiveHelper.getQuorumServers(conf));
 
     conf.setVar(HiveConf.ConfVars.HIVE_ZOOKEEPER_QUORUM, "node1:5666,node2,node3");
     conf.setVar(HiveConf.ConfVars.HIVE_ZOOKEEPER_CLIENT_PORT, "9999");
-    Assert.assertEquals("node1:5666,node2:9999,node3:9999", ZooKeeperHiveLockManager.getQuorumServers(conf));
+    Assert.assertEquals("node1:5666,node2:9999,node3:9999", ZooKeeperHiveHelper.getQuorumServers(conf));
   }
 }

Modified: hive/branches/llap/ql/src/test/org/apache/hadoop/hive/ql/metadata/TestHive.java
URL: http://svn.apache.org/viewvc/hive/branches/llap/ql/src/test/org/apache/hadoop/hive/ql/metadata/TestHive.java?rev=1631841&r1=1631840&r2=1631841&view=diff
==============================================================================
--- hive/branches/llap/ql/src/test/org/apache/hadoop/hive/ql/metadata/TestHive.java (original)
+++ hive/branches/llap/ql/src/test/org/apache/hadoop/hive/ql/metadata/TestHive.java Tue Oct 14 19:06:45 2014
@@ -21,14 +21,18 @@ package org.apache.hadoop.hive.ql.metada
 import static org.apache.hadoop.hive.metastore.MetaStoreUtils.DEFAULT_DATABASE_NAME;
 
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.HashMap;
 import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
+import java.util.regex.Pattern;
 
 import junit.framework.TestCase;
 
+import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hive.conf.HiveConf;
 import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
 import org.apache.hadoop.hive.metastore.MetaStoreUtils;
@@ -45,6 +49,7 @@ import org.apache.hadoop.hive.serde.serd
 import org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe;
 import org.apache.hadoop.hive.serde2.thrift.ThriftDeserializer;
 import org.apache.hadoop.hive.serde2.thrift.test.Complex;
+import org.apache.hadoop.hive.shims.ShimLoader;
 import org.apache.hadoop.mapred.SequenceFileInputFormat;
 import org.apache.hadoop.mapred.SequenceFileOutputFormat;
 import org.apache.hadoop.mapred.TextInputFormat;
@@ -63,6 +68,9 @@ public class TestHive extends TestCase {
   protected void setUp() throws Exception {
     super.setUp();
     hiveConf = new HiveConf(this.getClass());
+    // enable trash so it can be tested
+    hiveConf.setFloat("fs.trash.checkpoint.interval", 30);  // FS_TRASH_CHECKPOINT_INTERVAL_KEY (hadoop-2)
+    hiveConf.setFloat("fs.trash.interval", 30);             // FS_TRASH_INTERVAL_KEY (hadoop-2)
     SessionState.start(hiveConf);
     try {
       hm = Hive.get(hiveConf);
@@ -79,6 +87,9 @@ public class TestHive extends TestCase {
   protected void tearDown() throws Exception {
     try {
       super.tearDown();
+      // disable trash
+      hiveConf.setFloat("fs.trash.checkpoint.interval", 30);  // FS_TRASH_CHECKPOINT_INTERVAL_KEY (hadoop-2)
+      hiveConf.setFloat("fs.trash.interval", 30);             // FS_TRASH_INTERVAL_KEY (hadoop-2)
       Hive.closeCurrent();
     } catch (Exception e) {
       System.err.println(StringUtils.stringifyException(e));
@@ -294,7 +305,7 @@ public class TestHive extends TestCase {
     try {
       String dbName = "db_for_testgettables";
       String table1Name = "table1";
-      hm.dropDatabase(dbName, true, true);
+      hm.dropDatabase(dbName, true, true, true);
 
       Database db = new Database();
       db.setName(dbName);
@@ -330,16 +341,92 @@ public class TestHive extends TestCase {
 
       // Drop all tables
       for (String tableName : hm.getAllTables(dbName)) {
+        Table table = hm.getTable(dbName, tableName);
         hm.dropTable(dbName, tableName);
+        assertFalse(fs.exists(table.getPath()));
       }
       hm.dropDatabase(dbName);
     } catch (Throwable e) {
       System.err.println(StringUtils.stringifyException(e));
-      System.err.println("testGetTables() failed");
+      System.err.println("testGetAndDropTables() failed");
       throw e;
     }
   }
 
+  public void testDropTableTrash() throws Throwable {
+    if (!ShimLoader.getHadoopShims().supportTrashFeature()) {
+      return; // it's hadoop-1
+    }
+    try {
+      String dbName = "db_for_testdroptable";
+      hm.dropDatabase(dbName, true, true, true);
+
+      Database db = new Database();
+      db.setName(dbName);
+      hm.createDatabase(db);
+
+      List<String> ts = new ArrayList<String>(2);
+      String tableBaseName = "droptable";
+      ts.add(tableBaseName + "1");
+      ts.add(tableBaseName + "2");
+      Table tbl1 = createTestTable(dbName, ts.get(0));
+      hm.createTable(tbl1);
+      Table tbl2 = createTestTable(dbName, ts.get(1));
+      hm.createTable(tbl2);
+      // test dropping tables and trash behavior
+      Table table1 = hm.getTable(dbName, ts.get(0));
+      assertNotNull(table1);
+      assertEquals(ts.get(0), table1.getTableName());
+      Path path1 = table1.getPath();
+      FileSystem fs = path1.getFileSystem(hiveConf);
+      assertTrue(fs.exists(path1));
+      // drop table and check that trash works
+      Path trashDir = ShimLoader.getHadoopShims().getCurrentTrashPath(hiveConf, fs);
+      assertNotNull("trash directory should not be null", trashDir);
+      Path trash1 = mergePaths(trashDir, path1);
+      Path pathglob = trash1.suffix("*");;
+      FileStatus before[] = fs.globStatus(pathglob);
+      hm.dropTable(dbName, ts.get(0));
+      assertFalse(fs.exists(path1));
+      FileStatus after[] = fs.globStatus(pathglob);
+      assertTrue("trash dir before and after DROP TABLE noPURGE are not different",
+                 before.length != after.length);
+
+      // drop a table without saving to trash by setting the purge option
+      Table table2 = hm.getTable(dbName, ts.get(1));
+      assertNotNull(table2);
+      assertEquals(ts.get(1), table2.getTableName());
+      Path path2 = table2.getPath();
+      assertTrue(fs.exists(path2));
+      Path trash2 = mergePaths(trashDir, path2);
+      System.out.println("trashDir2 is " + trash2);
+      pathglob = trash2.suffix("*");
+      before = fs.globStatus(pathglob);
+      hm.dropTable(dbName, ts.get(1), true, true, true); // deleteData, ignoreUnknownTable, ifPurge
+      assertFalse(fs.exists(path2));
+      after = fs.globStatus(pathglob);
+      Arrays.sort(before);
+      Arrays.sort(after);
+      assertEquals("trash dir before and after DROP TABLE PURGE are different",
+                   before.length, after.length);
+      assertTrue("trash dir before and after DROP TABLE PURGE are different",
+                 Arrays.equals(before, after));
+
+      // Drop all tables
+      for (String tableName : hm.getAllTables(dbName)) {
+        Table table = hm.getTable(dbName, tableName);
+        hm.dropTable(dbName, tableName);
+        assertFalse(fs.exists(table.getPath()));
+      }
+      hm.dropDatabase(dbName);
+    } catch (Throwable e) {
+      System.err.println(StringUtils.stringifyException(e));
+      System.err.println("testDropTableTrash() failed");
+      throw e;
+    }
+  }
+
+
   public void testPartition() throws Throwable {
     try {
       String tableName = "table_for_testpartition";
@@ -533,4 +620,39 @@ public class TestHive extends TestCase {
     newHiveObj = Hive.get(newHconf);
     assertTrue(prevHiveObj != newHiveObj);
   }
+
+  // shamelessly copied from Path in hadoop-2
+  private static final String SEPARATOR = "/";
+  private static final char SEPARATOR_CHAR = '/';
+
+  private static final String CUR_DIR = ".";
+
+  private static final boolean WINDOWS
+      = System.getProperty("os.name").startsWith("Windows");
+
+  private static final Pattern hasDriveLetterSpecifier =
+      Pattern.compile("^/?[a-zA-Z]:");
+
+  private static Path mergePaths(Path path1, Path path2) {
+    String path2Str = path2.toUri().getPath();
+    path2Str = path2Str.substring(startPositionWithoutWindowsDrive(path2Str));
+    // Add path components explicitly, because simply concatenating two path
+    // string is not safe, for example:
+    // "/" + "/foo" yields "//foo", which will be parsed as authority in Path
+    return new Path(path1.toUri().getScheme(),
+        path1.toUri().getAuthority(),
+        path1.toUri().getPath() + path2Str);
+  }
+
+  private static int startPositionWithoutWindowsDrive(String path) {
+    if (hasWindowsDrive(path)) {
+      return path.charAt(0) ==  SEPARATOR_CHAR ? 3 : 2;
+    } else {
+      return 0;
+    }
+  }
+
+  private static boolean hasWindowsDrive(String path) {
+    return (WINDOWS && hasDriveLetterSpecifier.matcher(path).find());
+  }
 }

Modified: hive/branches/llap/ql/src/test/org/apache/hadoop/hive/ql/metadata/TestHiveRemote.java
URL: http://svn.apache.org/viewvc/hive/branches/llap/ql/src/test/org/apache/hadoop/hive/ql/metadata/TestHiveRemote.java?rev=1631841&r1=1631840&r2=1631841&view=diff
==============================================================================
--- hive/branches/llap/ql/src/test/org/apache/hadoop/hive/ql/metadata/TestHiveRemote.java (original)
+++ hive/branches/llap/ql/src/test/org/apache/hadoop/hive/ql/metadata/TestHiveRemote.java Tue Oct 14 19:06:45 2014
@@ -84,6 +84,13 @@ public class TestHiveRemote extends Test
   }
 
   /**
+   * Cannot control trash in remote metastore, so skip this test
+   */
+  @Override
+  public void testDropTableTrash() {
+  }
+
+  /**
    * Finds a free port.
    *
    * @return a free port

Modified: hive/branches/llap/ql/src/test/org/apache/hadoop/hive/ql/parse/TestIUD.java
URL: http://svn.apache.org/viewvc/hive/branches/llap/ql/src/test/org/apache/hadoop/hive/ql/parse/TestIUD.java?rev=1631841&r1=1631840&r2=1631841&view=diff
==============================================================================
--- hive/branches/llap/ql/src/test/org/apache/hadoop/hive/ql/parse/TestIUD.java (original)
+++ hive/branches/llap/ql/src/test/org/apache/hadoop/hive/ql/parse/TestIUD.java Tue Oct 14 19:06:45 2014
@@ -24,6 +24,9 @@ import org.junit.Before;
 import org.junit.BeforeClass;
 import org.junit.Test;
 
+/**
+ * various Parser tests for INSERT/UPDATE/DELETE
+ */
 public class TestIUD {
   private static HiveConf conf;
 
@@ -102,6 +105,18 @@ public class TestIUD {
       ast.toStringTree());
   }
   @Test
+  public void testUpdateWithWhereSingleSetExpr() throws ParseException {
+    ASTNode ast = parse("UPDATE src SET key = -3+(5*9)%8, val = cast(6.1 + c as INT), d = d - 1 WHERE value IS NULL");
+    Assert.assertEquals("AST doesn't match",
+      "(TOK_UPDATE_TABLE (TOK_TABNAME src) " +
+        "(TOK_SET_COLUMNS_CLAUSE " +
+        "(= (TOK_TABLE_OR_COL key) (+ (- 3) (% (* 5 9) 8))) " +
+        "(= (TOK_TABLE_OR_COL val) (TOK_FUNCTION TOK_INT (+ 6.1 (TOK_TABLE_OR_COL c)))) " +
+        "(= (TOK_TABLE_OR_COL d) (- (TOK_TABLE_OR_COL d) 1))) " +
+        "(TOK_WHERE (TOK_FUNCTION TOK_ISNULL (TOK_TABLE_OR_COL value))))",
+      ast.toStringTree());
+  }
+  @Test
   public void testUpdateWithWhereMultiSet() throws ParseException {
     ASTNode ast = parse("UPDATE src SET key = 3, value = 8 WHERE VALUE = 1230997");
     Assert.assertEquals("AST doesn't match", 
@@ -207,13 +222,13 @@ public class TestIUD {
   }
   @Test
   public void testInsertIntoTableFromAnonymousTable() throws ParseException {
-    ASTNode ast = parse("insert into table page_view values(1,2),(3,4)");
+    ASTNode ast = parse("insert into table page_view values(-1,2),(3,+4)");
     Assert.assertEquals("AST doesn't match",
       "(TOK_QUERY " +
         "(TOK_FROM " +
           "(TOK_VIRTUAL_TABLE " +
           "(TOK_VIRTUAL_TABREF TOK_ANONYMOUS) " +
-          "(TOK_VALUES_TABLE (TOK_VALUE_ROW 1 2) (TOK_VALUE_ROW 3 4)))) " +
+          "(TOK_VALUES_TABLE (TOK_VALUE_ROW (- 1) 2) (TOK_VALUE_ROW 3 (+ 4))))) " +
         "(TOK_INSERT (TOK_INSERT_INTO (TOK_TAB (TOK_TABNAME page_view))) " +
           "(TOK_SELECT (TOK_SELEXPR TOK_ALLCOLREF))))",
       ast.toStringTree());

Modified: hive/branches/llap/ql/src/test/org/apache/hadoop/hive/ql/parse/TestUpdateDeleteSemanticAnalyzer.java
URL: http://svn.apache.org/viewvc/hive/branches/llap/ql/src/test/org/apache/hadoop/hive/ql/parse/TestUpdateDeleteSemanticAnalyzer.java?rev=1631841&r1=1631840&r2=1631841&view=diff
==============================================================================
--- hive/branches/llap/ql/src/test/org/apache/hadoop/hive/ql/parse/TestUpdateDeleteSemanticAnalyzer.java (original)
+++ hive/branches/llap/ql/src/test/org/apache/hadoop/hive/ql/parse/TestUpdateDeleteSemanticAnalyzer.java Tue Oct 14 19:06:45 2014
@@ -198,7 +198,7 @@ public class TestUpdateDeleteSemanticAna
   @Test
   public void testInsertValues() throws Exception {
     try {
-      ReturnInfo rc = parseAndAnalyze("insert into table T values ('abc', 3), ('ghi', 5)",
+      ReturnInfo rc = parseAndAnalyze("insert into table T values ('abc', 3), ('ghi', null)",
           "testInsertValues");
 
       LOG.info(explain((SemanticAnalyzer)rc.sem, rc.plan, rc.ast.dump()));
@@ -266,9 +266,12 @@ public class TestUpdateDeleteSemanticAna
 
     // I have to create the tables here (rather than in setup()) because I need the Hive
     // connection, which is conviently created by the semantic analyzer.
-    db.createTable("T", Arrays.asList("a", "b"), null, OrcInputFormat.class, OrcOutputFormat.class);
+    Map<String, String> params = new HashMap<String, String>(1);
+    params.put(SemanticAnalyzer.ACID_TABLE_PROPERTY, "true");
+    db.createTable("T", Arrays.asList("a", "b"), null, OrcInputFormat.class,
+        OrcOutputFormat.class, 2, Arrays.asList("a"), params);
     db.createTable("U", Arrays.asList("a", "b"), Arrays.asList("ds"), OrcInputFormat.class,
-        OrcOutputFormat.class);
+        OrcOutputFormat.class, 2, Arrays.asList("a"), params);
     Table u = db.getTable("U");
     Map<String, String> partVals = new HashMap<String, String>(2);
     partVals.put("ds", "yesterday");
@@ -280,7 +283,7 @@ public class TestUpdateDeleteSemanticAna
     // validate the plan
     sem.validate();
 
-    QueryPlan plan = new QueryPlan(query, sem, 0L, testName);
+    QueryPlan plan = new QueryPlan(query, sem, 0L, testName, null);
 
     return new ReturnInfo(tree, sem, plan);
   }

Modified: hive/branches/llap/ql/src/test/org/apache/hadoop/hive/ql/plan/TestTezWork.java
URL: http://svn.apache.org/viewvc/hive/branches/llap/ql/src/test/org/apache/hadoop/hive/ql/plan/TestTezWork.java?rev=1631841&r1=1631840&r2=1631841&view=diff
==============================================================================
--- hive/branches/llap/ql/src/test/org/apache/hadoop/hive/ql/plan/TestTezWork.java (original)
+++ hive/branches/llap/ql/src/test/org/apache/hadoop/hive/ql/plan/TestTezWork.java Tue Oct 14 19:06:45 2014
@@ -23,11 +23,16 @@ import java.util.List;
 import junit.framework.Assert;
 
 import org.apache.hadoop.hive.ql.plan.TezEdgeProperty.EdgeType;
+import org.apache.hadoop.mapred.JobConf;
 import org.junit.Before;
 import org.junit.Test;
+import org.mockito.Mockito;
+import org.mockito.invocation.InvocationOnMock;
+import org.mockito.stubbing.Answer;
 
 public class TestTezWork {
 
+  private static final String MR_JAR_PROPERTY = "tmpjars";
   private List<BaseWork> nodes;
   private TezWork work;
 
@@ -156,4 +161,75 @@ public class TestTezWork {
       Assert.assertEquals(sorted.get(i), nodes.get(4-i));
     }
   }
+
+  @Test
+  public void testConfigureJars() throws Exception {
+    final JobConf conf = new JobConf();
+    conf.set(MR_JAR_PROPERTY, "file:///tmp/foo1.jar");
+    BaseWork baseWork = Mockito.mock(BaseWork.class);
+    Mockito.doAnswer(new Answer<Void>() {
+
+      @Override
+      public Void answer(InvocationOnMock invocation) throws Throwable {
+        conf.set(MR_JAR_PROPERTY, "file:///tmp/foo2.jar");
+        return null;
+      }
+
+    }).when(baseWork).configureJobConf(conf);
+
+    work.add(baseWork);
+    work.configureJobConfAndExtractJars(conf);
+    Assert.assertEquals("file:///tmp/foo1.jar,file:///tmp/foo2.jar", conf.get(MR_JAR_PROPERTY));
+  }
+
+  @Test
+  public void testConfigureJarsNoExtraJars() throws Exception {
+    final JobConf conf = new JobConf();
+    conf.set(MR_JAR_PROPERTY, "file:///tmp/foo1.jar");
+    BaseWork baseWork = Mockito.mock(BaseWork.class);
+
+    work.add(baseWork);
+    work.configureJobConfAndExtractJars(conf);
+    Assert.assertEquals("file:///tmp/foo1.jar", conf.get(MR_JAR_PROPERTY));
+  }
+
+  @Test
+  public void testConfigureJarsWithNull() throws Exception {
+    final JobConf conf = new JobConf();
+    conf.set(MR_JAR_PROPERTY, "file:///tmp/foo1.jar");
+    BaseWork baseWork = Mockito.mock(BaseWork.class);
+    Mockito.doAnswer(new Answer<Void>() {
+
+      @Override
+      public Void answer(InvocationOnMock invocation) throws Throwable {
+        conf.unset(MR_JAR_PROPERTY);
+        return null;
+      }
+
+    }).when(baseWork).configureJobConf(conf);
+
+    work.add(baseWork);
+    work.configureJobConfAndExtractJars(conf);
+    Assert.assertEquals("file:///tmp/foo1.jar", conf.get(MR_JAR_PROPERTY));
+  }
+
+  @Test
+  public void testConfigureJarsStartingWithNull() throws Exception {
+    final JobConf conf = new JobConf();
+    conf.unset(MR_JAR_PROPERTY);
+    BaseWork baseWork = Mockito.mock(BaseWork.class);
+    Mockito.doAnswer(new Answer<Void>() {
+
+      @Override
+      public Void answer(InvocationOnMock invocation) throws Throwable {
+        conf.setStrings(MR_JAR_PROPERTY, "file:///tmp/foo1.jar", "file:///tmp/foo2.jar");
+        return null;
+      }
+
+    }).when(baseWork).configureJobConf(conf);
+
+    work.add(baseWork);
+    work.configureJobConfAndExtractJars(conf);
+    Assert.assertEquals("file:///tmp/foo1.jar,file:///tmp/foo2.jar", conf.get(MR_JAR_PROPERTY));
+  }
 }

Modified: hive/branches/llap/ql/src/test/org/apache/hadoop/hive/ql/security/authorization/plugin/sqlstd/TestSQLStdHiveAccessControllerCLI.java
URL: http://svn.apache.org/viewvc/hive/branches/llap/ql/src/test/org/apache/hadoop/hive/ql/security/authorization/plugin/sqlstd/TestSQLStdHiveAccessControllerCLI.java?rev=1631841&r1=1631840&r2=1631841&view=diff
==============================================================================
--- hive/branches/llap/ql/src/test/org/apache/hadoop/hive/ql/security/authorization/plugin/sqlstd/TestSQLStdHiveAccessControllerCLI.java (original)
+++ hive/branches/llap/ql/src/test/org/apache/hadoop/hive/ql/security/authorization/plugin/sqlstd/TestSQLStdHiveAccessControllerCLI.java Tue Oct 14 19:06:45 2014
@@ -25,6 +25,8 @@ import org.apache.hadoop.hive.conf.HiveC
 import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
 import org.apache.hadoop.hive.ql.security.HadoopDefaultAuthenticator;
 import org.apache.hadoop.hive.ql.security.authorization.plugin.DisallowTransformHook;
+import org.apache.hadoop.hive.ql.security.authorization.plugin.HiveAuthorizer;
+import org.apache.hadoop.hive.ql.security.authorization.plugin.HiveAuthorizerFactory;
 import org.apache.hadoop.hive.ql.security.authorization.plugin.HiveAuthzPluginException;
 import org.apache.hadoop.hive.ql.security.authorization.plugin.HiveAuthzSessionContext;
 import org.apache.hadoop.hive.ql.security.authorization.plugin.HiveAuthzSessionContext.Builder;
@@ -77,8 +79,9 @@ public class TestSQLStdHiveAccessControl
     HiveConf processedConf = new HiveConf();
     processedConf.setBoolVar(ConfVars.HIVE_AUTHORIZATION_ENABLED, true);
     try {
-      SQLStdHiveAccessController accessController = new SQLStdHiveAccessController(null,
-          processedConf, new HadoopDefaultAuthenticator(), getCLISessionCtx());
+      HiveAuthorizerFactory authorizerFactory = new SQLStdHiveAuthorizerFactory();
+      HiveAuthorizer authorizer = authorizerFactory.createHiveAuthorizer(null, processedConf,
+          new HadoopDefaultAuthenticator(), getCLISessionCtx());
       fail("Exception expected");
     } catch (HiveAuthzPluginException e) {
       assertTrue(e.getMessage().contains(

Modified: hive/branches/llap/ql/src/test/org/apache/hadoop/hive/ql/txn/compactor/CompactorTest.java
URL: http://svn.apache.org/viewvc/hive/branches/llap/ql/src/test/org/apache/hadoop/hive/ql/txn/compactor/CompactorTest.java?rev=1631841&r1=1631840&r2=1631841&view=diff
==============================================================================
--- hive/branches/llap/ql/src/test/org/apache/hadoop/hive/ql/txn/compactor/CompactorTest.java (original)
+++ hive/branches/llap/ql/src/test/org/apache/hadoop/hive/ql/txn/compactor/CompactorTest.java Tue Oct 14 19:06:45 2014
@@ -63,12 +63,13 @@ public abstract class CompactorTest {
   protected CompactionTxnHandler txnHandler;
   protected IMetaStoreClient ms;
   protected long sleepTime = 1000;
+  protected HiveConf conf;
 
   private final MetaStoreThread.BooleanPointer stop = new MetaStoreThread.BooleanPointer();
   private final File tmpdir;
 
   protected CompactorTest() throws Exception {
-    HiveConf conf = new HiveConf();
+    conf = new HiveConf();
     TxnDbUtil.setConfValues(conf);
     TxnDbUtil.cleanDb();
     ms = new HiveMetaStoreClient(conf);
@@ -79,16 +80,20 @@ public abstract class CompactorTest {
     tmpdir.deleteOnExit();
   }
 
-  protected void startInitiator(HiveConf conf) throws Exception {
-    startThread('i', conf);
+  protected void startInitiator() throws Exception {
+    startThread('i', true);
   }
 
-  protected void startWorker(HiveConf conf) throws Exception {
-    startThread('w', conf);
+  protected void startWorker() throws Exception {
+    startThread('w', true);
   }
 
-  protected void startCleaner(HiveConf conf) throws Exception {
-    startThread('c', conf);
+  protected void startCleaner() throws Exception {
+    startThread('c', true);
+  }
+
+  protected void startCleaner(MetaStoreThread.BooleanPointer looped) throws Exception {
+    startThread('c', false, looped);
   }
 
   protected Table newTable(String dbName, String tableName, boolean partitioned) throws TException {
@@ -117,6 +122,9 @@ public abstract class CompactorTest {
 
     table.setParameters(parameters);
 
+    // drop the table first, in case some previous test created it
+    ms.dropTable(dbName, tableName);
+
     ms.createTable(table);
     return table;
   }
@@ -142,37 +150,27 @@ public abstract class CompactorTest {
     return txns.get(0);
   }
 
-  protected void addDeltaFile(HiveConf conf, Table t, Partition p, long minTxn, long maxTxn,
-                              int numRecords) throws Exception{
-    addFile(conf, t, p, minTxn, maxTxn, numRecords, FileType.DELTA, 2, true);
-  }
-
-  protected void addBaseFile(HiveConf conf, Table t, Partition p, long maxTxn,
-                             int numRecords) throws Exception{
-    addFile(conf, t, p, 0, maxTxn, numRecords, FileType.BASE, 2, true);
+  protected void addDeltaFile(Table t, Partition p, long minTxn, long maxTxn, int numRecords)
+      throws Exception {
+    addFile(t, p, minTxn, maxTxn, numRecords, FileType.DELTA, 2, true);
   }
 
-  protected void addLegacyFile(HiveConf conf, Table t, Partition p,
-                               int numRecords) throws Exception {
-    addFile(conf, t, p, 0, 0, numRecords, FileType.LEGACY, 2, true);
+  protected void addBaseFile(Table t, Partition p, long maxTxn, int numRecords) throws Exception {
+    addFile(t, p, 0, maxTxn, numRecords, FileType.BASE, 2, true);
   }
 
-  protected void addDeltaFile(HiveConf conf, Table t, Partition p, long minTxn, long maxTxn,
-                              int numRecords, int numBuckets, boolean allBucketsPresent)
-      throws Exception {
-    addFile(conf, t, p, minTxn, maxTxn, numRecords, FileType.DELTA, numBuckets, allBucketsPresent);
+  protected void addLegacyFile(Table t, Partition p, int numRecords) throws Exception {
+    addFile(t, p, 0, 0, numRecords, FileType.LEGACY, 2, true);
   }
 
-  protected void addBaseFile(HiveConf conf, Table t, Partition p, long maxTxn,
-                             int numRecords, int numBuckets, boolean allBucketsPresent)
-      throws Exception {
-    addFile(conf, t, p, 0, maxTxn, numRecords, FileType.BASE, numBuckets, allBucketsPresent);
+  protected void addDeltaFile(Table t, Partition p, long minTxn, long maxTxn, int numRecords,
+                              int numBuckets, boolean allBucketsPresent) throws Exception {
+    addFile(t, p, minTxn, maxTxn, numRecords, FileType.DELTA, numBuckets, allBucketsPresent);
   }
 
-  protected void addLegacyFile(HiveConf conf, Table t, Partition p,
-                               int numRecords, int numBuckets, boolean allBucketsPresent)
-      throws Exception {
-    addFile(conf, t, p, 0, 0, numRecords, FileType.LEGACY, numBuckets, allBucketsPresent);
+  protected void addBaseFile(Table t, Partition p, long maxTxn, int numRecords, int numBuckets,
+                             boolean allBucketsPresent) throws Exception {
+    addFile(t, p, 0, maxTxn, numRecords, FileType.BASE, numBuckets, allBucketsPresent);
   }
 
   protected List<Path> getDirectories(HiveConf conf, Table t, Partition p) throws Exception {
@@ -191,6 +189,10 @@ public abstract class CompactorTest {
     for (long tid : rsp.getTxn_ids()) txnHandler.commitTxn(new CommitTxnRequest(tid));
   }
 
+  protected void stopThread() {
+    stop.boolVal = true;
+  }
+
   private StorageDescriptor newStorageDescriptor(String location, List<Order> sortCols) {
     StorageDescriptor sd = new StorageDescriptor();
     List<FieldSchema> cols = new ArrayList<FieldSchema>(2);
@@ -214,9 +216,13 @@ public abstract class CompactorTest {
     return sd;
   }
 
-  // I can't do this with @Before because I want to be able to control the config file provided
-  // to each test.
-  private void startThread(char type, HiveConf conf) throws Exception {
+  // I can't do this with @Before because I want to be able to control when the thead starts
+  private void startThread(char type, boolean stopAfterOne) throws Exception {
+    startThread(type, stopAfterOne, new MetaStoreThread.BooleanPointer());
+  }
+
+  private void startThread(char type, boolean stopAfterOne, MetaStoreThread.BooleanPointer looped)
+    throws Exception {
     TxnDbUtil.setConfValues(conf);
     CompactorThread t = null;
     switch (type) {
@@ -227,9 +233,10 @@ public abstract class CompactorTest {
     }
     t.setThreadId((int) t.getId());
     t.setHiveConf(conf);
-    stop.boolVal = true;
-    t.init(stop);
-    t.run();
+    stop.boolVal = stopAfterOne;
+    t.init(stop, looped);
+    if (stopAfterOne) t.run();
+    else t.start();
   }
 
   private String getLocation(String tableName, String partValue) {
@@ -243,7 +250,7 @@ public abstract class CompactorTest {
 
   private enum FileType {BASE, DELTA, LEGACY};
 
-  private void addFile(HiveConf conf, Table t, Partition p, long minTxn, long maxTxn,
+  private void addFile(Table t, Partition p, long minTxn, long maxTxn,
                        int numRecords,  FileType type, int numBuckets,
                        boolean allBucketsPresent) throws Exception {
     String partValue = (p == null) ? null : p.getValues().get(0);
@@ -332,6 +339,7 @@ public abstract class CompactorTest {
     private final Configuration conf;
     private FSDataInputStream is = null;
     private final FileSystem fs;
+    private boolean lastWasDelete = true;
 
     MockRawReader(Configuration conf, List<Path> files) throws IOException {
       filesToRead = new Stack<Path>();
@@ -346,6 +354,15 @@ public abstract class CompactorTest {
     }
 
     @Override
+    public boolean isDelete(Text value) {
+      // Alternate between returning deleted and not.  This is easier than actually
+      // tracking operations. We test that this is getting properly called by checking that only
+      // half the records show up in base files after major compactions.
+      lastWasDelete = !lastWasDelete;
+      return lastWasDelete;
+    }
+
+    @Override
     public boolean next(RecordIdentifier identifier, Text text) throws IOException {
       if (is == null) {
         // Open the next file

Modified: hive/branches/llap/ql/src/test/org/apache/hadoop/hive/ql/txn/compactor/TestCleaner.java
URL: http://svn.apache.org/viewvc/hive/branches/llap/ql/src/test/org/apache/hadoop/hive/ql/txn/compactor/TestCleaner.java?rev=1631841&r1=1631840&r2=1631841&view=diff
==============================================================================
--- hive/branches/llap/ql/src/test/org/apache/hadoop/hive/ql/txn/compactor/TestCleaner.java (original)
+++ hive/branches/llap/ql/src/test/org/apache/hadoop/hive/ql/txn/compactor/TestCleaner.java Tue Oct 14 19:06:45 2014
@@ -18,21 +18,26 @@
 package org.apache.hadoop.hive.ql.txn.compactor;
 
 import junit.framework.Assert;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.metastore.MetaStoreThread;
 import org.apache.hadoop.hive.metastore.api.*;
 import org.apache.hadoop.hive.metastore.txn.CompactionInfo;
-import org.apache.hadoop.hive.metastore.txn.TxnDbUtil;
-import org.junit.Before;
 import org.junit.Test;
 
 import java.util.ArrayList;
 import java.util.List;
+import java.util.concurrent.TimeUnit;
 
 /**
  * Tests for the compactor Cleaner thread
  */
 public class TestCleaner extends CompactorTest {
+
+  static final private Log LOG = LogFactory.getLog(TestCleaner.class.getName());
+
   public TestCleaner() throws Exception {
     super();
   }
@@ -41,19 +46,17 @@ public class TestCleaner extends Compact
   public void nothing() throws Exception {
     // Test that the whole things works when there's nothing in the queue.  This is just a
     // survival test.
-    startCleaner(new HiveConf());
+    startCleaner();
   }
 
   @Test
   public void cleanupAfterMajorTableCompaction() throws Exception {
     Table t = newTable("default", "camtc", false);
 
-    HiveConf conf = new HiveConf();
-
-    addBaseFile(conf, t, null, 20L, 20);
-    addDeltaFile(conf, t, null, 21L, 22L, 2);
-    addDeltaFile(conf, t, null, 23L, 24L, 2);
-    addBaseFile(conf, t, null, 25L, 25);
+    addBaseFile(t, null, 20L, 20);
+    addDeltaFile(t, null, 21L, 22L, 2);
+    addDeltaFile(t, null, 23L, 24L, 2);
+    addBaseFile(t, null, 25L, 25);
 
     burnThroughTransactions(25);
 
@@ -63,7 +66,7 @@ public class TestCleaner extends Compact
     txnHandler.markCompacted(ci);
     txnHandler.setRunAs(ci.id, System.getProperty("user.name"));
 
-    startCleaner(conf);
+    startCleaner();
 
     // Check there are no compactions requests left.
     ShowCompactResponse rsp = txnHandler.showCompact(new ShowCompactRequest());
@@ -80,12 +83,10 @@ public class TestCleaner extends Compact
     Table t = newTable("default", "campc", true);
     Partition p = newPartition(t, "today");
 
-    HiveConf conf = new HiveConf();
-
-    addBaseFile(conf, t, p, 20L, 20);
-    addDeltaFile(conf, t, p, 21L, 22L, 2);
-    addDeltaFile(conf, t, p, 23L, 24L, 2);
-    addBaseFile(conf, t, p, 25L, 25);
+    addBaseFile(t, p, 20L, 20);
+    addDeltaFile(t, p, 21L, 22L, 2);
+    addDeltaFile(t, p, 23L, 24L, 2);
+    addBaseFile(t, p, 25L, 25);
 
     burnThroughTransactions(25);
 
@@ -96,7 +97,7 @@ public class TestCleaner extends Compact
     txnHandler.markCompacted(ci);
     txnHandler.setRunAs(ci.id, System.getProperty("user.name"));
 
-    startCleaner(conf);
+    startCleaner();
 
     // Check there are no compactions requests left.
     ShowCompactResponse rsp = txnHandler.showCompact(new ShowCompactRequest());
@@ -112,12 +113,10 @@ public class TestCleaner extends Compact
   public void cleanupAfterMinorTableCompaction() throws Exception {
     Table t = newTable("default", "camitc", false);
 
-    HiveConf conf = new HiveConf();
-
-    addBaseFile(conf, t, null, 20L, 20);
-    addDeltaFile(conf, t, null, 21L, 22L, 2);
-    addDeltaFile(conf, t, null, 23L, 24L, 2);
-    addDeltaFile(conf, t, null, 21L, 24L, 4);
+    addBaseFile(t, null, 20L, 20);
+    addDeltaFile(t, null, 21L, 22L, 2);
+    addDeltaFile(t, null, 23L, 24L, 2);
+    addDeltaFile(t, null, 21L, 24L, 4);
 
     burnThroughTransactions(25);
 
@@ -127,7 +126,7 @@ public class TestCleaner extends Compact
     txnHandler.markCompacted(ci);
     txnHandler.setRunAs(ci.id, System.getProperty("user.name"));
 
-    startCleaner(conf);
+    startCleaner();
 
     // Check there are no compactions requests left.
     ShowCompactResponse rsp = txnHandler.showCompact(new ShowCompactRequest());
@@ -151,12 +150,10 @@ public class TestCleaner extends Compact
     Table t = newTable("default", "camipc", true);
     Partition p = newPartition(t, "today");
 
-    HiveConf conf = new HiveConf();
-
-    addBaseFile(conf, t, p, 20L, 20);
-    addDeltaFile(conf, t, p, 21L, 22L, 2);
-    addDeltaFile(conf, t, p, 23L, 24L, 2);
-    addDeltaFile(conf, t, p, 21L, 24L, 4);
+    addBaseFile(t, p, 20L, 20);
+    addDeltaFile(t, p, 21L, 22L, 2);
+    addDeltaFile(t, p, 23L, 24L, 2);
+    addDeltaFile(t, p, 21L, 24L, 4);
 
     burnThroughTransactions(25);
 
@@ -167,7 +164,7 @@ public class TestCleaner extends Compact
     txnHandler.markCompacted(ci);
     txnHandler.setRunAs(ci.id, System.getProperty("user.name"));
 
-    startCleaner(conf);
+    startCleaner();
 
     // Check there are no compactions requests left.
     ShowCompactResponse rsp = txnHandler.showCompact(new ShowCompactRequest());
@@ -190,12 +187,10 @@ public class TestCleaner extends Compact
   public void blockedByLockTable() throws Exception {
     Table t = newTable("default", "bblt", false);
 
-    HiveConf conf = new HiveConf();
-
-    addBaseFile(conf, t, null, 20L, 20);
-    addDeltaFile(conf, t, null, 21L, 22L, 2);
-    addDeltaFile(conf, t, null, 23L, 24L, 2);
-    addDeltaFile(conf, t, null, 21L, 24L, 4);
+    addBaseFile(t, null, 20L, 20);
+    addDeltaFile(t, null, 21L, 22L, 2);
+    addDeltaFile(t, null, 23L, 24L, 2);
+    addDeltaFile(t, null, 21L, 24L, 4);
 
     burnThroughTransactions(25);
 
@@ -212,7 +207,7 @@ public class TestCleaner extends Compact
     LockRequest req = new LockRequest(components, "me", "localhost");
     LockResponse res = txnHandler.lock(req);
 
-    startCleaner(conf);
+    startCleaner();
 
     // Check there are no compactions requests left.
     ShowCompactResponse rsp = txnHandler.showCompact(new ShowCompactRequest());
@@ -228,12 +223,10 @@ public class TestCleaner extends Compact
     Table t = newTable("default", "bblp", true);
     Partition p = newPartition(t, "today");
 
-    HiveConf conf = new HiveConf();
-
-    addBaseFile(conf, t, p, 20L, 20);
-    addDeltaFile(conf, t, p, 21L, 22L, 2);
-    addDeltaFile(conf, t, p, 23L, 24L, 2);
-    addDeltaFile(conf, t, p, 21L, 24L, 4);
+    addBaseFile(t, p, 20L, 20);
+    addDeltaFile(t, p, 21L, 22L, 2);
+    addDeltaFile(t, p, 23L, 24L, 2);
+    addDeltaFile(t, p, 21L, 24L, 4);
 
     burnThroughTransactions(25);
 
@@ -244,7 +237,7 @@ public class TestCleaner extends Compact
     txnHandler.markCompacted(ci);
     txnHandler.setRunAs(ci.id, System.getProperty("user.name"));
 
-    LockComponent comp = new LockComponent(LockType.SHARED_WRITE, LockLevel.TABLE, "default");
+    LockComponent comp = new LockComponent(LockType.SHARED_WRITE, LockLevel.PARTITION, "default");
     comp.setTablename("bblp");
     comp.setPartitionname("ds=today");
     List<LockComponent> components = new ArrayList<LockComponent>(1);
@@ -252,7 +245,7 @@ public class TestCleaner extends Compact
     LockRequest req = new LockRequest(components, "me", "localhost");
     LockResponse res = txnHandler.lock(req);
 
-    startCleaner(conf);
+    startCleaner();
 
     // Check there are no compactions requests left.
     ShowCompactResponse rsp = txnHandler.showCompact(new ShowCompactRequest());
@@ -265,15 +258,154 @@ public class TestCleaner extends Compact
   }
 
   @Test
+  public void notBlockedBySubsequentLock() throws Exception {
+    Table t = newTable("default", "bblt", false);
+
+    // Set the run frequency low on this test so it doesn't take long
+    conf.setTimeVar(HiveConf.ConfVars.HIVE_COMPACTOR_CLEANER_RUN_INTERVAL, 100,
+        TimeUnit.MILLISECONDS);
+
+    addBaseFile(t, null, 20L, 20);
+    addDeltaFile(t, null, 21L, 22L, 2);
+    addDeltaFile(t, null, 23L, 24L, 2);
+    addDeltaFile(t, null, 21L, 24L, 4);
+
+    burnThroughTransactions(25);
+
+    CompactionRequest rqst = new CompactionRequest("default", "bblt", CompactionType.MINOR);
+    txnHandler.compact(rqst);
+    CompactionInfo ci = txnHandler.findNextToCompact("fred");
+    txnHandler.markCompacted(ci);
+    txnHandler.setRunAs(ci.id, System.getProperty("user.name"));
+
+    LockComponent comp = new LockComponent(LockType.SHARED_READ, LockLevel.TABLE, "default");
+    comp.setTablename("bblt");
+    List<LockComponent> components = new ArrayList<LockComponent>(1);
+    components.add(comp);
+    LockRequest req = new LockRequest(components, "me", "localhost");
+    LockResponse res = txnHandler.lock(req);
+
+    MetaStoreThread.BooleanPointer looped = new MetaStoreThread.BooleanPointer();
+    looped.boolVal = false;
+    startCleaner(looped);
+
+    // Make sure the compactor has a chance to run once
+    while (!looped.boolVal) {
+      Thread.currentThread().sleep(100);
+    }
+
+    // There should still be one request, as the locks still held.
+    ShowCompactResponse rsp = txnHandler.showCompact(new ShowCompactRequest());
+    List<ShowCompactResponseElement> compacts = rsp.getCompacts();
+    Assert.assertEquals(1, compacts.size());
+
+    // obtain a second lock.  This shouldn't block cleaner as it was acquired after the initial
+    // clean request
+    LockComponent comp2 = new LockComponent(LockType.SHARED_READ, LockLevel.TABLE, "default");
+    comp2.setTablename("bblt");
+    List<LockComponent> components2 = new ArrayList<LockComponent>(1);
+    components2.add(comp2);
+    LockRequest req2 = new LockRequest(components, "me", "localhost");
+    LockResponse res2 = txnHandler.lock(req2);
+
+    // Unlock the previous lock
+    txnHandler.unlock(new UnlockRequest(res.getLockid()));
+    looped.boolVal = false;
+
+    while (!looped.boolVal) {
+      Thread.currentThread().sleep(100);
+    }
+    stopThread();
+    Thread.currentThread().sleep(200);
+
+
+    // Check there are no compactions requests left.
+    rsp = txnHandler.showCompact(new ShowCompactRequest());
+    compacts = rsp.getCompacts();
+    Assert.assertEquals(0, compacts.size());
+  }
+
+  @Test
+  public void partitionNotBlockedBySubsequentLock() throws Exception {
+    Table t = newTable("default", "bblt", true);
+    Partition p = newPartition(t, "today");
+
+    // Set the run frequency low on this test so it doesn't take long
+    conf.setTimeVar(HiveConf.ConfVars.HIVE_COMPACTOR_CLEANER_RUN_INTERVAL, 100,
+        TimeUnit.MILLISECONDS);
+
+    addBaseFile(t, p, 20L, 20);
+    addDeltaFile(t, p, 21L, 22L, 2);
+    addDeltaFile(t, p, 23L, 24L, 2);
+    addDeltaFile(t, p, 21L, 24L, 4);
+
+    burnThroughTransactions(25);
+
+    CompactionRequest rqst = new CompactionRequest("default", "bblt", CompactionType.MINOR);
+    rqst.setPartitionname("ds=today");
+    txnHandler.compact(rqst);
+    CompactionInfo ci = txnHandler.findNextToCompact("fred");
+    txnHandler.markCompacted(ci);
+    txnHandler.setRunAs(ci.id, System.getProperty("user.name"));
+
+    LockComponent comp = new LockComponent(LockType.SHARED_READ, LockLevel.PARTITION, "default");
+    comp.setTablename("bblt");
+    comp.setPartitionname("ds=today");
+    List<LockComponent> components = new ArrayList<LockComponent>(1);
+    components.add(comp);
+    LockRequest req = new LockRequest(components, "me", "localhost");
+    LockResponse res = txnHandler.lock(req);
+
+    MetaStoreThread.BooleanPointer looped = new MetaStoreThread.BooleanPointer();
+    looped.boolVal = false;
+    startCleaner(looped);
+
+    // Make sure the compactor has a chance to run once
+    while (!looped.boolVal) {
+      Thread.currentThread().sleep(100);
+    }
+
+    // There should still be one request, as the locks still held.
+    ShowCompactResponse rsp = txnHandler.showCompact(new ShowCompactRequest());
+    List<ShowCompactResponseElement> compacts = rsp.getCompacts();
+    Assert.assertEquals(1, compacts.size());
+
+
+    // obtain a second lock.  This shouldn't block cleaner as it was acquired after the initial
+    // clean request
+    LockComponent comp2 = new LockComponent(LockType.SHARED_READ, LockLevel.PARTITION, "default");
+    comp2.setTablename("bblt");
+    comp2.setPartitionname("ds=today");
+    List<LockComponent> components2 = new ArrayList<LockComponent>(1);
+    components2.add(comp2);
+    LockRequest req2 = new LockRequest(components, "me", "localhost");
+    LockResponse res2 = txnHandler.lock(req2);
+
+    // Unlock the previous lock
+    txnHandler.unlock(new UnlockRequest(res.getLockid()));
+    looped.boolVal = false;
+
+    while (!looped.boolVal) {
+      Thread.currentThread().sleep(100);
+    }
+    stopThread();
+    Thread.currentThread().sleep(200);
+
+
+    // Check there are no compactions requests left.
+    rsp = txnHandler.showCompact(new ShowCompactRequest());
+    compacts = rsp.getCompacts();
+    Assert.assertEquals(0, compacts.size());
+  }
+
+  @Test
   public void cleanupAfterMajorPartitionCompactionNoBase() throws Exception {
     Table t = newTable("default", "campcnb", true);
     Partition p = newPartition(t, "today");
 
-    HiveConf conf = new HiveConf();
-
-    addDeltaFile(conf, t, p, 1L, 22L, 22);
-    addDeltaFile(conf, t, p, 23L, 24L, 2);
-    addBaseFile(conf, t, p, 25L, 25);
+    addDeltaFile(t, p, 1L, 22L, 22);
+    addDeltaFile(t, p, 23L, 24L, 2);
+    addBaseFile(t, p, 25L, 25);
 
     burnThroughTransactions(25);
 
@@ -284,7 +416,7 @@ public class TestCleaner extends Compact
     txnHandler.markCompacted(ci);
     txnHandler.setRunAs(ci.id, System.getProperty("user.name"));
 
-    startCleaner(conf);
+    startCleaner();
 
     // Check there are no compactions requests left.
     ShowCompactResponse rsp = txnHandler.showCompact(new ShowCompactRequest());
@@ -295,9 +427,4 @@ public class TestCleaner extends Compact
     Assert.assertEquals(1, paths.size());
     Assert.assertEquals("base_25", paths.get(0).getName());
   }
-
-  @Before
-  public void setUpTxnDb() throws Exception {
-    TxnDbUtil.setConfValues(new HiveConf());
-  }
 }

Modified: hive/branches/llap/ql/src/test/org/apache/hadoop/hive/ql/txn/compactor/TestInitiator.java
URL: http://svn.apache.org/viewvc/hive/branches/llap/ql/src/test/org/apache/hadoop/hive/ql/txn/compactor/TestInitiator.java?rev=1631841&r1=1631840&r2=1631841&view=diff
==============================================================================
--- hive/branches/llap/ql/src/test/org/apache/hadoop/hive/ql/txn/compactor/TestInitiator.java (original)
+++ hive/branches/llap/ql/src/test/org/apache/hadoop/hive/ql/txn/compactor/TestInitiator.java Tue Oct 14 19:06:45 2014
@@ -47,7 +47,7 @@ public class TestInitiator extends Compa
   public void nothing() throws Exception {
     // Test that the whole things works when there's nothing in the queue.  This is just a
     // survival test.
-    startInitiator(new HiveConf());
+    startInitiator();
   }
 
   @Test
@@ -63,7 +63,7 @@ public class TestInitiator extends Compa
     txnHandler.findNextToCompact(Worker.hostname() + "-193892");
     txnHandler.findNextToCompact("nosuchhost-193892");
 
-    startInitiator(new HiveConf());
+    startInitiator();
 
     ShowCompactResponse rsp = txnHandler.showCompact(new ShowCompactRequest());
     List<ShowCompactResponseElement> compacts = rsp.getCompacts();
@@ -89,10 +89,9 @@ public class TestInitiator extends Compa
 
     txnHandler.findNextToCompact("nosuchhost-193892");
 
-    HiveConf conf = new HiveConf();
     conf.setTimeVar(HiveConf.ConfVars.HIVE_COMPACTOR_WORKER_TIMEOUT, 1L, TimeUnit.MILLISECONDS);
 
-    startInitiator(conf);
+    startInitiator();
 
     ShowCompactResponse rsp = txnHandler.showCompact(new ShowCompactRequest());
     List<ShowCompactResponseElement> compacts = rsp.getCompacts();
@@ -104,7 +103,6 @@ public class TestInitiator extends Compa
   public void majorCompactOnTableTooManyAborts() throws Exception {
     Table t = newTable("default", "mcottma", false);
 
-    HiveConf conf = new HiveConf();
     HiveConf.setIntVar(conf, HiveConf.ConfVars.HIVE_COMPACTOR_ABORTEDTXN_THRESHOLD, 10);
 
     for (int i = 0; i < 11; i++) {
@@ -119,7 +117,7 @@ public class TestInitiator extends Compa
       txnHandler.abortTxn(new AbortTxnRequest(txnid));
     }
 
-    startInitiator(conf);
+    startInitiator();
 
     ShowCompactResponse rsp = txnHandler.showCompact(new ShowCompactRequest());
     List<ShowCompactResponseElement> compacts = rsp.getCompacts();
@@ -134,7 +132,6 @@ public class TestInitiator extends Compa
     Table t = newTable("default", "mcoptma", true);
     Partition p = newPartition(t, "today");
 
-    HiveConf conf = new HiveConf();
     HiveConf.setIntVar(conf, HiveConf.ConfVars.HIVE_COMPACTOR_ABORTEDTXN_THRESHOLD, 10);
 
     for (int i = 0; i < 11; i++) {
@@ -150,7 +147,7 @@ public class TestInitiator extends Compa
       txnHandler.abortTxn(new AbortTxnRequest(txnid));
     }
 
-    startInitiator(conf);
+    startInitiator();
 
     ShowCompactResponse rsp = txnHandler.showCompact(new ShowCompactRequest());
     List<ShowCompactResponseElement> compacts = rsp.getCompacts();
@@ -168,7 +165,6 @@ public class TestInitiator extends Compa
       Partition p = newPartition(t, "day-" + i);
     }
 
-    HiveConf conf = new HiveConf();
     HiveConf.setIntVar(conf, HiveConf.ConfVars.HIVE_COMPACTOR_ABORTEDTXN_THRESHOLD, 10);
 
     for (int i = 0; i < 11; i++) {
@@ -184,7 +180,7 @@ public class TestInitiator extends Compa
       txnHandler.abortTxn(new AbortTxnRequest(txnid));
     }
 
-    startInitiator(conf);
+    startInitiator();
 
     ShowCompactResponse rsp = txnHandler.showCompact(new ShowCompactRequest());
     Assert.assertEquals(0, rsp.getCompactsSize());
@@ -197,8 +193,6 @@ public class TestInitiator extends Compa
     // accidently clean it too.
     Table t = newTable("default", "ceat", false);
 
-    HiveConf conf = new HiveConf();
-
     long txnid = openTxn();
     LockComponent comp = new LockComponent(LockType.SHARED_WRITE, LockLevel.TABLE, "default");
     comp.setTablename("ceat");
@@ -216,7 +210,7 @@ public class TestInitiator extends Compa
     GetOpenTxnsResponse openTxns = txnHandler.getOpenTxns();
     Assert.assertEquals(101, openTxns.getOpen_txnsSize());
 
-    startInitiator(conf);
+    startInitiator();
 
     openTxns = txnHandler.getOpenTxns();
     Assert.assertEquals(1, openTxns.getOpen_txnsSize());
@@ -228,7 +222,6 @@ public class TestInitiator extends Compa
     parameters.put("NO_AUTO_COMPACTION", "true");
     Table t = newTable("default", "ncwncs", false, parameters);
 
-    HiveConf conf = new HiveConf();
     HiveConf.setIntVar(conf, HiveConf.ConfVars.HIVE_COMPACTOR_ABORTEDTXN_THRESHOLD, 10);
 
     for (int i = 0; i < 11; i++) {
@@ -243,7 +236,7 @@ public class TestInitiator extends Compa
       txnHandler.abortTxn(new AbortTxnRequest(txnid));
     }
 
-    startInitiator(conf);
+    startInitiator();
 
     ShowCompactResponse rsp = txnHandler.showCompact(new ShowCompactRequest());
     Assert.assertEquals(0, rsp.getCompactsSize());
@@ -253,7 +246,6 @@ public class TestInitiator extends Compa
   public void noCompactWhenCompactAlreadyScheduled() throws Exception {
     Table t = newTable("default", "ncwcas", false);
 
-    HiveConf conf = new HiveConf();
     HiveConf.setIntVar(conf, HiveConf.ConfVars.HIVE_COMPACTOR_ABORTEDTXN_THRESHOLD, 10);
 
     for (int i = 0; i < 11; i++) {
@@ -277,7 +269,7 @@ public class TestInitiator extends Compa
     Assert.assertEquals("initiated", compacts.get(0).getState());
     Assert.assertEquals("ncwcas", compacts.get(0).getTablename());
 
-    startInitiator(conf);
+    startInitiator();
 
     rsp = txnHandler.showCompact(new ShowCompactRequest());
     compacts = rsp.getCompacts();
@@ -291,11 +283,9 @@ public class TestInitiator extends Compa
   public void compactTableHighDeltaPct() throws Exception {
     Table t = newTable("default", "cthdp", false);
 
-    HiveConf conf = new HiveConf();
-
-    addBaseFile(conf, t, null, 20L, 20);
-    addDeltaFile(conf, t, null, 21L, 22L, 2);
-    addDeltaFile(conf, t, null, 23L, 24L, 2);
+    addBaseFile(t, null, 20L, 20);
+    addDeltaFile(t, null, 21L, 22L, 2);
+    addDeltaFile(t, null, 23L, 24L, 2);
 
     burnThroughTransactions(23);
 
@@ -309,7 +299,7 @@ public class TestInitiator extends Compa
     LockResponse res = txnHandler.lock(req);
     txnHandler.commitTxn(new CommitTxnRequest(txnid));
 
-    startInitiator(conf);
+    startInitiator();
 
     ShowCompactResponse rsp = txnHandler.showCompact(new ShowCompactRequest());
     List<ShowCompactResponseElement> compacts = rsp.getCompacts();
@@ -324,11 +314,9 @@ public class TestInitiator extends Compa
     Table t = newTable("default", "cphdp", true);
     Partition p = newPartition(t, "today");
 
-    HiveConf conf = new HiveConf();
-
-    addBaseFile(conf, t, p, 20L, 20);
-    addDeltaFile(conf, t, p, 21L, 22L, 2);
-    addDeltaFile(conf, t, p, 23L, 24L, 2);
+    addBaseFile(t, p, 20L, 20);
+    addDeltaFile(t, p, 21L, 22L, 2);
+    addDeltaFile(t, p, 23L, 24L, 2);
 
     burnThroughTransactions(23);
 
@@ -343,7 +331,7 @@ public class TestInitiator extends Compa
     LockResponse res = txnHandler.lock(req);
     txnHandler.commitTxn(new CommitTxnRequest(txnid));
 
-    startInitiator(conf);
+    startInitiator();
 
     ShowCompactResponse rsp = txnHandler.showCompact(new ShowCompactRequest());
     List<ShowCompactResponseElement> compacts = rsp.getCompacts();
@@ -358,11 +346,9 @@ public class TestInitiator extends Compa
   public void noCompactTableDeltaPctNotHighEnough() throws Exception {
     Table t = newTable("default", "nctdpnhe", false);
 
-    HiveConf conf = new HiveConf();
-
-    addBaseFile(conf, t, null, 50L, 50);
-    addDeltaFile(conf, t, null, 21L, 22L, 2);
-    addDeltaFile(conf, t, null, 23L, 24L, 2);
+    addBaseFile(t, null, 50L, 50);
+    addDeltaFile(t, null, 21L, 22L, 2);
+    addDeltaFile(t, null, 23L, 24L, 2);
 
     burnThroughTransactions(53);
 
@@ -376,7 +362,7 @@ public class TestInitiator extends Compa
     LockResponse res = txnHandler.lock(req);
     txnHandler.commitTxn(new CommitTxnRequest(txnid));
 
-    startInitiator(conf);
+    startInitiator();
 
     ShowCompactResponse rsp = txnHandler.showCompact(new ShowCompactRequest());
     Assert.assertEquals(0, rsp.getCompactsSize());
@@ -386,20 +372,18 @@ public class TestInitiator extends Compa
   public void compactTableTooManyDeltas() throws Exception {
     Table t = newTable("default", "cttmd", false);
 
-    HiveConf conf = new HiveConf();
-
-    addBaseFile(conf, t, null, 200L, 200);
-    addDeltaFile(conf, t, null, 201L, 201L, 1);
-    addDeltaFile(conf, t, null, 202L, 202L, 1);
-    addDeltaFile(conf, t, null, 203L, 203L, 1);
-    addDeltaFile(conf, t, null, 204L, 204L, 1);
-    addDeltaFile(conf, t, null, 205L, 205L, 1);
-    addDeltaFile(conf, t, null, 206L, 206L, 1);
-    addDeltaFile(conf, t, null, 207L, 207L, 1);
-    addDeltaFile(conf, t, null, 208L, 208L, 1);
-    addDeltaFile(conf, t, null, 209L, 209L, 1);
-    addDeltaFile(conf, t, null, 210L, 210L, 1);
-    addDeltaFile(conf, t, null, 211L, 211L, 1);
+    addBaseFile(t, null, 200L, 200);
+    addDeltaFile(t, null, 201L, 201L, 1);
+    addDeltaFile(t, null, 202L, 202L, 1);
+    addDeltaFile(t, null, 203L, 203L, 1);
+    addDeltaFile(t, null, 204L, 204L, 1);
+    addDeltaFile(t, null, 205L, 205L, 1);
+    addDeltaFile(t, null, 206L, 206L, 1);
+    addDeltaFile(t, null, 207L, 207L, 1);
+    addDeltaFile(t, null, 208L, 208L, 1);
+    addDeltaFile(t, null, 209L, 209L, 1);
+    addDeltaFile(t, null, 210L, 210L, 1);
+    addDeltaFile(t, null, 211L, 211L, 1);
 
     burnThroughTransactions(210);
 
@@ -413,7 +397,7 @@ public class TestInitiator extends Compa
     LockResponse res = txnHandler.lock(req);
     txnHandler.commitTxn(new CommitTxnRequest(txnid));
 
-    startInitiator(conf);
+    startInitiator();
 
     ShowCompactResponse rsp = txnHandler.showCompact(new ShowCompactRequest());
     List<ShowCompactResponseElement> compacts = rsp.getCompacts();
@@ -428,20 +412,18 @@ public class TestInitiator extends Compa
     Table t = newTable("default", "cptmd", true);
     Partition p = newPartition(t, "today");
 
-    HiveConf conf = new HiveConf();
-
-    addBaseFile(conf, t, p, 200L, 200);
-    addDeltaFile(conf, t, p, 201L, 201L, 1);
-    addDeltaFile(conf, t, p, 202L, 202L, 1);
-    addDeltaFile(conf, t, p, 203L, 203L, 1);
-    addDeltaFile(conf, t, p, 204L, 204L, 1);
-    addDeltaFile(conf, t, p, 205L, 205L, 1);
-    addDeltaFile(conf, t, p, 206L, 206L, 1);
-    addDeltaFile(conf, t, p, 207L, 207L, 1);
-    addDeltaFile(conf, t, p, 208L, 208L, 1);
-    addDeltaFile(conf, t, p, 209L, 209L, 1);
-    addDeltaFile(conf, t, p, 210L, 210L, 1);
-    addDeltaFile(conf, t, p, 211L, 211L, 1);
+    addBaseFile(t, p, 200L, 200);
+    addDeltaFile(t, p, 201L, 201L, 1);
+    addDeltaFile(t, p, 202L, 202L, 1);
+    addDeltaFile(t, p, 203L, 203L, 1);
+    addDeltaFile(t, p, 204L, 204L, 1);
+    addDeltaFile(t, p, 205L, 205L, 1);
+    addDeltaFile(t, p, 206L, 206L, 1);
+    addDeltaFile(t, p, 207L, 207L, 1);
+    addDeltaFile(t, p, 208L, 208L, 1);
+    addDeltaFile(t, p, 209L, 209L, 1);
+    addDeltaFile(t, p, 210L, 210L, 1);
+    addDeltaFile(t, p, 211L, 211L, 1);
 
     burnThroughTransactions(210);
 
@@ -456,7 +438,7 @@ public class TestInitiator extends Compa
     LockResponse res = txnHandler.lock(req);
     txnHandler.commitTxn(new CommitTxnRequest(txnid));
 
-    startInitiator(conf);
+    startInitiator();
 
     ShowCompactResponse rsp = txnHandler.showCompact(new ShowCompactRequest());
     List<ShowCompactResponseElement> compacts = rsp.getCompacts();
@@ -471,11 +453,9 @@ public class TestInitiator extends Compa
   public void noCompactTableNotEnoughDeltas() throws Exception {
     Table t = newTable("default", "nctned", false);
 
-    HiveConf conf = new HiveConf();
-
-    addBaseFile(conf, t, null, 200L, 200);
-    addDeltaFile(conf, t, null, 201L, 205L, 5);
-    addDeltaFile(conf, t, null, 206L, 211L, 6);
+    addBaseFile(t, null, 200L, 200);
+    addDeltaFile(t, null, 201L, 205L, 5);
+    addDeltaFile(t, null, 206L, 211L, 6);
 
     burnThroughTransactions(210);
 
@@ -489,7 +469,7 @@ public class TestInitiator extends Compa
     LockResponse res = txnHandler.lock(req);
     txnHandler.commitTxn(new CommitTxnRequest(txnid));
 
-    startInitiator(conf);
+    startInitiator();
 
     ShowCompactResponse rsp = txnHandler.showCompact(new ShowCompactRequest());
     Assert.assertEquals(0, rsp.getCompactsSize());
@@ -499,20 +479,18 @@ public class TestInitiator extends Compa
   public void chooseMajorOverMinorWhenBothValid() throws Exception {
     Table t = newTable("default", "cmomwbv", false);
 
-    HiveConf conf = new HiveConf();
-
-    addBaseFile(conf, t, null, 200L, 200);
-    addDeltaFile(conf, t, null, 201L, 211L, 11);
-    addDeltaFile(conf, t, null, 212L, 222L, 11);
-    addDeltaFile(conf, t, null, 223L, 233L, 11);
-    addDeltaFile(conf, t, null, 234L, 244L, 11);
-    addDeltaFile(conf, t, null, 245L, 255L, 11);
-    addDeltaFile(conf, t, null, 256L, 266L, 11);
-    addDeltaFile(conf, t, null, 267L, 277L, 11);
-    addDeltaFile(conf, t, null, 278L, 288L, 11);
-    addDeltaFile(conf, t, null, 289L, 299L, 11);
-    addDeltaFile(conf, t, null, 300L, 310L, 11);
-    addDeltaFile(conf, t, null, 311L, 321L, 11);
+    addBaseFile(t, null, 200L, 200);
+    addDeltaFile(t, null, 201L, 211L, 11);
+    addDeltaFile(t, null, 212L, 222L, 11);
+    addDeltaFile(t, null, 223L, 233L, 11);
+    addDeltaFile(t, null, 234L, 244L, 11);
+    addDeltaFile(t, null, 245L, 255L, 11);
+    addDeltaFile(t, null, 256L, 266L, 11);
+    addDeltaFile(t, null, 267L, 277L, 11);
+    addDeltaFile(t, null, 278L, 288L, 11);
+    addDeltaFile(t, null, 289L, 299L, 11);
+    addDeltaFile(t, null, 300L, 310L, 11);
+    addDeltaFile(t, null, 311L, 321L, 11);
 
     burnThroughTransactions(320);
 
@@ -526,7 +504,7 @@ public class TestInitiator extends Compa
     LockResponse res = txnHandler.lock(req);
     txnHandler.commitTxn(new CommitTxnRequest(txnid));
 
-    startInitiator(conf);
+    startInitiator();
 
     ShowCompactResponse rsp = txnHandler.showCompact(new ShowCompactRequest());
     List<ShowCompactResponseElement> compacts = rsp.getCompacts();
@@ -541,19 +519,17 @@ public class TestInitiator extends Compa
     Table t = newTable("default", "ednb", true);
     Partition p = newPartition(t, "today");
 
-    HiveConf conf = new HiveConf();
-
-    addDeltaFile(conf, t, p, 1L, 201L, 200);
-    addDeltaFile(conf, t, p, 202L, 202L, 1);
-    addDeltaFile(conf, t, p, 203L, 203L, 1);
-    addDeltaFile(conf, t, p, 204L, 204L, 1);
-    addDeltaFile(conf, t, p, 205L, 205L, 1);
-    addDeltaFile(conf, t, p, 206L, 206L, 1);
-    addDeltaFile(conf, t, p, 207L, 207L, 1);
-    addDeltaFile(conf, t, p, 208L, 208L, 1);
-    addDeltaFile(conf, t, p, 209L, 209L, 1);
-    addDeltaFile(conf, t, p, 210L, 210L, 1);
-    addDeltaFile(conf, t, p, 211L, 211L, 1);
+    addDeltaFile(t, p, 1L, 201L, 200);
+    addDeltaFile(t, p, 202L, 202L, 1);
+    addDeltaFile(t, p, 203L, 203L, 1);
+    addDeltaFile(t, p, 204L, 204L, 1);
+    addDeltaFile(t, p, 205L, 205L, 1);
+    addDeltaFile(t, p, 206L, 206L, 1);
+    addDeltaFile(t, p, 207L, 207L, 1);
+    addDeltaFile(t, p, 208L, 208L, 1);
+    addDeltaFile(t, p, 209L, 209L, 1);
+    addDeltaFile(t, p, 210L, 210L, 1);
+    addDeltaFile(t, p, 211L, 211L, 1);
 
     burnThroughTransactions(210);
 
@@ -568,7 +544,7 @@ public class TestInitiator extends Compa
     LockResponse res = txnHandler.lock(req);
     txnHandler.commitTxn(new CommitTxnRequest(txnid));
 
-    startInitiator(conf);
+    startInitiator();
 
     ShowCompactResponse rsp = txnHandler.showCompact(new ShowCompactRequest());
     List<ShowCompactResponseElement> compacts = rsp.getCompacts();
@@ -584,11 +560,9 @@ public class TestInitiator extends Compa
     Table t = newTable("default", "ttospgocr", true);
     Partition p = newPartition(t, "today");
 
-    HiveConf conf = new HiveConf();
-
-    addBaseFile(conf, t, p, 20L, 20);
-    addDeltaFile(conf, t, p, 21L, 22L, 2);
-    addDeltaFile(conf, t, p, 23L, 24L, 2);
+    addBaseFile(t, p, 20L, 20);
+    addDeltaFile(t, p, 21L, 22L, 2);
+    addDeltaFile(t, p, 23L, 24L, 2);
 
     burnThroughTransactions(23);
 
@@ -614,7 +588,7 @@ public class TestInitiator extends Compa
     res = txnHandler.lock(req);
     txnHandler.commitTxn(new CommitTxnRequest(txnid));
 
-    startInitiator(conf);
+    startInitiator();
 
     ShowCompactResponse rsp = txnHandler.showCompact(new ShowCompactRequest());
     List<ShowCompactResponseElement> compacts = rsp.getCompacts();
@@ -626,9 +600,4 @@ public class TestInitiator extends Compa
   }
 
   // TODO test compactions with legacy file types
-
-  @Before
-  public void setUpTxnDb() throws Exception {
-    TxnDbUtil.setConfValues(new HiveConf());
-  }
 }

Modified: hive/branches/llap/ql/src/test/org/apache/hadoop/hive/ql/txn/compactor/TestWorker.java
URL: http://svn.apache.org/viewvc/hive/branches/llap/ql/src/test/org/apache/hadoop/hive/ql/txn/compactor/TestWorker.java?rev=1631841&r1=1631840&r2=1631841&view=diff
==============================================================================
--- hive/branches/llap/ql/src/test/org/apache/hadoop/hive/ql/txn/compactor/TestWorker.java (original)
+++ hive/branches/llap/ql/src/test/org/apache/hadoop/hive/ql/txn/compactor/TestWorker.java Tue Oct 14 19:06:45 2014
@@ -48,7 +48,7 @@ public class TestWorker extends Compacto
   public void nothing() throws Exception {
     // Test that the whole things works when there's nothing in the queue.  This is just a
     // survival test.
-    startWorker(new HiveConf());
+    startWorker();
   }
 
   @Test
@@ -205,19 +205,17 @@ public class TestWorker extends Compacto
 
     Table t = newTable("default", "st", false, new HashMap<String, String>(), sortCols);
 
-    HiveConf conf = new HiveConf();
-
-    addBaseFile(conf, t, null, 20L, 20);
-    addDeltaFile(conf, t, null, 21L, 22L, 2);
-    addDeltaFile(conf, t, null, 23L, 24L, 2);
-    addDeltaFile(conf, t, null, 21L, 24L, 4);
+    addBaseFile(t, null, 20L, 20);
+    addDeltaFile(t, null, 21L, 22L, 2);
+    addDeltaFile(t, null, 23L, 24L, 2);
+    addDeltaFile(t, null, 21L, 24L, 4);
 
     burnThroughTransactions(25);
 
     CompactionRequest rqst = new CompactionRequest("default", "st", CompactionType.MINOR);
     txnHandler.compact(rqst);
 
-    startWorker(new HiveConf());
+    startWorker();
 
     // There should still be four directories in the location.
     FileSystem fs = FileSystem.get(conf);
@@ -232,12 +230,11 @@ public class TestWorker extends Compacto
 
     Table t = newTable("default", "sp", true, new HashMap<String, String>(), sortCols);
     Partition p = newPartition(t, "today", sortCols);
-    HiveConf conf = new HiveConf();
 
-    addBaseFile(conf, t, p, 20L, 20);
-    addDeltaFile(conf, t, p, 21L, 22L, 2);
-    addDeltaFile(conf, t, p, 23L, 24L, 2);
-    addDeltaFile(conf, t, p, 21L, 24L, 4);
+    addBaseFile(t, p, 20L, 20);
+    addDeltaFile(t, p, 21L, 22L, 2);
+    addDeltaFile(t, p, 23L, 24L, 2);
+    addDeltaFile(t, p, 21L, 24L, 4);
 
     burnThroughTransactions(25);
 
@@ -245,7 +242,7 @@ public class TestWorker extends Compacto
     rqst.setPartitionname("ds=today");
     txnHandler.compact(rqst);
 
-    startWorker(new HiveConf());
+    startWorker();
 
     // There should still be four directories in the location.
     FileSystem fs = FileSystem.get(conf);
@@ -258,18 +255,16 @@ public class TestWorker extends Compacto
     LOG.debug("Starting minorTableWithBase");
     Table t = newTable("default", "mtwb", false);
 
-    HiveConf conf = new HiveConf();
-
-    addBaseFile(conf, t, null, 20L, 20);
-    addDeltaFile(conf, t, null, 21L, 22L, 2);
-    addDeltaFile(conf, t, null, 23L, 24L, 2);
+    addBaseFile(t, null, 20L, 20);
+    addDeltaFile(t, null, 21L, 22L, 2);
+    addDeltaFile(t, null, 23L, 24L, 2);
 
     burnThroughTransactions(25);
 
     CompactionRequest rqst = new CompactionRequest("default", "mtwb", CompactionType.MINOR);
     txnHandler.compact(rqst);
 
-    startWorker(conf);
+    startWorker();
 
     ShowCompactResponse rsp = txnHandler.showCompact(new ShowCompactRequest());
     List<ShowCompactResponseElement> compacts = rsp.getCompacts();
@@ -304,11 +299,10 @@ public class TestWorker extends Compacto
   public void minorPartitionWithBase() throws Exception {
     Table t = newTable("default", "mpwb", true);
     Partition p = newPartition(t, "today");
-    HiveConf conf = new HiveConf();
 
-    addBaseFile(conf, t, p, 20L, 20);
-    addDeltaFile(conf, t, p, 21L, 22L, 2);
-    addDeltaFile(conf, t, p, 23L, 24L, 2);
+    addBaseFile(t, p, 20L, 20);
+    addDeltaFile(t, p, 21L, 22L, 2);
+    addDeltaFile(t, p, 23L, 24L, 2);
 
     burnThroughTransactions(25);
 
@@ -316,7 +310,7 @@ public class TestWorker extends Compacto
     rqst.setPartitionname("ds=today");
     txnHandler.compact(rqst);
 
-    startWorker(new HiveConf());
+    startWorker();
 
     ShowCompactResponse rsp = txnHandler.showCompact(new ShowCompactRequest());
     List<ShowCompactResponseElement> compacts = rsp.getCompacts();
@@ -351,17 +345,15 @@ public class TestWorker extends Compacto
     LOG.debug("Starting minorTableWithBase");
     Table t = newTable("default", "mtnb", false);
 
-    HiveConf conf = new HiveConf();
-
-    addDeltaFile(conf, t, null, 1L, 2L, 2);
-    addDeltaFile(conf, t, null, 3L, 4L, 2);
+    addDeltaFile(t, null, 1L, 2L, 2);
+    addDeltaFile(t, null, 3L, 4L, 2);
 
     burnThroughTransactions(5);
 
     CompactionRequest rqst = new CompactionRequest("default", "mtnb", CompactionType.MINOR);
     txnHandler.compact(rqst);
 
-    startWorker(new HiveConf());
+    startWorker();
 
     ShowCompactResponse rsp = txnHandler.showCompact(new ShowCompactRequest());
     List<ShowCompactResponseElement> compacts = rsp.getCompacts();
@@ -396,18 +388,16 @@ public class TestWorker extends Compacto
     LOG.debug("Starting majorTableWithBase");
     Table t = newTable("default", "matwb", false);
 
-    HiveConf conf = new HiveConf();
-
-    addBaseFile(conf, t, null, 20L, 20);
-    addDeltaFile(conf, t, null, 21L, 22L, 2);
-    addDeltaFile(conf, t, null, 23L, 24L, 2);
+    addBaseFile(t, null, 20L, 20);
+    addDeltaFile(t, null, 21L, 22L, 2);
+    addDeltaFile(t, null, 23L, 24L, 2);
 
     burnThroughTransactions(25);
 
     CompactionRequest rqst = new CompactionRequest("default", "matwb", CompactionType.MAJOR);
     txnHandler.compact(rqst);
 
-    startWorker(new HiveConf());
+    startWorker();
 
     ShowCompactResponse rsp = txnHandler.showCompact(new ShowCompactRequest());
     List<ShowCompactResponseElement> compacts = rsp.getCompacts();
@@ -428,8 +418,8 @@ public class TestWorker extends Compacto
         Assert.assertEquals(2, buckets.length);
         Assert.assertTrue(buckets[0].getPath().getName().matches("bucket_0000[01]"));
         Assert.assertTrue(buckets[1].getPath().getName().matches("bucket_0000[01]"));
-        Assert.assertEquals(1248L, buckets[0].getLen());
-        Assert.assertEquals(1248L, buckets[1].getLen());
+        Assert.assertEquals(624L, buckets[0].getLen());
+        Assert.assertEquals(624L, buckets[1].getLen());
       } else {
         LOG.debug("This is not the file you are looking for " + stat[i].getPath().getName());
       }
@@ -442,11 +432,10 @@ public class TestWorker extends Compacto
     LOG.debug("Starting majorPartitionWithBase");
     Table t = newTable("default", "mapwb", true);
     Partition p = newPartition(t, "today");
-    HiveConf conf = new HiveConf();
 
-    addBaseFile(conf, t, p, 20L, 20);
-    addDeltaFile(conf, t, p, 21L, 22L, 2);
-    addDeltaFile(conf, t, p, 23L, 24L, 2);
+    addBaseFile(t, p, 20L, 20);
+    addDeltaFile(t, p, 21L, 22L, 2);
+    addDeltaFile(t, p, 23L, 24L, 2);
 
     burnThroughTransactions(25);
 
@@ -454,7 +443,7 @@ public class TestWorker extends Compacto
     rqst.setPartitionname("ds=today");
     txnHandler.compact(rqst);
 
-    startWorker(new HiveConf());
+    startWorker();
 
     ShowCompactResponse rsp = txnHandler.showCompact(new ShowCompactRequest());
     List<ShowCompactResponseElement> compacts = rsp.getCompacts();
@@ -475,8 +464,8 @@ public class TestWorker extends Compacto
         Assert.assertEquals(2, buckets.length);
         Assert.assertTrue(buckets[0].getPath().getName().matches("bucket_0000[01]"));
         Assert.assertTrue(buckets[1].getPath().getName().matches("bucket_0000[01]"));
-        Assert.assertEquals(1248L, buckets[0].getLen());
-        Assert.assertEquals(1248L, buckets[1].getLen());
+        Assert.assertEquals(624L, buckets[0].getLen());
+        Assert.assertEquals(624L, buckets[1].getLen());
       } else {
         LOG.debug("This is not the file you are looking for " + stat[i].getPath().getName());
       }
@@ -489,17 +478,15 @@ public class TestWorker extends Compacto
     LOG.debug("Starting majorTableNoBase");
     Table t = newTable("default", "matnb", false);
 
-    HiveConf conf = new HiveConf();
-
-    addDeltaFile(conf, t, null, 1L, 2L, 2);
-    addDeltaFile(conf, t, null, 3L, 4L, 2);
+    addDeltaFile(t, null, 1L, 2L, 2);
+    addDeltaFile(t, null, 3L, 4L, 2);
 
     burnThroughTransactions(5);
 
     CompactionRequest rqst = new CompactionRequest("default", "matnb", CompactionType.MAJOR);
     txnHandler.compact(rqst);
 
-    startWorker(new HiveConf());
+    startWorker();
 
     ShowCompactResponse rsp = txnHandler.showCompact(new ShowCompactRequest());
     List<ShowCompactResponseElement> compacts = rsp.getCompacts();
@@ -520,8 +507,8 @@ public class TestWorker extends Compacto
         Assert.assertEquals(2, buckets.length);
         Assert.assertTrue(buckets[0].getPath().getName().matches("bucket_0000[01]"));
         Assert.assertTrue(buckets[1].getPath().getName().matches("bucket_0000[01]"));
-        Assert.assertEquals(208L, buckets[0].getLen());
-        Assert.assertEquals(208L, buckets[1].getLen());
+        Assert.assertEquals(104L, buckets[0].getLen());
+        Assert.assertEquals(104L, buckets[1].getLen());
       } else {
         LOG.debug("This is not the file you are looking for " + stat[i].getPath().getName());
       }
@@ -534,18 +521,16 @@ public class TestWorker extends Compacto
     LOG.debug("Starting majorTableLegacy");
     Table t = newTable("default", "matl", false);
 
-    HiveConf conf = new HiveConf();
-
-    addLegacyFile(conf, t, null, 20);
-    addDeltaFile(conf, t, null, 21L, 22L, 2);
-    addDeltaFile(conf, t, null, 23L, 24L, 2);
+    addLegacyFile(t, null, 20);
+    addDeltaFile(t, null, 21L, 22L, 2);
+    addDeltaFile(t, null, 23L, 24L, 2);
 
     burnThroughTransactions(25);
 
     CompactionRequest rqst = new CompactionRequest("default", "matl", CompactionType.MAJOR);
     txnHandler.compact(rqst);
 
-    startWorker(new HiveConf());
+    startWorker();
 
     ShowCompactResponse rsp = txnHandler.showCompact(new ShowCompactRequest());
     List<ShowCompactResponseElement> compacts = rsp.getCompacts();
@@ -566,8 +551,8 @@ public class TestWorker extends Compacto
         Assert.assertEquals(2, buckets.length);
         Assert.assertTrue(buckets[0].getPath().getName().matches("bucket_0000[01]"));
         Assert.assertTrue(buckets[1].getPath().getName().matches("bucket_0000[01]"));
-        Assert.assertEquals(1248L, buckets[0].getLen());
-        Assert.assertEquals(1248L, buckets[1].getLen());
+        Assert.assertEquals(624L, buckets[0].getLen());
+        Assert.assertEquals(624L, buckets[1].getLen());
       } else {
         LOG.debug("This is not the file you are looking for " + stat[i].getPath().getName());
       }
@@ -580,18 +565,16 @@ public class TestWorker extends Compacto
     LOG.debug("Starting minorTableLegacy");
     Table t = newTable("default", "mtl", false);
 
-    HiveConf conf = new HiveConf();
-
-    addLegacyFile(conf, t, null, 20);
-    addDeltaFile(conf, t, null, 21L, 22L, 2);
-    addDeltaFile(conf, t, null, 23L, 24L, 2);
+    addLegacyFile(t, null, 20);
+    addDeltaFile(t, null, 21L, 22L, 2);
+    addDeltaFile(t, null, 23L, 24L, 2);
 
     burnThroughTransactions(25);
 
     CompactionRequest rqst = new CompactionRequest("default", "mtl", CompactionType.MINOR);
     txnHandler.compact(rqst);
 
-    startWorker(new HiveConf());
+    startWorker();
 
     ShowCompactResponse rsp = txnHandler.showCompact(new ShowCompactRequest());
     List<ShowCompactResponseElement> compacts = rsp.getCompacts();
@@ -622,11 +605,11 @@ public class TestWorker extends Compacto
   public void majorPartitionWithBaseMissingBuckets() throws Exception {
     Table t = newTable("default", "mapwbmb", true);
     Partition p = newPartition(t, "today");
-    HiveConf conf = new HiveConf();
 
-    addBaseFile(conf, t, p, 20L, 20, 2, false);
-    addDeltaFile(conf, t, p, 21L, 22L, 2, 2, false);
-    addDeltaFile(conf, t, p, 23L, 24L, 2);
+
+    addBaseFile(t, p, 20L, 20, 2, false);
+    addDeltaFile(t, p, 21L, 22L, 2, 2, false);
+    addDeltaFile(t, p, 23L, 26L, 4);
 
     burnThroughTransactions(25);
 
@@ -634,7 +617,7 @@ public class TestWorker extends Compacto
     rqst.setPartitionname("ds=today");
     txnHandler.compact(rqst);
 
-    startWorker(new HiveConf());
+    startWorker();
 
     ShowCompactResponse rsp = txnHandler.showCompact(new ShowCompactRequest());
     List<ShowCompactResponseElement> compacts = rsp.getCompacts();
@@ -649,7 +632,7 @@ public class TestWorker extends Compacto
     // Find the new delta file and make sure it has the right contents
     boolean sawNewBase = false;
     for (int i = 0; i < stat.length; i++) {
-      if (stat[i].getPath().getName().equals("base_0000024")) {
+      if (stat[i].getPath().getName().equals("base_0000026")) {
         sawNewBase = true;
         FileStatus[] buckets = fs.listStatus(stat[i].getPath());
         Assert.assertEquals(2, buckets.length);
@@ -658,10 +641,12 @@ public class TestWorker extends Compacto
         // Bucket 0 should be small and bucket 1 should be large, make sure that's the case
         Assert.assertTrue(
             ("bucket_00000".equals(buckets[0].getPath().getName()) && 104L == buckets[0].getLen()
-            && "bucket_00001".equals(buckets[1].getPath().getName()) && 1248L == buckets[1] .getLen())
+            && "bucket_00001".equals(buckets[1].getPath().getName()) && 676L == buckets[1]
+                .getLen())
             ||
             ("bucket_00000".equals(buckets[1].getPath().getName()) && 104L == buckets[1].getLen()
-            && "bucket_00001".equals(buckets[0].getPath().getName()) && 1248L == buckets[0] .getLen())
+            && "bucket_00001".equals(buckets[0].getPath().getName()) && 676L == buckets[0]
+                .getLen())
         );
       } else {
         LOG.debug("This is not the file you are looking for " + stat[i].getPath().getName());
@@ -669,9 +654,4 @@ public class TestWorker extends Compacto
     }
     Assert.assertTrue(sawNewBase);
   }
-
-  @Before
-  public void setUpTxnDb() throws Exception {
-    TxnDbUtil.setConfValues(new HiveConf());
-  }
 }

Modified: hive/branches/llap/ql/src/test/org/apache/hadoop/hive/ql/udf/TestUDFMath.java
URL: http://svn.apache.org/viewvc/hive/branches/llap/ql/src/test/org/apache/hadoop/hive/ql/udf/TestUDFMath.java?rev=1631841&r1=1631840&r2=1631841&view=diff
==============================================================================
--- hive/branches/llap/ql/src/test/org/apache/hadoop/hive/ql/udf/TestUDFMath.java (original)
+++ hive/branches/llap/ql/src/test/org/apache/hadoop/hive/ql/udf/TestUDFMath.java Tue Oct 14 19:06:45 2014
@@ -98,14 +98,6 @@ public class TestUDFMath {
     input = createDecimal("7.38905609893065");
     DoubleWritable res = udf.evaluate(input);
     Assert.assertEquals(2.0, res.get(), 0.000001);
-    
-    DoubleWritable input = new DoubleWritable(9.0);
-    res = udf.evaluate(createDecimal("3.0"), input);
-    Assert.assertEquals(2.0, res.get(), 0.000001);
-
-    DoubleWritable base = new DoubleWritable(3.0);
-    res = udf.evaluate(base, createDecimal("9.0"));
-    Assert.assertEquals(2.0, res.get(), 0.000001);
 
     res = udf.evaluate(createDecimal("3.0"), createDecimal("9.0"));
     Assert.assertEquals(2.0, res.get(), 0.000001);

Modified: hive/branches/llap/ql/src/test/queries/clientnegative/acid_overwrite.q
URL: http://svn.apache.org/viewvc/hive/branches/llap/ql/src/test/queries/clientnegative/acid_overwrite.q?rev=1631841&r1=1631840&r2=1631841&view=diff
==============================================================================
--- hive/branches/llap/ql/src/test/queries/clientnegative/acid_overwrite.q (original)
+++ hive/branches/llap/ql/src/test/queries/clientnegative/acid_overwrite.q Tue Oct 14 19:06:45 2014
@@ -1,9 +1,8 @@
 set hive.support.concurrency=true;
 set hive.txn.manager=org.apache.hadoop.hive.ql.lockmgr.DbTxnManager;
-set hive.input.format=org.apache.hadoop.hive.ql.io.HiveInputFormat;
 set hive.enforce.bucketing=true;
 
-create table acid_uanp(a int, b varchar(128)) clustered by (a) into 2 buckets stored as orc;
+create table acid_uanp(a int, b varchar(128)) clustered by (a) into 2 buckets stored as orc TBLPROPERTIES ('transactional'='true');
 
 insert into table acid_uanp select cint, cast(cstring1 as varchar(128)) from alltypesorc where cint < 0 order by cint limit 10;
 insert overwrite table acid_uanp select cint, cast(cstring1 as varchar(128)) from alltypesorc;

Modified: hive/branches/llap/ql/src/test/queries/clientnegative/authorization_uri_create_table1.q
URL: http://svn.apache.org/viewvc/hive/branches/llap/ql/src/test/queries/clientnegative/authorization_uri_create_table1.q?rev=1631841&r1=1631840&r2=1631841&view=diff
==============================================================================
--- hive/branches/llap/ql/src/test/queries/clientnegative/authorization_uri_create_table1.q (original)
+++ hive/branches/llap/ql/src/test/queries/clientnegative/authorization_uri_create_table1.q Tue Oct 14 19:06:45 2014
@@ -7,6 +7,6 @@ dfs ${system:test.dfs.mkdir} ${system:te
 dfs -touchz ${system:test.tmp.dir}/a_uri_crtab1/1.txt;
 dfs -chmod 555 ${system:test.tmp.dir}/a_uri_crtab1/1.txt;
 
-create table t1(i int) location '${system:test.tmp.dir}/a_uri_crtab_ext';
+create table t1(i int) location '${system:test.tmp.dir}/a_uri_crtab1';
 
 -- Attempt to create table with dir that does not have write permission should fail

Modified: hive/branches/llap/ql/src/test/queries/clientnegative/update_partition_col.q
URL: http://svn.apache.org/viewvc/hive/branches/llap/ql/src/test/queries/clientnegative/update_partition_col.q?rev=1631841&r1=1631840&r2=1631841&view=diff
==============================================================================
--- hive/branches/llap/ql/src/test/queries/clientnegative/update_partition_col.q (original)
+++ hive/branches/llap/ql/src/test/queries/clientnegative/update_partition_col.q Tue Oct 14 19:06:45 2014
@@ -1,8 +1,7 @@
 set hive.support.concurrency=true;
 set hive.txn.manager=org.apache.hadoop.hive.ql.lockmgr.DbTxnManager;
-set hive.input.format=org.apache.hadoop.hive.ql.io.HiveInputFormat;
 set hive.enforce.bucketing=true;
 
-create table foo(a int, b varchar(128)) partitioned by (ds string) clustered by (a) into 2 buckets stored as orc;
+create table foo(a int, b varchar(128)) partitioned by (ds string) clustered by (a) into 2 buckets stored as orc TBLPROPERTIES ('transactional'='true');
 
 update foo set ds = 'fred';



Mime
View raw message