Return-Path: X-Original-To: apmail-hive-commits-archive@www.apache.org Delivered-To: apmail-hive-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 9ABD21012E for ; Tue, 5 Nov 2013 07:03:38 +0000 (UTC) Received: (qmail 94709 invoked by uid 500); 5 Nov 2013 07:03:33 -0000 Delivered-To: apmail-hive-commits-archive@hive.apache.org Received: (qmail 94510 invoked by uid 500); 5 Nov 2013 07:03:31 -0000 Mailing-List: contact commits-help@hive.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: hive-dev@hive.apache.org Delivered-To: mailing list commits@hive.apache.org Received: (qmail 94471 invoked by uid 99); 5 Nov 2013 07:03:24 -0000 Received: from athena.apache.org (HELO athena.apache.org) (140.211.11.136) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 05 Nov 2013 07:03:24 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=5.0 tests=ALL_TRUSTED,T_FILL_THIS_FORM_SHORT X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 05 Nov 2013 07:03:19 +0000 Received: from eris.apache.org (localhost [127.0.0.1]) by eris.apache.org (Postfix) with ESMTP id 806EF2388A68; Tue, 5 Nov 2013 07:02:59 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r1538880 [3/46] - in /hive/branches/tez: ./ ant/ ant/src/org/apache/hadoop/hive/ant/ beeline/ beeline/src/java/org/apache/hive/beeline/ beeline/src/main/ beeline/src/test/org/apache/hive/beeline/src/test/ cli/ common/ common/src/java/conf/ ... Date: Tue, 05 Nov 2013 07:01:58 -0000 To: commits@hive.apache.org From: gunther@apache.org X-Mailer: svnmailer-1.0.9 Message-Id: <20131105070259.806EF2388A68@eris.apache.org> X-Virus-Checked: Checked by ClamAV on apache.org Modified: hive/branches/tez/hcatalog/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/SkeletonHBaseTest.java URL: http://svn.apache.org/viewvc/hive/branches/tez/hcatalog/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/SkeletonHBaseTest.java?rev=1538880&r1=1538879&r2=1538880&view=diff ============================================================================== --- hive/branches/tez/hcatalog/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/SkeletonHBaseTest.java (original) +++ hive/branches/tez/hcatalog/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/SkeletonHBaseTest.java Tue Nov 5 07:01:32 2013 @@ -36,14 +36,13 @@ import org.apache.hadoop.hbase.HTableDes import org.apache.hadoop.hbase.client.HBaseAdmin; import org.apache.hadoop.hive.conf.HiveConf; import org.junit.AfterClass; -import org.junit.BeforeClass; /** * Base class for HBase Tests which need a mini cluster instance */ public abstract class SkeletonHBaseTest { - protected static String TEST_DIR = "/tmp/build/test/data/"; + protected static String TEST_DIR = System.getProperty("test.tmp.dir", "target/tmp/"); protected final static String DEFAULT_CONTEXT_HANDLE = "default"; @@ -56,20 +55,15 @@ public abstract class SkeletonHBaseTest */ protected static Configuration testConf = null; - protected void createTable(String tableName, String[] families) { - try { - HBaseAdmin admin = new HBaseAdmin(getHbaseConf()); - HTableDescriptor tableDesc = new HTableDescriptor(tableName); - for (String family : families) { - HColumnDescriptor columnDescriptor = new HColumnDescriptor(family); - tableDesc.addFamily(columnDescriptor); - } - admin.createTable(tableDesc); - } catch (Exception e) { - e.printStackTrace(); - throw new IllegalStateException(e); + protected void createTable(String tableName, String[] families) throws IOException { + HBaseAdmin admin = new HBaseAdmin(getHbaseConf()); + HTableDescriptor tableDesc = new HTableDescriptor(tableName); + for (String family : families) { + HColumnDescriptor columnDescriptor = new HColumnDescriptor(family); + tableDesc.addFamily(columnDescriptor); } - + admin.createTable(tableDesc); + admin.close(); } protected String newTableName(String prefix) { @@ -78,21 +72,20 @@ public abstract class SkeletonHBaseTest do { name = prefix + "_" + Math.abs(new Random().nextLong()); } while (tableNames.contains(name) && --tries > 0); - if (tableNames.contains(name)) + if (tableNames.contains(name)) { throw new IllegalStateException("Couldn't find a unique table name, tableNames size: " + tableNames.size()); + } tableNames.add(name); return name; } - /** * startup an hbase cluster instance before a test suite runs */ - @BeforeClass - public static void setup() { - if (!contextMap.containsKey(getContextHandle())) + public static void setupSkeletonHBaseTest() { + if (!contextMap.containsKey(getContextHandle())) { contextMap.put(getContextHandle(), new Context(getContextHandle())); - + } contextMap.get(getContextHandle()).start(); } Modified: hive/branches/tez/hcatalog/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/TestHBaseBulkOutputFormat.java URL: http://svn.apache.org/viewvc/hive/branches/tez/hcatalog/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/TestHBaseBulkOutputFormat.java?rev=1538880&r1=1538879&r2=1538880&view=diff ============================================================================== --- hive/branches/tez/hcatalog/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/TestHBaseBulkOutputFormat.java (original) +++ hive/branches/tez/hcatalog/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/TestHBaseBulkOutputFormat.java Tue Nov 5 07:01:32 2013 @@ -66,6 +66,7 @@ import org.apache.hcatalog.mapreduce.HCa import org.apache.hcatalog.mapreduce.HCatOutputFormat; import org.apache.hcatalog.mapreduce.OutputJobInfo; +import org.junit.BeforeClass; import org.junit.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -89,6 +90,11 @@ public class TestHBaseBulkOutputFormat e private final HiveConf allConf; private final HCatDriver hcatDriver; + @BeforeClass + public static void setup() throws Throwable { + setupSkeletonHBaseTest(); + } + public TestHBaseBulkOutputFormat() { allConf = getHiveConf(); allConf.set(HiveConf.ConfVars.SEMANTIC_ANALYZER_HOOK.varname, Modified: hive/branches/tez/hcatalog/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/TestHBaseDirectOutputFormat.java URL: http://svn.apache.org/viewvc/hive/branches/tez/hcatalog/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/TestHBaseDirectOutputFormat.java?rev=1538880&r1=1538879&r2=1538880&view=diff ============================================================================== --- hive/branches/tez/hcatalog/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/TestHBaseDirectOutputFormat.java (original) +++ hive/branches/tez/hcatalog/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/TestHBaseDirectOutputFormat.java Tue Nov 5 07:01:32 2013 @@ -63,6 +63,7 @@ import org.apache.hcatalog.hbase.snapsho import org.apache.hcatalog.mapreduce.HCatInputFormat; import org.apache.hcatalog.mapreduce.HCatOutputFormat; import org.apache.hcatalog.mapreduce.OutputJobInfo; +import org.junit.BeforeClass; import org.junit.Test; import java.io.IOException; @@ -83,6 +84,11 @@ public class TestHBaseDirectOutputFormat private final HiveConf allConf; private final HCatDriver hcatDriver; + @BeforeClass + public static void setup() throws Throwable { + setupSkeletonHBaseTest(); + } + public TestHBaseDirectOutputFormat() { allConf = getHiveConf(); allConf.set(HiveConf.ConfVars.SEMANTIC_ANALYZER_HOOK.varname, Modified: hive/branches/tez/hcatalog/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/TestHBaseHCatStorageHandler.java URL: http://svn.apache.org/viewvc/hive/branches/tez/hcatalog/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/TestHBaseHCatStorageHandler.java?rev=1538880&r1=1538879&r2=1538880&view=diff ============================================================================== --- hive/branches/tez/hcatalog/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/TestHBaseHCatStorageHandler.java (original) +++ hive/branches/tez/hcatalog/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/TestHBaseHCatStorageHandler.java Tue Nov 5 07:01:32 2013 @@ -41,6 +41,7 @@ import org.apache.hcatalog.cli.SemanticA import org.apache.hcatalog.hbase.snapshot.RevisionManager; import org.apache.hcatalog.hbase.snapshot.RevisionManagerConfiguration; import org.apache.zookeeper.KeeperException.NoNodeException; +import org.junit.BeforeClass; import org.junit.Test; public class TestHBaseHCatStorageHandler extends SkeletonHBaseTest { @@ -49,6 +50,11 @@ public class TestHBaseHCatStorageHandler private static HCatDriver hcatDriver; private static Warehouse wh; + @BeforeClass + public static void setup() throws Throwable { + setupSkeletonHBaseTest(); + } + public void Initialize() throws Exception { hcatConf = getHiveConf(); Modified: hive/branches/tez/hcatalog/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/TestHCatHBaseInputFormat.java URL: http://svn.apache.org/viewvc/hive/branches/tez/hcatalog/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/TestHCatHBaseInputFormat.java?rev=1538880&r1=1538879&r2=1538880&view=diff ============================================================================== --- hive/branches/tez/hcatalog/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/TestHCatHBaseInputFormat.java (original) +++ hive/branches/tez/hcatalog/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/TestHCatHBaseInputFormat.java Tue Nov 5 07:01:32 2013 @@ -73,6 +73,7 @@ import org.apache.hcatalog.hbase.snapsho import org.apache.hcatalog.mapreduce.HCatInputFormat; import org.apache.hcatalog.mapreduce.InputJobInfo; import org.apache.hcatalog.mapreduce.PartInfo; +import org.junit.BeforeClass; import org.junit.Test; public class TestHCatHBaseInputFormat extends SkeletonHBaseTest { @@ -83,6 +84,11 @@ public class TestHCatHBaseInputFormat ex private final byte[] QUALIFIER1 = Bytes.toBytes("testQualifier1"); private final byte[] QUALIFIER2 = Bytes.toBytes("testQualifier2"); + @BeforeClass + public static void setup() throws Throwable { + setupSkeletonHBaseTest(); + } + public TestHCatHBaseInputFormat() throws Exception { hcatConf = getHiveConf(); hcatConf.set(ConfVars.SEMANTIC_ANALYZER_HOOK.varname, Modified: hive/branches/tez/hcatalog/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/TestSnapshots.java URL: http://svn.apache.org/viewvc/hive/branches/tez/hcatalog/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/TestSnapshots.java?rev=1538880&r1=1538879&r2=1538880&view=diff ============================================================================== --- hive/branches/tez/hcatalog/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/TestSnapshots.java (original) +++ hive/branches/tez/hcatalog/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/TestSnapshots.java Tue Nov 5 07:01:32 2013 @@ -41,12 +41,18 @@ import org.apache.hcatalog.common.HCatUt import org.apache.hcatalog.hbase.snapshot.TableSnapshot; import org.apache.hcatalog.mapreduce.HCatInputFormat; import org.apache.hcatalog.mapreduce.InputJobInfo; +import org.junit.BeforeClass; import org.junit.Test; public class TestSnapshots extends SkeletonHBaseTest { private static HiveConf hcatConf; private static HCatDriver hcatDriver; + @BeforeClass + public static void setup() throws Throwable { + setupSkeletonHBaseTest(); + } + public void Initialize() throws Exception { hcatConf = getHiveConf(); hcatConf.set(ConfVars.SEMANTIC_ANALYZER_HOOK.varname, Modified: hive/branches/tez/hcatalog/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/snapshot/TestIDGenerator.java URL: http://svn.apache.org/viewvc/hive/branches/tez/hcatalog/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/snapshot/TestIDGenerator.java?rev=1538880&r1=1538879&r2=1538880&view=diff ============================================================================== --- hive/branches/tez/hcatalog/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/snapshot/TestIDGenerator.java (original) +++ hive/branches/tez/hcatalog/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/snapshot/TestIDGenerator.java Tue Nov 5 07:01:32 2013 @@ -26,10 +26,16 @@ import java.util.HashMap; import org.apache.hcatalog.hbase.SkeletonHBaseTest; import org.junit.Assert; +import org.junit.BeforeClass; import org.junit.Test; public class TestIDGenerator extends SkeletonHBaseTest { + @BeforeClass + public static void setup() throws Throwable { + setupSkeletonHBaseTest(); + } + @Test public void testIDGeneration() throws Exception { Modified: hive/branches/tez/hcatalog/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/snapshot/TestRevisionManager.java URL: http://svn.apache.org/viewvc/hive/branches/tez/hcatalog/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/snapshot/TestRevisionManager.java?rev=1538880&r1=1538879&r2=1538880&view=diff ============================================================================== --- hive/branches/tez/hcatalog/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/snapshot/TestRevisionManager.java (original) +++ hive/branches/tez/hcatalog/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/snapshot/TestRevisionManager.java Tue Nov 5 07:01:32 2013 @@ -32,10 +32,16 @@ import org.apache.hcatalog.hbase.snapsho import org.apache.zookeeper.KeeperException; import org.apache.zookeeper.ZooKeeper; import org.apache.zookeeper.data.Stat; +import org.junit.BeforeClass; import org.junit.Test; public class TestRevisionManager extends SkeletonHBaseTest { + @BeforeClass + public static void setup() throws Throwable { + setupSkeletonHBaseTest(); + } + @Test public void testBasicZNodeCreation() throws IOException, KeeperException, InterruptedException { Modified: hive/branches/tez/hcatalog/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/snapshot/TestRevisionManagerEndpoint.java URL: http://svn.apache.org/viewvc/hive/branches/tez/hcatalog/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/snapshot/TestRevisionManagerEndpoint.java?rev=1538880&r1=1538879&r2=1538880&view=diff ============================================================================== --- hive/branches/tez/hcatalog/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/snapshot/TestRevisionManagerEndpoint.java (original) +++ hive/branches/tez/hcatalog/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/snapshot/TestRevisionManagerEndpoint.java Tue Nov 5 07:01:32 2013 @@ -31,17 +31,20 @@ import org.apache.hadoop.conf.Configurat import org.apache.hadoop.hbase.coprocessor.CoprocessorHost; import org.apache.hcatalog.hbase.SkeletonHBaseTest; import org.junit.Assert; +import org.junit.BeforeClass; import org.junit.Test; public class TestRevisionManagerEndpoint extends SkeletonHBaseTest { - static { + @BeforeClass + public static void setup() throws Throwable { // test case specific mini cluster settings testConf = new Configuration(false); testConf.setStrings(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY, "org.apache.hcatalog.hbase.snapshot.RevisionManagerEndpoint", "org.apache.hadoop.hbase.coprocessor.GenericEndpoint"); testConf.set(RMConstants.REVISION_MGR_ENDPOINT_IMPL_CLASS, MockRM.class.getName()); + setupSkeletonHBaseTest(); } /** Modified: hive/branches/tez/hcatalog/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/snapshot/TestZNodeSetUp.java URL: http://svn.apache.org/viewvc/hive/branches/tez/hcatalog/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/snapshot/TestZNodeSetUp.java?rev=1538880&r1=1538879&r2=1538880&view=diff ============================================================================== --- hive/branches/tez/hcatalog/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/snapshot/TestZNodeSetUp.java (original) +++ hive/branches/tez/hcatalog/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/snapshot/TestZNodeSetUp.java Tue Nov 5 07:01:32 2013 @@ -38,6 +38,7 @@ import org.apache.hcatalog.cli.SemanticA import org.apache.hcatalog.hbase.SkeletonHBaseTest; import org.apache.zookeeper.ZooKeeper; import org.apache.zookeeper.data.Stat; +import org.junit.BeforeClass; import org.junit.Test; @@ -46,6 +47,11 @@ public class TestZNodeSetUp extends Skel private static HiveConf hcatConf; private static HCatDriver hcatDriver; + @BeforeClass + public static void setup() throws Throwable { + setupSkeletonHBaseTest(); + } + public void Initialize() throws Exception { hcatConf = getHiveConf(); Modified: hive/branches/tez/hcatalog/webhcat/java-client/src/test/java/org/apache/hcatalog/api/TestHCatClient.java URL: http://svn.apache.org/viewvc/hive/branches/tez/hcatalog/webhcat/java-client/src/test/java/org/apache/hcatalog/api/TestHCatClient.java?rev=1538880&r1=1538879&r2=1538880&view=diff ============================================================================== --- hive/branches/tez/hcatalog/webhcat/java-client/src/test/java/org/apache/hcatalog/api/TestHCatClient.java (original) +++ hive/branches/tez/hcatalog/webhcat/java-client/src/test/java/org/apache/hcatalog/api/TestHCatClient.java Tue Nov 5 07:01:32 2013 @@ -36,7 +36,6 @@ import org.apache.hadoop.hive.ql.io.RCFi import org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe; import org.apache.hadoop.mapred.TextInputFormat; import org.apache.hcatalog.cli.SemanticAnalysis.HCatSemanticAnalyzer; -import org.apache.hadoop.hive.conf.HiveConf.ConfVars; import org.apache.hcatalog.common.HCatConstants; import org.apache.hcatalog.common.HCatException; import org.apache.hcatalog.data.schema.HCatFieldSchema; @@ -52,6 +51,7 @@ import static org.junit.Assert.assertEqu import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.fail; /** * @deprecated Use/modify {@link org.apache.hive.hcatalog.api.TestHCatClient} instead @@ -123,9 +123,9 @@ public class TestHCatClient { assertTrue(testDb.getComment() == null); assertTrue(testDb.getProperties().size() == 0); String warehouseDir = System - .getProperty(ConfVars.METASTOREWAREHOUSE.varname, "/user/hive/warehouse"); - assertTrue(testDb.getLocation().equals( - "file:" + warehouseDir + "/" + db + ".db")); + .getProperty("test.warehouse.dir", "/user/hive/warehouse"); + String expectedDir = warehouseDir.replaceAll("\\\\", "/").replaceFirst("pfile:///", "pfile:/"); + assertEquals(expectedDir + "/" + db + ".db", testDb.getLocation()); ArrayList cols = new ArrayList(); cols.add(new HCatFieldSchema("id", Type.INT, "id comment")); cols.add(new HCatFieldSchema("value", Type.STRING, "value comment")); @@ -145,6 +145,7 @@ public class TestHCatClient { // will result in an exception. try { client.createTable(tableDesc); + fail("Expected exception"); } catch (HCatException e) { assertTrue(e.getMessage().contains( "AlreadyExistsException while creating table.")); @@ -159,8 +160,7 @@ public class TestHCatClient { TextInputFormat.class.getName())); assertTrue(table2.getOutputFileFormat().equalsIgnoreCase( IgnoreKeyTextOutputFormat.class.getName())); - assertTrue(table2.getLocation().equalsIgnoreCase( - "file:" + warehouseDir + "/" + db + ".db/" + tableTwo)); + assertEquals((expectedDir + "/" + db + ".db/" + tableTwo).toLowerCase(), table2.getLocation().toLowerCase()); client.close(); } Modified: hive/branches/tez/hcatalog/webhcat/java-client/src/test/java/org/apache/hive/hcatalog/api/TestHCatClient.java URL: http://svn.apache.org/viewvc/hive/branches/tez/hcatalog/webhcat/java-client/src/test/java/org/apache/hive/hcatalog/api/TestHCatClient.java?rev=1538880&r1=1538879&r2=1538880&view=diff ============================================================================== --- hive/branches/tez/hcatalog/webhcat/java-client/src/test/java/org/apache/hive/hcatalog/api/TestHCatClient.java (original) +++ hive/branches/tez/hcatalog/webhcat/java-client/src/test/java/org/apache/hive/hcatalog/api/TestHCatClient.java Tue Nov 5 07:01:32 2013 @@ -36,7 +36,6 @@ import org.apache.hadoop.hive.ql.io.RCFi import org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe; import org.apache.hadoop.mapred.TextInputFormat; import org.apache.hive.hcatalog.cli.SemanticAnalysis.HCatSemanticAnalyzer; -import org.apache.hadoop.hive.conf.HiveConf.ConfVars; import org.apache.hive.hcatalog.common.HCatConstants; import org.apache.hive.hcatalog.common.HCatException; import org.apache.hive.hcatalog.data.schema.HCatFieldSchema; @@ -48,6 +47,7 @@ import org.junit.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import static org.junit.Assert.fail; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; @@ -120,13 +120,9 @@ public class TestHCatClient { assertTrue(testDb.getComment() == null); assertTrue(testDb.getProperties().size() == 0); String warehouseDir = System - .getProperty(ConfVars.METASTOREWAREHOUSE.varname, "/user/hive/warehouse"); - String expectedDir = warehouseDir.replaceAll("\\\\", "/"); - if (!expectedDir.startsWith("/")) { - expectedDir = "/" + expectedDir; - } - assertTrue(testDb.getLocation().equals( - "file:" + expectedDir + "/" + db + ".db")); + .getProperty("test.warehouse.dir", "/user/hive/warehouse"); + String expectedDir = warehouseDir.replaceAll("\\\\", "/").replaceFirst("pfile:///", "pfile:/"); + assertEquals(expectedDir + "/" + db + ".db", testDb.getLocation()); ArrayList cols = new ArrayList(); cols.add(new HCatFieldSchema("id", Type.INT, "id comment")); cols.add(new HCatFieldSchema("value", Type.STRING, "value comment")); @@ -146,6 +142,7 @@ public class TestHCatClient { // will result in an exception. try { client.createTable(tableDesc); + fail("Expected exception"); } catch (HCatException e) { assertTrue(e.getMessage().contains( "AlreadyExistsException while creating table.")); @@ -160,8 +157,7 @@ public class TestHCatClient { TextInputFormat.class.getName())); assertTrue(table2.getOutputFileFormat().equalsIgnoreCase( IgnoreKeyTextOutputFormat.class.getName())); - assertTrue(table2.getLocation().equalsIgnoreCase( - "file:" + expectedDir + "/" + db + ".db/" + tableTwo)); + assertEquals((expectedDir + "/" + db + ".db/" + tableTwo).toLowerCase(), table2.getLocation().toLowerCase()); client.close(); } Modified: hive/branches/tez/hcatalog/webhcat/svr/src/main/config/webhcat-default.xml URL: http://svn.apache.org/viewvc/hive/branches/tez/hcatalog/webhcat/svr/src/main/config/webhcat-default.xml?rev=1538880&r1=1538879&r2=1538880&view=diff ============================================================================== --- hive/branches/tez/hcatalog/webhcat/svr/src/main/config/webhcat-default.xml (original) +++ hive/branches/tez/hcatalog/webhcat/svr/src/main/config/webhcat-default.xml Tue Nov 5 07:01:32 2013 @@ -116,6 +116,25 @@ + templeton.hive.home + hive-0.13.0-SNAPSHOT-bin.tar.gz/hive-0.13.0-SNAPSHOT-bin + + The path to the Hive home within the tar. This is needed if Hive is not installed on all + nodes in the cluster and needs to be shipped to the target node in the cluster to execute Pig + job which uses HCat, Hive query, etc. Has no effect if templeton.hive.archive is not set. + + + + templeton.hcat.home + hive-0.13.0-SNAPSHOT-bin.tar.gz/hive-0.13.0-SNAPSHOT-bin/hcatalog + + The path to the HCat home within the tar. This is needed if Hive is not installed on all + nodes in the cluster and needs to be shipped to the target node in the cluster to execute Pig + job which uses HCat, Hive query, etc. Has no effect if templeton.hive.archive is not set. + + + + templeton.hive.properties hive.metastore.local=false,hive.metastore.uris=thrift://localhost:9933,hive.metastore.sasl.enabled=false Properties to set when running hive. Modified: hive/branches/tez/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/AppConfig.java URL: http://svn.apache.org/viewvc/hive/branches/tez/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/AppConfig.java?rev=1538880&r1=1538879&r2=1538880&view=diff ============================================================================== --- hive/branches/tez/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/AppConfig.java (original) +++ hive/branches/tez/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/AppConfig.java Tue Nov 5 07:01:32 2013 @@ -91,6 +91,14 @@ public class AppConfig extends Configura public static final String PYTHON_NAME = "templeton.python"; public static final String HIVE_ARCHIVE_NAME = "templeton.hive.archive"; public static final String HIVE_PATH_NAME = "templeton.hive.path"; + /** + * see webhcat-default.xml + */ + public static final String HIVE_HOME_PATH = "templeton.hive.home"; + /** + * see webhcat-default.xml + */ + public static final String HCAT_HOME_PATH = "templeton.hcat.home"; public static final String HIVE_PROPS_NAME = "templeton.hive.properties"; public static final String LIB_JARS_NAME = "templeton.libjars"; public static final String PIG_ARCHIVE_NAME = "templeton.pig.archive"; Modified: hive/branches/tez/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/DeleteDelegator.java URL: http://svn.apache.org/viewvc/hive/branches/tez/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/DeleteDelegator.java?rev=1538880&r1=1538879&r2=1538880&view=diff ============================================================================== --- hive/branches/tez/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/DeleteDelegator.java (original) +++ hive/branches/tez/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/DeleteDelegator.java Tue Nov 5 07:01:32 2013 @@ -19,7 +19,10 @@ package org.apache.hive.hcatalog.templeton; import java.io.IOException; +import java.util.List; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; import org.apache.hadoop.hive.shims.HadoopShims.WebHCatJTShim; import org.apache.hadoop.hive.shims.ShimLoader; import org.apache.hadoop.mapred.JobID; @@ -30,6 +33,7 @@ import org.apache.hive.hcatalog.templeto * Delete a job */ public class DeleteDelegator extends TempletonDelegator { + private static final Log LOG = LogFactory.getLog(DeleteDelegator.class); public DeleteDelegator(AppConfig appConf) { super(appConf); } @@ -47,9 +51,16 @@ public class DeleteDelegator extends Tem throw new BadParam("Invalid jobid: " + id); tracker.killJob(jobid); state = new JobState(id, Main.getAppConfigInstance()); - String childid = state.getChildId(); - if (childid != null) - tracker.killJob(StatusDelegator.StringToJobID(childid)); + List children = state.getChildren(); + if (children != null) { + for (JobState child : children) { + try { + tracker.killJob(StatusDelegator.StringToJobID(child.getId())); + } catch (IOException e) { + LOG.warn("templeton: fail to kill job " + child.getId()); + } + } + } return StatusDelegator.makeStatus(tracker, jobid, state); } catch (IllegalStateException e) { throw new BadParam(e.getMessage()); Modified: hive/branches/tez/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/HcatDelegator.java URL: http://svn.apache.org/viewvc/hive/branches/tez/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/HcatDelegator.java?rev=1538880&r1=1538879&r2=1538880&view=diff ============================================================================== --- hive/branches/tez/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/HcatDelegator.java (original) +++ hive/branches/tez/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/HcatDelegator.java Tue Nov 5 07:01:32 2013 @@ -107,12 +107,12 @@ public class HcatDelegator extends Launc } LOG.info("Main.getAppConfigInstance().get(AppConfig.UNIT_TEST_MODE)=" + Main.getAppConfigInstance().get(AppConfig.UNIT_TEST_MODE)); - if(System.getProperty("hive.metastore.warehouse.dir") != null) { + if(System.getProperty("test.warehouse.dir") != null) { /*when running in unit test mode, pass this property to HCat, which will in turn pass it to Hive to make sure that Hive tries to write to a directory that exists.*/ args.add("-D"); - args.add("hive.metastore.warehouse.dir=" + System.getProperty("hive.metastore.warehouse.dir")); + args.add("hive.metastore.warehouse.dir=" + System.getProperty("test.warehouse.dir")); } return args; } @@ -645,7 +645,7 @@ public class HcatDelegator extends Launc } catch (HcatException e) { if (e.execBean.stderr.contains("SemanticException") && e.execBean.stderr.contains("Partition not found")) { - String emsg = "Partition " + partition + " for table " + String emsg = "Partition " + partition + " for table " + table + " does not exist" + db + "." + table + " does not exist"; return JsonBuilder.create() .put("error", emsg) Modified: hive/branches/tez/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/HiveDelegator.java URL: http://svn.apache.org/viewvc/hive/branches/tez/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/HiveDelegator.java?rev=1538880&r1=1538879&r2=1538880&view=diff ============================================================================== --- hive/branches/tez/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/HiveDelegator.java (original) +++ hive/branches/tez/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/HiveDelegator.java Tue Nov 5 07:01:32 2013 @@ -128,7 +128,7 @@ public class HiveDelegator extends Launc if (appConf.hiveArchive() != null && !appConf.hiveArchive().equals("")) { - args.add("-archives"); + args.add(ARCHIVES); args.add(appConf.hiveArchive()); } Modified: hive/branches/tez/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/JarDelegator.java URL: http://svn.apache.org/viewvc/hive/branches/tez/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/JarDelegator.java?rev=1538880&r1=1538879&r2=1538880&view=diff ============================================================================== --- hive/branches/tez/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/JarDelegator.java (original) +++ hive/branches/tez/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/JarDelegator.java Tue Nov 5 07:01:32 2013 @@ -43,14 +43,14 @@ public class JarDelegator extends Launch String libjars, String files, List jarArgs, List defines, String statusdir, String callback, - boolean usehcatalog, String completedUrl, + boolean usesHcatalog, String completedUrl, boolean enablelog, JobType jobType) throws NotAuthorizedException, BadParam, BusyException, QueueException, ExecuteException, IOException, InterruptedException { runAs = user; List args = makeArgs(jar, mainClass, libjars, files, jarArgs, defines, - statusdir, usehcatalog, completedUrl, enablelog, jobType); + statusdir, usesHcatalog, completedUrl, enablelog, jobType); return enqueueController(user, userArgs, callback, args); } @@ -58,7 +58,7 @@ public class JarDelegator extends Launch private List makeArgs(String jar, String mainClass, String libjars, String files, List jarArgs, List defines, - String statusdir, boolean usehcatalog, String completedUrl, + String statusdir, boolean usesHcatalog, String completedUrl, boolean enablelog, JobType jobType) throws BadParam, IOException, InterruptedException { ArrayList args = new ArrayList(); @@ -72,7 +72,7 @@ public class JarDelegator extends Launch TempletonUtils.addCmdForWindows(args); //check if the rest command specified explicitly to use hcatalog - if(usehcatalog){ + if(usesHcatalog){ addHiveMetaStoreTokenArg(); } Modified: hive/branches/tez/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/LauncherDelegator.java URL: http://svn.apache.org/viewvc/hive/branches/tez/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/LauncherDelegator.java?rev=1538880&r1=1538879&r2=1538880&view=diff ============================================================================== --- hive/branches/tez/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/LauncherDelegator.java (original) +++ hive/branches/tez/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/LauncherDelegator.java Tue Nov 5 07:01:32 2013 @@ -97,6 +97,9 @@ public class LauncherDelegator extends T private String queueAsUser(UserGroupInformation ugi, final List args) throws IOException, InterruptedException { + if(LOG.isDebugEnabled()) { + LOG.debug("Launching job: " + args); + } return ugi.doAs(new PrivilegedExceptionAction() { public String run() throws Exception { String[] array = new String[args.size()]; Modified: hive/branches/tez/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/ListDelegator.java URL: http://svn.apache.org/viewvc/hive/branches/tez/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/ListDelegator.java?rev=1538880&r1=1538879&r2=1538880&view=diff ============================================================================== --- hive/branches/tez/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/ListDelegator.java (original) +++ hive/branches/tez/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/ListDelegator.java Tue Nov 5 07:01:32 2013 @@ -26,7 +26,6 @@ import org.apache.hadoop.hive.shims.Hado import org.apache.hadoop.hive.shims.ShimLoader; import org.apache.hadoop.mapred.JobStatus; import org.apache.hadoop.security.UserGroupInformation; -import org.apache.hive.hcatalog.templeton.tool.JobState; /** * List jobs owned by a user. @@ -50,17 +49,9 @@ public class ListDelegator extends Templ if (jobs != null) { for (JobStatus job : jobs) { - JobState state = null; - try { - String id = job.getJobID().toString(); - state = new JobState(id, Main.getAppConfigInstance()); - if (showall || user.equals(state.getUser())) - ids.add(id); - } finally { - if (state != null) { - state.close(); - } - } + String id = job.getJobID().toString(); + if (showall || user.equals(job.getUsername())) + ids.add(id); } } Modified: hive/branches/tez/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/Main.java URL: http://svn.apache.org/viewvc/hive/branches/tez/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/Main.java?rev=1538880&r1=1538879&r2=1538880&view=diff ============================================================================== --- hive/branches/tez/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/Main.java (original) +++ hive/branches/tez/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/Main.java Tue Nov 5 07:01:32 2013 @@ -206,10 +206,13 @@ public class Main { public FilterHolder makeAuthFilter() { FilterHolder authFilter = new FilterHolder(AuthFilter.class); if (UserGroupInformation.isSecurityEnabled()) { + //http://hadoop.apache.org/docs/r1.1.1/api/org/apache/hadoop/security/authentication/server/AuthenticationFilter.html authFilter.setInitParameter("dfs.web.authentication.signature.secret", conf.kerberosSecret()); + //https://svn.apache.org/repos/asf/hadoop/common/branches/branch-1.2/src/packages/templates/conf/hdfs-site.xml authFilter.setInitParameter("dfs.web.authentication.kerberos.principal", conf.kerberosPrincipal()); + //http://https://svn.apache.org/repos/asf/hadoop/common/branches/branch-1.2/src/packages/templates/conf/hdfs-site.xml authFilter.setInitParameter("dfs.web.authentication.kerberos.keytab", conf.kerberosKeytab()); } Modified: hive/branches/tez/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/PigDelegator.java URL: http://svn.apache.org/viewvc/hive/branches/tez/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/PigDelegator.java?rev=1538880&r1=1538879&r2=1538880&view=diff ============================================================================== --- hive/branches/tez/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/PigDelegator.java (original) +++ hive/branches/tez/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/PigDelegator.java Tue Nov 5 07:01:32 2013 @@ -29,6 +29,7 @@ import java.util.Map; import org.apache.commons.exec.ExecuteException; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.hive.hcatalog.templeton.tool.JobSubmissionConstants; import org.apache.hive.hcatalog.templeton.tool.TempletonControllerJob; import org.apache.hive.hcatalog.templeton.tool.TempletonUtils; @@ -47,13 +48,13 @@ public class PigDelegator extends Launch String execute, String srcFile, List pigArgs, String otherFiles, String statusdir, String callback, - boolean usehcatalog, String completedUrl, boolean enablelog) + boolean usesHcatalog, String completedUrl, boolean enablelog) throws NotAuthorizedException, BadParam, BusyException, QueueException, ExecuteException, IOException, InterruptedException { runAs = user; List args = makeArgs(execute, srcFile, pigArgs, - otherFiles, statusdir, usehcatalog, completedUrl, enablelog); + otherFiles, statusdir, usesHcatalog, completedUrl, enablelog); return enqueueController(user, userArgs, callback, args); } @@ -64,7 +65,7 @@ public class PigDelegator extends Launch * @param pigArgs pig command line arguments * @param otherFiles files to be copied to the map reduce cluster * @param statusdir status dir location - * @param usehcatalog whether the command uses hcatalog/needs to connect + * @param usesHcatalog whether the command uses hcatalog/needs to connect * to hive metastore server * @param completedUrl call back url * @return list of arguments @@ -74,10 +75,14 @@ public class PigDelegator extends Launch */ private List makeArgs(String execute, String srcFile, List pigArgs, String otherFiles, - String statusdir, boolean usehcatalog, + String statusdir, boolean usesHcatalog, String completedUrl, boolean enablelog) throws BadParam, IOException, InterruptedException { ArrayList args = new ArrayList(); + //check if the REST command specified explicitly to use hcatalog + // or if it says that implicitly using the pig -useHCatalog arg + boolean needsMetastoreAccess = usesHcatalog || hasPigArgUseHcat(pigArgs); + try { ArrayList allFiles = new ArrayList(); if (TempletonUtils.isset(srcFile)) { @@ -89,12 +94,32 @@ public class PigDelegator extends Launch } args.addAll(makeLauncherArgs(appConf, statusdir, completedUrl, allFiles, enablelog, JobType.PIG)); - if (appConf.pigArchive() != null && !appConf.pigArchive().equals("")) - { - args.add("-archives"); - args.add(appConf.pigArchive()); + boolean shipPigTar = appConf.pigArchive() != null && !appConf.pigArchive().equals(""); + boolean shipHiveTar = needsMetastoreAccess && appConf.hiveArchive() != null + && !appConf.hiveArchive().equals(""); + if(shipPigTar || shipHiveTar) { + args.add(ARCHIVES); + StringBuilder archives = new StringBuilder(); + if(shipPigTar) { + archives.append(appConf.pigArchive()); + } + if(shipPigTar && shipHiveTar) { + archives.append(","); + } + if(shipHiveTar) { + archives.append(appConf.hiveArchive()); + } + args.add(archives.toString()); + } + if(shipHiveTar) { + addDef(args, JobSubmissionConstants.PigConstants.HIVE_HOME, + appConf.get(AppConfig.HIVE_HOME_PATH)); + addDef(args, JobSubmissionConstants.PigConstants.HCAT_HOME, + appConf.get(AppConfig.HCAT_HOME_PATH)); + //Pig which uses HCat will pass this to HCat so that it can find the metastore + addDef(args, JobSubmissionConstants.PigConstants.PIG_OPTS, + appConf.get(AppConfig.HIVE_PROPS_NAME)); } - args.add("--"); TempletonUtils.addCmdForWindows(args); args.add(appConf.pigPath()); @@ -104,9 +129,7 @@ public class PigDelegator extends Launch for (String pigArg : pigArgs) { args.add(TempletonUtils.quoteForWindows(pigArg)); } - //check if the REST command specified explicitly to use hcatalog - // or if it says that implicitly using the pig -useHCatalog arg - if(usehcatalog || hasPigArgUseHcat(pigArgs)){ + if(needsMetastoreAccess) { addHiveMetaStoreTokenArg(); } Modified: hive/branches/tez/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/QueueStatusBean.java URL: http://svn.apache.org/viewvc/hive/branches/tez/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/QueueStatusBean.java?rev=1538880&r1=1538879&r2=1538880&view=diff ============================================================================== --- hive/branches/tez/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/QueueStatusBean.java (original) +++ hive/branches/tez/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/QueueStatusBean.java Tue Nov 5 07:01:32 2013 @@ -57,12 +57,10 @@ public class QueueStatusBean { this.profile = profile; id = profile.getJobID().toString(); - parentId = state.getId(); - if (id.equals(parentId)) - parentId = null; + parentId = state.getParent(); percentComplete = state.getPercentComplete(); exitValue = state.getExitValue(); - user = state.getUser(); + user = profile.getUser(); callback = state.getCallback(); completed = state.getCompleteStatus(); userargs = state.getUserArgs(); Modified: hive/branches/tez/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/Server.java URL: http://svn.apache.org/viewvc/hive/branches/tez/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/Server.java?rev=1538880&r1=1538879&r2=1538880&view=diff ============================================================================== --- hive/branches/tez/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/Server.java (original) +++ hive/branches/tez/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/Server.java Tue Nov 5 07:01:32 2013 @@ -635,7 +635,7 @@ public class Server { /** * Run a MapReduce Jar job. * Params correspond to the REST api params - * @param usehcatalog if {@code true}, means the Jar uses HCat and thus needs to access + * @param usesHcatalog if {@code true}, means the Jar uses HCat and thus needs to access * metastore, which requires additional steps for WebHCat to perform in a secure cluster. * @param callback URL which WebHCat will call when the hive job finishes * @see org.apache.hive.hcatalog.templeton.tool.TempletonControllerJob @@ -651,7 +651,7 @@ public class Server { @FormParam("define") List defines, @FormParam("statusdir") String statusdir, @FormParam("callback") String callback, - @FormParam("usehcatalog") boolean usehcatalog, + @FormParam("usehcatalog") boolean usesHcatalog, @FormParam("enablelog") boolean enablelog) throws NotAuthorizedException, BusyException, BadParam, QueueException, ExecuteException, IOException, InterruptedException { @@ -677,14 +677,14 @@ public class Server { return d.run(getDoAsUser(), userArgs, jar, mainClass, libjars, files, args, defines, - statusdir, callback, usehcatalog, getCompletedUrl(), enablelog, JobType.JAR); + statusdir, callback, usesHcatalog, getCompletedUrl(), enablelog, JobType.JAR); } /** * Run a Pig job. - * Params correspond to the REST api params. If '-useHCatalog' is in the {@code pigArgs, usehcatalog}, + * Params correspond to the REST api params. If '-useHCatalog' is in the {@code pigArgs, usesHcatalog}, * is interpreted as true. - * @param usehcatalog if {@code true}, means the Pig script uses HCat and thus needs to access + * @param usesHcatalog if {@code true}, means the Pig script uses HCat and thus needs to access * metastore, which requires additional steps for WebHCat to perform in a secure cluster. * This does nothing to ensure that Pig is installed on target node in the cluster. * @param callback URL which WebHCat will call when the hive job finishes @@ -699,7 +699,7 @@ public class Server { @FormParam("files") String otherFiles, @FormParam("statusdir") String statusdir, @FormParam("callback") String callback, - @FormParam("usehcatalog") boolean usehcatalog, + @FormParam("usehcatalog") boolean usesHcatalog, @FormParam("enablelog") boolean enablelog) throws NotAuthorizedException, BusyException, BadParam, QueueException, ExecuteException, IOException, InterruptedException { @@ -725,7 +725,7 @@ public class Server { return d.run(getDoAsUser(), userArgs, execute, srcFile, pigArgs, otherFiles, - statusdir, callback, usehcatalog, getCompletedUrl(), enablelog); + statusdir, callback, usesHcatalog, getCompletedUrl(), enablelog); } /** @@ -855,15 +855,59 @@ public class Server { } /** - * Return all the known job ids for this user. + * Return all the known job ids for this user based on the optional filter conditions. + *

+ * Example usages: + * 1. curl -s 'http://localhost:50111/templeton/v1/jobs?user.name=hsubramaniyan' + * Return all the Job IDs submitted by hsubramaniyan + * 2. curl -s + * 'http://localhost:50111/templeton/v1/jobs?user.name=hsubramaniyan&showall=true' + * Return all the Job IDs that are visible to hsubramaniyan + * 3. curl -s + * 'http://localhost:50111/templeton/v1/jobs?user.name=hsubramaniyan&jobid=job_201312091733_0003' + * Return all the Job IDs for hsubramaniyan after job_201312091733_0003. + * 4. curl -s 'http://localhost:50111/templeton/v1/jobs? + * user.name=hsubramaniyan&jobid=job_201312091733_0003&numrecords=5' + * Return the first 5(atmost) Job IDs submitted by hsubramaniyan after job_201312091733_0003. + * 5. curl -s + * 'http://localhost:50111/templeton/v1/jobs?user.name=hsubramaniyan&numrecords=5' + * Return the first 5(atmost) Job IDs submitted by hsubramaniyan after sorting the Job ID list + * lexicographically. + *

+ *

+ * Supporting pagination using "jobid" and "numrecords" parameters: + * Step 1: Get the start "jobid" = job_xxx_000, "numrecords" = n + * Step 2: Issue a curl command by specifying the user-defined "numrecords" and "jobid" + * Step 3: If list obtained from Step 2 has size equal to "numrecords", retrieve the list's + * last record and get the Job Id of the last record as job_yyy_k, else quit. + * Step 4: set "jobid"=job_yyy_k and go to step 2. + *

+ * @param fields If "fields" set to "*", the request will return full details of the job. + * If "fields" is missing, will only return the job ID. Currently the value can only + * be "*", other values are not allowed and will throw exception. + * @param showall If "showall" is set to "true", the request will return all jobs the user + * has permission to view, not only the jobs belonging to the user. + * @param jobid If "jobid" is present, the records whose Job Id is lexicographically greater + * than "jobid" are only returned. For example, if "jobid" = "job_201312091733_0001", + * the jobs whose Job ID is greater than "job_201312091733_0001" are returned. The number of + * records returned depends on the value of "numrecords". + * @param numrecords If the "jobid" and "numrecords" parameters are present, the top #numrecords + * records appearing after "jobid" will be returned after sorting the Job Id list + * lexicographically. + * If "jobid" parameter is missing and "numrecords" is present, the top #numrecords will + * be returned after lexicographically sorting the Job Id list. If "jobid" parameter is present + * and "numrecords" is missing, all the records whose Job Id is greater than "jobid" are returned. + * @return list of job items based on the filter conditions specified by the user. */ @GET @Path("jobs") @Produces({MediaType.APPLICATION_JSON}) public List showJobList(@QueryParam("fields") String fields, - @QueryParam("showall") boolean showall) + @QueryParam("showall") boolean showall, + @QueryParam("jobid") String jobid, + @QueryParam("numrecords") String numrecords) throws NotAuthorizedException, BadParam, IOException, InterruptedException { - + verifyUser(); boolean showDetails = false; @@ -877,7 +921,46 @@ public class Server { ListDelegator ld = new ListDelegator(appConf); List list = ld.run(getDoAsUser(), showall); List detailList = new ArrayList(); + int currRecord = 0; + int numRecords; + + // Parse numrecords to an integer + try { + if (numrecords != null) { + numRecords = Integer.parseInt(numrecords); + if (numRecords <= 0) { + throw new BadParam("numrecords should be an integer > 0"); + } + } + else { + numRecords = -1; + } + } + catch(Exception e) { + throw new BadParam("Invalid numrecords format: numrecords should be an integer > 0"); + } + + // Sort the list lexicographically + Collections.sort(list); + for (String job : list) { + // If numRecords = -1, fetch all records. + // Hence skip all the below checks when numRecords = -1. + if (numRecords != -1) { + // If currRecord >= numRecords, we have already fetched the top #numRecords + if (currRecord >= numRecords) { + break; + } + // If the current record needs to be returned based on the + // filter conditions specified by the user, increment the counter + else if ((jobid != null && job.compareTo(jobid) > 0) || jobid == null) { + currRecord++; + } + // The current record should not be included in the output detailList. + else { + continue; + } + } JobItemBean jobItem = new JobItemBean(); jobItem.id = job; if (showDetails) { Modified: hive/branches/tez/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/StatusDelegator.java URL: http://svn.apache.org/viewvc/hive/branches/tez/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/StatusDelegator.java?rev=1538880&r1=1538879&r2=1538880&view=diff ============================================================================== --- hive/branches/tez/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/StatusDelegator.java (original) +++ hive/branches/tez/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/StatusDelegator.java Tue Nov 5 07:01:32 2013 @@ -31,7 +31,13 @@ import org.apache.hadoop.mapred.JobStatu import org.apache.hive.hcatalog.templeton.tool.JobState; /** - * Fetch the status of a given job id in the queue. + * Fetch the status of a given job id in the queue. There are three sources of the info + * 1. Query result from JobTracker + * 2. JobState saved by TempletonControllerJob when monitoring the TempletonControllerJob + * 3. TempletonControllerJob put a JobState for every job it launches, so child job can + * retrieve its parent job by its JobState + * + * Currently there is no permission restriction, any user can query any job */ public class StatusDelegator extends TempletonDelegator { private static final Log LOG = LogFactory.getLog(StatusDelegator.class); @@ -63,40 +69,19 @@ public class StatusDelegator extends Tem } } - public static QueueStatusBean makeStatus(WebHCatJTShim tracker, + static QueueStatusBean makeStatus(WebHCatJTShim tracker, JobID jobid, - String childid, JobState state) throws BadParam, IOException { - JobID bestid = jobid; - if (childid != null) - bestid = StatusDelegator.StringToJobID(childid); - - JobStatus status = tracker.getJobStatus(bestid); - JobProfile profile = tracker.getJobProfile(bestid); - - if (status == null || profile == null) { - if (bestid != jobid) { // Corrupt childid, retry. - LOG.error("Corrupt child id " + childid + " for " + jobid); - bestid = jobid; - status = tracker.getJobStatus(bestid); - profile = tracker.getJobProfile(bestid); - } - } + JobStatus status = tracker.getJobStatus(jobid); + JobProfile profile = tracker.getJobProfile(jobid); if (status == null || profile == null) // No such job. - throw new BadParam("Could not find job " + bestid); + throw new BadParam("Could not find job " + jobid); return new QueueStatusBean(state, status, profile); } - public static QueueStatusBean makeStatus(WebHCatJTShim tracker, - JobID jobid, - JobState state) - throws BadParam, IOException { - return makeStatus(tracker, jobid, state.getChildId(), state); - } - /** * A version of JobID.forName with our app specific error handling. */ Modified: hive/branches/tez/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/TempletonDelegator.java URL: http://svn.apache.org/viewvc/hive/branches/tez/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/TempletonDelegator.java?rev=1538880&r1=1538879&r2=1538880&view=diff ============================================================================== --- hive/branches/tez/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/TempletonDelegator.java (original) +++ hive/branches/tez/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/TempletonDelegator.java Tue Nov 5 07:01:32 2013 @@ -24,6 +24,11 @@ package org.apache.hive.hcatalog.templet * or hive. */ public class TempletonDelegator { + /** + * http://hadoop.apache.org/docs/r1.0.4/commands_manual.html#Generic+Options + */ + public static final String ARCHIVES = "-archives"; + protected AppConfig appConf; public TempletonDelegator(AppConfig appConf) { Modified: hive/branches/tez/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/JobState.java URL: http://svn.apache.org/viewvc/hive/branches/tez/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/JobState.java?rev=1538880&r1=1538879&r2=1538880&view=diff ============================================================================== --- hive/branches/tez/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/JobState.java (original) +++ hive/branches/tez/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/JobState.java Tue Nov 5 07:01:32 2013 @@ -126,19 +126,6 @@ public class JobState { } /** - * The child id of TempletonControllerJob - */ - public String getChildId() - throws IOException { - return getField("childid"); - } - - public void setChildId(String childid) - throws IOException { - setField("childid", childid); - } - - /** * Add a jobid to the list of children of this job. * * @param jobid @@ -151,6 +138,9 @@ public class JobState { } catch (Exception e) { // There are none or they're not readable. } + if (jobids==null) { + jobids = ""; + } if (!jobids.equals("")) { jobids += ","; } @@ -159,6 +149,18 @@ public class JobState { } /** + * Set parent job of this job + * @param id + */ + public void setParent(String id) throws IOException { + setField("parent", id); + } + + public String getParent() throws IOException { + return getField("parent"); + } + + /** * Get a list of jobstates for jobs that are children of this job. * @throws IOException */ @@ -171,31 +173,6 @@ public class JobState { } /** - * Save a comma-separated list of jobids that are children - * of this job. - * @param jobids - * @throws IOException - */ - public void setChildren(String jobids) throws IOException { - setField("children", jobids); - } - - /** - * Set the list of child jobs of this job - * @param children - */ - public void setChildren(List children) throws IOException { - String val = ""; - for (JobState jobstate : children) { - if (!val.equals("")) { - val += ","; - } - val += jobstate.getId(); - } - setField("children", val); - } - - /** * The system exit value of the job. */ public Long getExitValue() Modified: hive/branches/tez/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/JobSubmissionConstants.java URL: http://svn.apache.org/viewvc/hive/branches/tez/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/JobSubmissionConstants.java?rev=1538880&r1=1538879&r2=1538880&view=diff ============================================================================== --- hive/branches/tez/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/JobSubmissionConstants.java (original) +++ hive/branches/tez/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/JobSubmissionConstants.java Tue Nov 5 07:01:32 2013 @@ -34,4 +34,12 @@ public interface JobSubmissionConstants public static final int WATCHER_TIMEOUT_SECS = 10; public static final int KEEP_ALIVE_MSEC = 60 * 1000; public static final String TOKEN_FILE_ARG_PLACEHOLDER = "__WEBHCAT_TOKEN_FILE_LOCATION__"; + /** + * constants needed for Pig job submission + */ + public static interface PigConstants { + public static final String HIVE_HOME = "HIVE_HOME"; + public static final String HCAT_HOME = "HCAT_HOME"; + public static final String PIG_OPTS = "PIG_OPTS"; + } } Modified: hive/branches/tez/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/LaunchMapper.java URL: http://svn.apache.org/viewvc/hive/branches/tez/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/LaunchMapper.java?rev=1538880&r1=1538879&r2=1538880&view=diff ============================================================================== --- hive/branches/tez/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/LaunchMapper.java (original) +++ hive/branches/tez/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/LaunchMapper.java Tue Nov 5 07:01:32 2013 @@ -33,6 +33,7 @@ import org.apache.hive.hcatalog.templeto import org.apache.hive.hcatalog.templeton.LauncherDelegator; import java.io.BufferedReader; +import java.io.File; import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; @@ -65,8 +66,25 @@ public class LaunchMapper extends Mapper * it will end up in 'syslog' of this Map task. For example, look for KeepAlive heartbeat msgs. */ private static final Log LOG = LogFactory.getLog(LaunchMapper.class); - - + /** + * When a Pig job is submitted and it uses HCat, WebHCat may be configured to ship hive tar + * to the target node. Pig on the target node needs some env vars configured. + */ + private static void handlePigEnvVars(Configuration conf, Map env) { + if(conf.get(PigConstants.HIVE_HOME) != null) { + env.put(PigConstants.HIVE_HOME, new File(conf.get(PigConstants.HIVE_HOME)).getAbsolutePath()); + } + if(conf.get(PigConstants.HCAT_HOME) != null) { + env.put(PigConstants.HCAT_HOME, new File(conf.get(PigConstants.HCAT_HOME)).getAbsolutePath()); + } + if(conf.get(PigConstants.PIG_OPTS) != null) { + StringBuilder pigOpts = new StringBuilder(); + for(String prop : conf.get(PigConstants.PIG_OPTS).split(",")) { + pigOpts.append("-D").append(prop).append(" "); + } + env.put(PigConstants.PIG_OPTS, pigOpts.toString()); + } + } protected Process startJob(Context context, String user, String overrideClasspath) throws IOException, InterruptedException { Configuration conf = context.getConfiguration(); @@ -79,8 +97,8 @@ public class LaunchMapper extends Mapper removeEnv.add("hadoop-command"); removeEnv.add("CLASS"); removeEnv.add("mapredcommand"); - Map env = TempletonUtils.hadoopUserEnv(user, - overrideClasspath); + Map env = TempletonUtils.hadoopUserEnv(user, overrideClasspath); + handlePigEnvVars(conf, env); List jarArgsList = new LinkedList(Arrays.asList(jarArgs)); String tokenFile = System.getenv("HADOOP_TOKEN_FILE_LOCATION"); @@ -261,8 +279,15 @@ public class LaunchMapper extends Mapper if (percent != null || childid != null) { state = new JobState(jobid.toString(), conf); - state.setPercentComplete(percent); - state.setChildId(childid); + if (percent != null) { + state.setPercentComplete(percent); + } + if (childid != null) { + JobState childState = new JobState(childid, conf); + childState.setParent(jobid.toString()); + state.addChild(childid); + state.close(); + } } } catch (IOException e) { LOG.error("templeton: state error: ", e); Modified: hive/branches/tez/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/TempletonControllerJob.java URL: http://svn.apache.org/viewvc/hive/branches/tez/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/TempletonControllerJob.java?rev=1538880&r1=1538879&r2=1538880&view=diff ============================================================================== --- hive/branches/tez/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/TempletonControllerJob.java (original) +++ hive/branches/tez/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/TempletonControllerJob.java Tue Nov 5 07:01:32 2013 @@ -22,6 +22,7 @@ import java.io.File; import java.io.IOException; import java.net.URI; import java.security.PrivilegedExceptionAction; +import java.util.Arrays; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -182,6 +183,9 @@ public class TempletonControllerJob exte @Override public int run(String[] args) throws IOException, InterruptedException, ClassNotFoundException, TException { + if(LOG.isDebugEnabled()) { + LOG.debug("Preparing to submit job: " + Arrays.toString(args)); + } Configuration conf = getConf(); conf.set(JAR_ARGS_NAME, TempletonUtils.encodeArray(args)); Modified: hive/branches/tez/hcatalog/webhcat/svr/src/test/java/org/apache/hive/hcatalog/templeton/TestWebHCatE2e.java URL: http://svn.apache.org/viewvc/hive/branches/tez/hcatalog/webhcat/svr/src/test/java/org/apache/hive/hcatalog/templeton/TestWebHCatE2e.java?rev=1538880&r1=1538879&r2=1538880&view=diff ============================================================================== --- hive/branches/tez/hcatalog/webhcat/svr/src/test/java/org/apache/hive/hcatalog/templeton/TestWebHCatE2e.java (original) +++ hive/branches/tez/hcatalog/webhcat/svr/src/test/java/org/apache/hive/hcatalog/templeton/TestWebHCatE2e.java Tue Nov 5 07:01:32 2013 @@ -44,7 +44,7 @@ import java.util.Map; /** * A set of tests exercising e2e WebHCat DDL APIs. These tests are somewhat * between WebHCat e2e (hcatalog/src/tests/e2e/templeton) tests and simple58 - * + * * unit tests. This will start a WebHCat server and make REST calls to it. * It doesn't need Hadoop or (standalone) metastore to be running. * Running this is much simpler than e2e tests. @@ -75,7 +75,7 @@ public class TestWebHCatE2e { LOG.warn("Unable to find free port; using default: " + webhcatPort); } templetonBaseUrl = templetonBaseUrl.replace("50111", Integer.toString(webhcatPort)); - templetonServer = new Main(new String[] {"-D" + + templetonServer = new Main(new String[] {"-D" + AppConfig.UNIT_TEST_MODE + "=true", "-D" + AppConfig.PORT + "=" + webhcatPort}); LOG.info("Starting Main; WebHCat using port: " + webhcatPort); templetonServer.run(); @@ -156,7 +156,7 @@ public class TestWebHCatE2e { public void createDataBase() throws IOException { Map props = new HashMap(); props.put("comment", "Hello, there"); - props.put("location", "file://" + System.getProperty("hive.metastore.warehouse.dir")); + props.put("location", System.getProperty("test.warehouse.dir")); Map props2 = new HashMap(); props2.put("prop", "val"); props.put("properties", props2); Modified: hive/branches/tez/hcatalog/webhcat/svr/src/test/java/org/apache/hive/hcatalog/templeton/tool/TestJobIDParser.java URL: http://svn.apache.org/viewvc/hive/branches/tez/hcatalog/webhcat/svr/src/test/java/org/apache/hive/hcatalog/templeton/tool/TestJobIDParser.java?rev=1538880&r1=1538879&r2=1538880&view=diff ============================================================================== --- hive/branches/tez/hcatalog/webhcat/svr/src/test/java/org/apache/hive/hcatalog/templeton/tool/TestJobIDParser.java (original) +++ hive/branches/tez/hcatalog/webhcat/svr/src/test/java/org/apache/hive/hcatalog/templeton/tool/TestJobIDParser.java Tue Nov 5 07:01:32 2013 @@ -29,7 +29,7 @@ import junit.framework.Assert; public class TestJobIDParser { @Test public void testParsePig() throws IOException { - String errFileName = "../../src/test/data/status/pig"; + String errFileName = "src/test/data/status/pig"; PigJobIDParser pigJobIDParser = new PigJobIDParser(errFileName, new Configuration()); List jobs = pigJobIDParser.parseJobID(); Assert.assertEquals(jobs.size(), 1); @@ -37,7 +37,7 @@ public class TestJobIDParser { @Test public void testParseHive() throws IOException { - String errFileName = "../../src/test/data/status/hive"; + String errFileName = "src/test/data/status/hive"; HiveJobIDParser hiveJobIDParser = new HiveJobIDParser(errFileName, new Configuration()); List jobs = hiveJobIDParser.parseJobID(); Assert.assertEquals(jobs.size(), 1); @@ -45,7 +45,7 @@ public class TestJobIDParser { @Test public void testParseJar() throws IOException { - String errFileName = "../../src/test/data/status/jar"; + String errFileName = "src/test/data/status/jar"; JarJobIDParser jarJobIDParser = new JarJobIDParser(errFileName, new Configuration()); List jobs = jarJobIDParser.parseJobID(); Assert.assertEquals(jobs.size(), 1); @@ -53,7 +53,7 @@ public class TestJobIDParser { @Test public void testParseStreaming() throws IOException { - String errFileName = "../../src/test/data/status/streaming"; + String errFileName = "src/test/data/status/streaming"; JarJobIDParser jarJobIDParser = new JarJobIDParser(errFileName, new Configuration()); List jobs = jarJobIDParser.parseJobID(); Assert.assertEquals(jobs.size(), 1); Modified: hive/branches/tez/hcatalog/webhcat/svr/src/test/java/org/apache/hive/hcatalog/templeton/tool/TestTempletonUtils.java URL: http://svn.apache.org/viewvc/hive/branches/tez/hcatalog/webhcat/svr/src/test/java/org/apache/hive/hcatalog/templeton/tool/TestTempletonUtils.java?rev=1538880&r1=1538879&r2=1538880&view=diff ============================================================================== --- hive/branches/tez/hcatalog/webhcat/svr/src/test/java/org/apache/hive/hcatalog/templeton/tool/TestTempletonUtils.java (original) +++ hive/branches/tez/hcatalog/webhcat/svr/src/test/java/org/apache/hive/hcatalog/templeton/tool/TestTempletonUtils.java Tue Nov 5 07:01:32 2013 @@ -35,7 +35,7 @@ public class TestTempletonUtils { "2011-12-15 18:12:21,758 [main] INFO org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.MapReduceLauncher - More information at: http://localhost:50030/jobdetails.jsp?jobid=job_201112140012_0047", "2011-12-15 18:12:46,907 [main] INFO org.apache.pig.tools.pigstats.SimplePigStats - Script Statistics: " }; - public static final String testDataDir = System.getProperty("test.data.dir"); + public static final String testDataDir = System.getProperty("test.tmp.dir"); File tmpFile; File usrFile; Modified: hive/branches/tez/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestLocationQueries.java URL: http://svn.apache.org/viewvc/hive/branches/tez/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestLocationQueries.java?rev=1538880&r1=1538724&r2=1538880&view=diff ============================================================================== --- hive/branches/tez/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestLocationQueries.java (original) +++ hive/branches/tez/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestLocationQueries.java Tue Nov 5 07:01:32 2013 @@ -24,6 +24,8 @@ import java.io.FileReader; import java.util.regex.Matcher; import java.util.regex.Pattern; +import org.apache.hadoop.hive.ql.QTestUtil.MiniClusterType; + /** * Suite for testing location. e.g. if "alter table alter partition * location" is run, do the partitions end up in the correct location. @@ -82,7 +84,7 @@ public class TestLocationQueries extends return failedCount; } - public CheckResults(String outDir, String logDir, boolean miniMr, + public CheckResults(String outDir, String logDir, MiniClusterType miniMr, String hadoopVer, String locationSubdir) throws Exception { @@ -102,8 +104,9 @@ public class TestLocationQueries extends File[] qfiles = setupQFiles(testNames); QTestUtil[] qt = new QTestUtil[qfiles.length]; + for (int i = 0; i < qfiles.length; i++) { - qt[i] = new CheckResults(resDir, logDir, false, "0.20", "parta"); + qt[i] = new CheckResults(resDir, logDir, MiniClusterType.none, "0.20", "parta"); qt[i].addFile(qfiles[i]); qt[i].clearTestSideEffects(); } Modified: hive/branches/tez/itests/qtest/pom.xml URL: http://svn.apache.org/viewvc/hive/branches/tez/itests/qtest/pom.xml?rev=1538880&r1=1538724&r2=1538880&view=diff ============================================================================== --- hive/branches/tez/itests/qtest/pom.xml (original) +++ hive/branches/tez/itests/qtest/pom.xml Tue Nov 5 07:01:32 2013 @@ -38,6 +38,7 @@ false list_bucket_dml_10.q,input16_cc.q,scriptfile1.q,scriptfile1_win.q,bucket4.q,bucketmapjoin6.q,disable_merge_for_bucketing.q,reduce_deduplicate.q,smb_mapjoin_8.q,join1.q,groupby2.q,bucketizedhiveinputformat.q,bucketmapjoin7.q,optrstat_groupby.q,bucket_num_reducers.q,bucket5.q,load_fs2.q,bucket_num_reducers2.q,infer_bucket_sort_merge.q,infer_bucket_sort_reducers_power_two.q,infer_bucket_sort_dyn_part.q,infer_bucket_sort_bucketed_table.q,infer_bucket_sort_map_operators.q,infer_bucket_sort_num_buckets.q,leftsemijoin_mr.q,schemeAuthority.q,schemeAuthority2.q,truncate_column_buckets.q,remote_script.q,,load_hdfs_file_with_space_in_the_name.q,parallel_orderby.q,import_exported_table.q cluster_tasklog_retrieval.q,minimr_broken_pipe.q,mapreduce_stack_trace.q,mapreduce_stack_trace_turnoff.q,mapreduce_stack_trace_hadoop20.q,mapreduce_stack_trace_turnoff_hadoop20.q + tez_join_tests.q,tez_joins_explain.q,mrr.q,tez_dml.q add_part_exist.q,alter1.q,alter2.q,alter4.q,alter5.q,alter_rename_partition.q,alter_rename_partition_authorization.q,archive.q,archive_corrupt.q,archive_multi.q,archive_mr_1806.q,archive_multi_mr_1806.q,authorization_1.q,authorization_2.q,authorization_4.q,authorization_5.q,authorization_6.q,authorization_7.q,ba_table1.q,ba_table2.q,ba_table3.q,ba_table_udfs.q,binary_table_bincolserde.q,binary_table_colserde.q,cluster.q,columnarserde_create_shortcut.q,combine2.q,constant_prop.q,create_nested_type.q,create_or_replace_view.q,create_struct_table.q,create_union_table.q,database.q,database_location.q,database_properties.q,ddltime.q,describe_database_json.q,drop_database_removes_partition_dirs.q,escape1.q,escape2.q,exim_00_nonpart_empty.q,exim_01_nonpart.q,exim_02_00_part_empty.q,exim_02_part.q,exim_03_nonpart_over_compat.q,exim_04_all_part.q,exim_04_evolved_parts.q,exim_05_some_part.q,exim_06_one_part.q,exim_07_all_part_over_nonoverlap.q,exim_08_nonpart_rena me.q,exim_09_part_spec_nonoverlap.q,exim_10_external_managed.q,exim_11_managed_external.q,exim_12_external_location.q,exim_13_managed_location.q,exim_14_managed_location_over_existing.q,exim_15_external_part.q,exim_16_part_external.q,exim_17_part_managed.q,exim_18_part_external.q,exim_19_00_part_external_location.q,exim_19_part_external_location.q,exim_20_part_managed_location.q,exim_21_export_authsuccess.q,exim_22_import_exist_authsuccess.q,exim_23_import_part_authsuccess.q,exim_24_import_nonexist_authsuccess.q,global_limit.q,groupby_complex_types.q,groupby_complex_types_multi_single_reducer.q,index_auth.q,index_auto.q,index_auto_empty.q,index_bitmap.q,index_bitmap1.q,index_bitmap2.q,index_bitmap3.q,index_bitmap_auto.q,index_bitmap_rc.q,index_compact.q,index_compact_1.q,index_compact_2.q,index_compact_3.q,index_stale_partitioned.q,init_file.q,input16.q,input16_cc.q,input46.q,input_columnarserde.q,input_dynamicserde.q,input_lazyserde.q,input_testxpath3.q,input_testxpath4.q,insert2_o verwrite_partitions.q,insertexternal1.q,join_thrift.q,lateral_view.q,load_binary_data.q,load_exist_part_authsuccess.q,load_nonpart_authsuccess.q,load_part_authsuccess.q,loadpart_err.q,lock1.q,lock2.q,lock3.q,lock4.q,merge_dynamic_partition.q,multi_insert.q,multi_insert_move_tasks_share_dependencies.q,null_column.q,ppd_clusterby.q,query_with_semi.q,rename_column.q,sample6.q,sample_islocalmode_hook.q,set_processor_namespaces.q,show_tables.q,source.q,split_sample.q,str_to_map.q,transform1.q,udaf_collect_set.q,udaf_context_ngrams.q,udaf_histogram_numeric.q,udaf_ngrams.q,udaf_percentile_approx.q,udf_array.q,udf_bitmap_and.q,udf_bitmap_or.q,udf_explode.q,udf_format_number.q,udf_map.q,udf_map_keys.q,udf_map_values.q,udf_max.q,udf_min.q,udf_named_struct.q,udf_percentile.q,udf_printf.q,udf_sentences.q,udf_sort_array.q,udf_split.q,udf_struct.q,udf_substr.q,udf_translate.q,udf_union.q,udf_xpath.q,udtf_stack.q,view.q,virtual_column.q @@ -119,7 +120,6 @@ tests - @@ -191,6 +191,11 @@ ${hadoop-23.version} test + + commons-logging + commons-logging + ${commons-logging.version} + @@ -207,6 +212,7 @@ + + + + + + + + + + qSortSet; private static final String SORT_SUFFIX = ".sorted"; public static final HashSet srcTables = new HashSet(); + private static MiniClusterType clusterType = MiniClusterType.none; private ParseDriver pd; private Hive db; protected HiveConf conf; @@ -204,7 +206,7 @@ public class QTestUtil { } public QTestUtil(String outDir, String logDir) throws Exception { - this(outDir, logDir, false, "0.20"); + this(outDir, logDir, MiniClusterType.none, "0.20"); } public String getOutputDirectory() { @@ -242,9 +244,8 @@ public class QTestUtil { conf.setVar(HiveConf.ConfVars.METASTORE_RAW_STORE_IMPL, "org.apache.hadoop.hive.metastore.VerifyingObjectStore"); - if (miniMr) { + if (mr != null) { assert dfs != null; - assert mr != null; mr.setupConfiguration(conf); @@ -297,21 +298,46 @@ public class QTestUtil { return uriStr; } - public QTestUtil(String outDir, String logDir, boolean miniMr, String hadoopVer) + public enum MiniClusterType { + mr, + tez, + none; + + public static MiniClusterType valueForString(String type) { + if (type.equals("miniMR")) { + return mr; + } else if (type.equals("tez")) { + return tez; + } else { + return none; + } + } + } + + public QTestUtil(String outDir, String logDir, MiniClusterType clusterType, String hadoopVer) throws Exception { this.outDir = outDir; this.logDir = logDir; conf = new HiveConf(Driver.class); - this.miniMr = miniMr; + this.miniMr = (clusterType == MiniClusterType.mr); this.hadoopVer = getHadoopMainVersion(hadoopVer); qMap = new TreeMap(); qSkipSet = new HashSet(); qSortSet = new HashSet(); + this.clusterType = clusterType; - if (miniMr) { - dfs = ShimLoader.getHadoopShims().getMiniDfs(conf, 4, true, null); + HadoopShims shims = ShimLoader.getHadoopShims(); + if (clusterType != MiniClusterType.none) { + dfs = shims.getMiniDfs(conf, 4, true, null); FileSystem fs = dfs.getFileSystem(); - mr = ShimLoader.getHadoopShims().getMiniMrCluster(conf, 4, getHdfsUriString(fs.getUri().toString()), 1); + if (clusterType == MiniClusterType.tez) { + if (!(shims instanceof Hadoop23Shims)) { + throw new Exception("Cannot run tez on hadoop-1, Version: "+this.hadoopVer); + } + mr = ((Hadoop23Shims)shims).getMiniTezCluster(conf, 4, getHdfsUriString(fs.getUri().toString()), 1); + } else { + mr = shims.getMiniMrCluster(conf, 4, getHdfsUriString(fs.getUri().toString()), 1); + } } initConf(); @@ -761,6 +787,11 @@ public class QTestUtil { ss.err = new CachingPrintStream(fo, true, "UTF-8"); ss.setIsSilent(true); SessionState oldSs = SessionState.get(); + + if (oldSs != null && clusterType == MiniClusterType.tez) { + oldSs.close(); + } + if (oldSs != null && oldSs.out != null && oldSs.out != System.out) { oldSs.out.close(); } @@ -1442,7 +1473,7 @@ public class QTestUtil { { QTestUtil[] qt = new QTestUtil[qfiles.length]; for (int i = 0; i < qfiles.length; i++) { - qt[i] = new QTestUtil(resDir, logDir, false, "0.20"); + qt[i] = new QTestUtil(resDir, logDir, MiniClusterType.none, "0.20"); qt[i].addFile(qfiles[i]); qt[i].clearTestSideEffects(); }