hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From gunt...@apache.org
Subject svn commit: r1552848 - in /hive/branches/tez: ./ data/scripts/ hcatalog/core/src/main/java/org/apache/hive/hcatalog/common/ hcatalog/core/src/test/java/org/apache/hcatalog/mapreduce/ hcatalog/core/src/test/java/org/apache/hive/hcatalog/ hcatalog/core/s...
Date Sat, 21 Dec 2013 00:15:48 GMT
Author: gunther
Date: Sat Dec 21 00:15:47 2013
New Revision: 1552848

URL: http://svn.apache.org/r1552848
Log:
Merge latest trunk into branch. (Gunther Hagleitner)

Added:
    hive/branches/tez/data/scripts/input20_script.py
      - copied unchanged from r1552841, hive/trunk/data/scripts/input20_script.py
Modified:
    hive/branches/tez/   (props changed)
    hive/branches/tez/hcatalog/core/src/main/java/org/apache/hive/hcatalog/common/HCatUtil.java
    hive/branches/tez/hcatalog/core/src/test/java/org/apache/hcatalog/mapreduce/HCatBaseTest.java
    hive/branches/tez/hcatalog/core/src/test/java/org/apache/hcatalog/mapreduce/TestHCatPartitionPublish.java
    hive/branches/tez/hcatalog/core/src/test/java/org/apache/hcatalog/mapreduce/TestMultiOutputFormat.java
    hive/branches/tez/hcatalog/core/src/test/java/org/apache/hive/hcatalog/HcatTestUtils.java
    hive/branches/tez/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/HCatBaseTest.java
    hive/branches/tez/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatPartitionPublish.java
    hive/branches/tez/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestMultiOutputFormat.java
    hive/branches/tez/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hcatalog/pig/TestHCatLoader.java
    hive/branches/tez/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hcatalog/pig/TestHCatLoaderStorer.java
    hive/branches/tez/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hcatalog/pig/TestHCatStorerMulti.java
    hive/branches/tez/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hcatalog/pig/TestHCatStorerWrapper.java
    hive/branches/tez/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoader.java
    hive/branches/tez/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatStorerMulti.java
    hive/branches/tez/hcatalog/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/SkeletonHBaseTest.java
    hive/branches/tez/hcatalog/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/TestHCatHBaseInputFormat.java
    hive/branches/tez/hcatalog/storage-handlers/hbase/src/test/org/apache/hive/hcatalog/hbase/SkeletonHBaseTest.java
    hive/branches/tez/hcatalog/webhcat/java-client/src/test/java/org/apache/hcatalog/api/TestHCatClient.java
    hive/branches/tez/hcatalog/webhcat/java-client/src/test/java/org/apache/hive/hcatalog/api/TestHCatClient.java
    hive/branches/tez/itests/hcatalog-unit/src/test/java/org/apache/hcatalog/mapreduce/TestSequenceFileReadWrite.java
    hive/branches/tez/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/hbase/TestPigHBaseStorageHandler.java
    hive/branches/tez/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/mapreduce/TestSequenceFileReadWrite.java
    hive/branches/tez/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
    hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
    hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/metadata/formatting/TextMetaDataFormatter.java
    hive/branches/tez/ql/src/test/queries/clientpositive/input20.q
    hive/branches/tez/ql/src/test/queries/clientpositive/input33.q
    hive/branches/tez/ql/src/test/results/beelinepositive/input20.q.out
    hive/branches/tez/ql/src/test/results/beelinepositive/input33.q.out
    hive/branches/tez/ql/src/test/results/beelinepositive/newline.q.out
    hive/branches/tez/ql/src/test/results/clientpositive/input20.q.out
    hive/branches/tez/ql/src/test/results/clientpositive/input33.q.out

Propchange: hive/branches/tez/
------------------------------------------------------------------------------
  Merged /hive/trunk:r1552793-1552841

Modified: hive/branches/tez/hcatalog/core/src/main/java/org/apache/hive/hcatalog/common/HCatUtil.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/hcatalog/core/src/main/java/org/apache/hive/hcatalog/common/HCatUtil.java?rev=1552848&r1=1552847&r2=1552848&view=diff
==============================================================================
--- hive/branches/tez/hcatalog/core/src/main/java/org/apache/hive/hcatalog/common/HCatUtil.java (original)
+++ hive/branches/tez/hcatalog/core/src/main/java/org/apache/hive/hcatalog/common/HCatUtil.java Sat Dec 21 00:15:47 2013
@@ -21,6 +21,7 @@ package org.apache.hive.hcatalog.common;
 
 import java.io.ByteArrayInputStream;
 import java.io.ByteArrayOutputStream;
+import java.io.File;
 import java.io.IOException;
 import java.io.ObjectInputStream;
 import java.io.ObjectOutputStream;
@@ -645,4 +646,10 @@ public class HCatUtil {
       return true;
     return false;
   }
+  /**
+   * Used by various tests to make sure the path is safe for Windows
+   */
+  public static String makePathASafeFileName(String filePath) {
+    return new File(filePath).getPath().replaceAll("\\\\", "/");
+  }
 }

Modified: hive/branches/tez/hcatalog/core/src/test/java/org/apache/hcatalog/mapreduce/HCatBaseTest.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/hcatalog/core/src/test/java/org/apache/hcatalog/mapreduce/HCatBaseTest.java?rev=1552848&r1=1552847&r2=1552848&view=diff
==============================================================================
--- hive/branches/tez/hcatalog/core/src/test/java/org/apache/hcatalog/mapreduce/HCatBaseTest.java (original)
+++ hive/branches/tez/hcatalog/core/src/test/java/org/apache/hcatalog/mapreduce/HCatBaseTest.java Sat Dec 21 00:15:47 2013
@@ -41,8 +41,7 @@ import java.io.IOException;
  */
 public class HCatBaseTest {
   protected static final Logger LOG = LoggerFactory.getLogger(HCatBaseTest.class);
-  protected static final String TEST_DATA_DIR = System.getProperty("user.dir") +
-      "/build/test/data/" + HCatBaseTest.class.getCanonicalName();
+  protected static final String TEST_DATA_DIR = org.apache.hive.hcatalog.mapreduce.HCatBaseTest.TEST_DATA_DIR;
   protected static final String TEST_WAREHOUSE_DIR = TEST_DATA_DIR + "/warehouse";
 
   protected HiveConf hiveConf = null;

Modified: hive/branches/tez/hcatalog/core/src/test/java/org/apache/hcatalog/mapreduce/TestHCatPartitionPublish.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/hcatalog/core/src/test/java/org/apache/hcatalog/mapreduce/TestHCatPartitionPublish.java?rev=1552848&r1=1552847&r2=1552848&view=diff
==============================================================================
--- hive/branches/tez/hcatalog/core/src/test/java/org/apache/hcatalog/mapreduce/TestHCatPartitionPublish.java (original)
+++ hive/branches/tez/hcatalog/core/src/test/java/org/apache/hcatalog/mapreduce/TestHCatPartitionPublish.java Sat Dec 21 00:15:47 2013
@@ -79,12 +79,12 @@ public class TestHCatPartitionPublish {
 
   @BeforeClass
   public static void setup() throws Exception {
+    File workDir = org.apache.hive.hcatalog.mapreduce.TestHCatPartitionPublish.handleWorkDir();
     conf.set("yarn.scheduler.capacity.root.queues", "default");
     conf.set("yarn.scheduler.capacity.root.default.capacity", "100");
 
     fs = FileSystem.get(conf);
-    System.setProperty("hadoop.log.dir", new File(fs.getWorkingDirectory()
-        .toString(), "/logs").getAbsolutePath());
+    System.setProperty("hadoop.log.dir", new File(workDir, "/logs").getAbsolutePath());
     // LocalJobRunner does not work with mapreduce OutputCommitter. So need
     // to use MiniMRCluster. MAPREDUCE-2350
     mrCluster = new MiniMRCluster(1, fs.getUri().toString(), 1, null, null,
@@ -99,6 +99,7 @@ public class TestHCatPartitionPublish {
 
     MetaStoreUtils.startMetaStore(msPort, ShimLoader
         .getHadoopThriftAuthBridge());
+    Thread.sleep(10000);
     isServerRunning = true;
     securityManager = System.getSecurityManager();
     System.setSecurityManager(new NoExitSecurityManager());

Modified: hive/branches/tez/hcatalog/core/src/test/java/org/apache/hcatalog/mapreduce/TestMultiOutputFormat.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/hcatalog/core/src/test/java/org/apache/hcatalog/mapreduce/TestMultiOutputFormat.java?rev=1552848&r1=1552847&r2=1552848&view=diff
==============================================================================
--- hive/branches/tez/hcatalog/core/src/test/java/org/apache/hcatalog/mapreduce/TestMultiOutputFormat.java (original)
+++ hive/branches/tez/hcatalog/core/src/test/java/org/apache/hcatalog/mapreduce/TestMultiOutputFormat.java Sat Dec 21 00:15:47 2013
@@ -67,7 +67,7 @@ public class TestMultiOutputFormat {
 
   private static final Logger LOG = LoggerFactory.getLogger(TestMultiOutputFormat.class);
   private static File workDir;
-  private static Configuration mrConf = null;
+  private static JobConf mrConf = null;
   private static FileSystem fs = null;
   private static MiniMRCluster mrCluster = null;
 
@@ -82,9 +82,8 @@ public class TestMultiOutputFormat {
     System.setProperty("hadoop.log.dir", new File(workDir, "/logs").getAbsolutePath());
     // LocalJobRunner does not work with mapreduce OutputCommitter. So need
     // to use MiniMRCluster. MAPREDUCE-2350
-    mrCluster = new MiniMRCluster(1, fs.getUri().toString(), 1, null, null,
-      new JobConf(conf));
-    mrConf = mrCluster.createJobConf();
+    mrConf = new JobConf(conf);
+    mrCluster = new MiniMRCluster(1, fs.getUri().toString(), 1, null, null, mrConf);
   }
 
   private static void createWorkDir() throws IOException {

Modified: hive/branches/tez/hcatalog/core/src/test/java/org/apache/hive/hcatalog/HcatTestUtils.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/hcatalog/core/src/test/java/org/apache/hive/hcatalog/HcatTestUtils.java?rev=1552848&r1=1552847&r2=1552848&view=diff
==============================================================================
--- hive/branches/tez/hcatalog/core/src/test/java/org/apache/hive/hcatalog/HcatTestUtils.java (original)
+++ hive/branches/tez/hcatalog/core/src/test/java/org/apache/hive/hcatalog/HcatTestUtils.java Sat Dec 21 00:15:47 2013
@@ -97,4 +97,10 @@ public class HcatTestUtils {
     }
 
   }
+  /**
+   * Used by various tests to make sure the path is safe for Windows
+   */
+  public static String makePathASafeFileName(String filePath) {
+    return new File(filePath).getPath().replaceAll("\\\\", "/");
+  }
 }

Modified: hive/branches/tez/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/HCatBaseTest.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/HCatBaseTest.java?rev=1552848&r1=1552847&r2=1552848&view=diff
==============================================================================
--- hive/branches/tez/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/HCatBaseTest.java (original)
+++ hive/branches/tez/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/HCatBaseTest.java Sat Dec 21 00:15:47 2013
@@ -25,6 +25,7 @@ import org.apache.hadoop.hive.conf.HiveC
 import org.apache.hadoop.hive.metastore.HiveMetaStoreClient;
 import org.apache.hadoop.hive.ql.Driver;
 import org.apache.hadoop.hive.ql.session.SessionState;
+import org.apache.hive.hcatalog.common.HCatUtil;
 import org.apache.pig.PigServer;
 import org.junit.Assert;
 import org.junit.Before;
@@ -40,8 +41,8 @@ import java.io.IOException;
  */
 public class HCatBaseTest {
   protected static final Logger LOG = LoggerFactory.getLogger(HCatBaseTest.class);
-  protected static final String TEST_DATA_DIR =
-      "/tmp/build/test/data/" + HCatBaseTest.class.getCanonicalName();
+  public static final String TEST_DATA_DIR = HCatUtil.makePathASafeFileName(System.getProperty("user.dir") +
+          "/build/test/data/" + HCatBaseTest.class.getCanonicalName() + "-" + System.currentTimeMillis());
   protected static final String TEST_WAREHOUSE_DIR = TEST_DATA_DIR + "/warehouse";
 
   protected HiveConf hiveConf = null;

Modified: hive/branches/tez/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatPartitionPublish.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatPartitionPublish.java?rev=1552848&r1=1552847&r2=1552848&view=diff
==============================================================================
--- hive/branches/tez/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatPartitionPublish.java (original)
+++ hive/branches/tez/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatPartitionPublish.java Sat Dec 21 00:15:47 2013
@@ -76,13 +76,17 @@ public class TestHCatPartitionPublish {
   private static SecurityManager securityManager;
   private static Configuration conf = new Configuration(true);
 
-  @BeforeClass
-  public static void setup() throws Exception {
-    String testDir = System.getProperty("test.tmp.dir", "./");
+  public static File handleWorkDir() throws IOException {
+    String testDir = System.getProperty("test.data.dir", "./");
     testDir = testDir + "/test_hcat_partitionpublish_" + Math.abs(new Random().nextLong()) + "/";
     File workDir = new File(new File(testDir).getCanonicalPath());
     FileUtil.fullyDelete(workDir);
     workDir.mkdirs();
+    return workDir;
+  }
+  @BeforeClass
+  public static void setup() throws Exception {
+    File workDir = handleWorkDir();
     conf.set("yarn.scheduler.capacity.root.queues", "default");
     conf.set("yarn.scheduler.capacity.root.default.capacity", "100");
 

Modified: hive/branches/tez/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestMultiOutputFormat.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestMultiOutputFormat.java?rev=1552848&r1=1552847&r2=1552848&view=diff
==============================================================================
--- hive/branches/tez/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestMultiOutputFormat.java (original)
+++ hive/branches/tez/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestMultiOutputFormat.java Sat Dec 21 00:15:47 2013
@@ -64,7 +64,7 @@ public class TestMultiOutputFormat {
 
   private static final Logger LOG = LoggerFactory.getLogger(TestMultiOutputFormat.class);
   private static File workDir;
-  private static Configuration mrConf = null;
+  private static JobConf mrConf = null;
   private static FileSystem fs = null;
   private static MiniMRCluster mrCluster = null;
 
@@ -79,9 +79,8 @@ public class TestMultiOutputFormat {
     System.setProperty("hadoop.log.dir", new File(workDir, "/logs").getAbsolutePath());
     // LocalJobRunner does not work with mapreduce OutputCommitter. So need
     // to use MiniMRCluster. MAPREDUCE-2350
-    mrCluster = new MiniMRCluster(1, fs.getUri().toString(), 1, null, null,
-      new JobConf(conf));
-    mrConf = mrCluster.createJobConf();
+    mrConf = new JobConf(conf);
+    mrCluster = new MiniMRCluster(1, fs.getUri().toString(), 1, null, null, mrConf);
   }
 
   private static void createWorkDir() throws IOException {

Modified: hive/branches/tez/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hcatalog/pig/TestHCatLoader.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hcatalog/pig/TestHCatLoader.java?rev=1552848&r1=1552847&r2=1552848&view=diff
==============================================================================
--- hive/branches/tez/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hcatalog/pig/TestHCatLoader.java (original)
+++ hive/branches/tez/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hcatalog/pig/TestHCatLoader.java Sat Dec 21 00:15:47 2013
@@ -45,6 +45,7 @@ import org.apache.hadoop.mapreduce.Job;
 import org.apache.hcatalog.HcatTestUtils;
 import org.apache.hcatalog.common.HCatConstants;
 import org.apache.hcatalog.data.Pair;
+import org.apache.hive.hcatalog.common.HCatUtil;
 import org.apache.pig.ExecType;
 import org.apache.pig.PigServer;
 import org.apache.pig.ResourceStatistics;
@@ -60,8 +61,9 @@ import org.junit.Test;
  * @deprecated Use/modify {@link org.apache.hive.hcatalog.pig.TestHCatLoader} instead
  */
 public class TestHCatLoader {
-  private static final String TEST_DATA_DIR = System.getProperty("java.io.tmpdir") + File.separator
-      + TestHCatLoader.class.getCanonicalName() + "-" + System.currentTimeMillis();
+  private static final String TEST_DATA_DIR = HCatUtil.makePathASafeFileName(
+          System.getProperty("java.io.tmpdir") + File.separator + TestHCatLoader.class.getCanonicalName() + "-" +
+                  System.currentTimeMillis());
   private static final String TEST_WAREHOUSE_DIR = TEST_DATA_DIR + "/warehouse";
   private static final String BASIC_FILE_NAME = TEST_DATA_DIR + "/basic.input.data";
   private static final String COMPLEX_FILE_NAME = TEST_DATA_DIR + "/complex.input.data";
@@ -409,8 +411,8 @@ public class TestHCatLoader {
     assertEquals(0, driver.run("drop table if exists " + tbl).getResponseCode());
     assertEquals(0, driver.run("create external table " + tbl +
       " (a string, b boolean) row format delimited fields terminated by '\t'" +
-      " stored as textfile location 'file://" +
-      inputDataDir.getAbsolutePath() + "'").getResponseCode());
+      " stored as textfile location 'file:///" +
+      inputDataDir.getPath().replaceAll("\\\\", "/") + "'").getResponseCode());
 
     Properties properties = new Properties();
     properties.setProperty(HCatConstants.HCAT_DATA_CONVERT_BOOLEAN_TO_INTEGER, "true");

Modified: hive/branches/tez/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hcatalog/pig/TestHCatLoaderStorer.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hcatalog/pig/TestHCatLoaderStorer.java?rev=1552848&r1=1552847&r2=1552848&view=diff
==============================================================================
--- hive/branches/tez/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hcatalog/pig/TestHCatLoaderStorer.java (original)
+++ hive/branches/tez/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hcatalog/pig/TestHCatLoaderStorer.java Sat Dec 21 00:15:47 2013
@@ -69,7 +69,7 @@ public class TestHCatLoaderStorer extend
       " (my_small_int smallint, my_tiny_int tinyint)" +
       " row format delimited fields terminated by '\t' stored as textfile").getResponseCode());
     Assert.assertEquals(0, driver.run("load data local inpath '" +
-      dataDir.getAbsolutePath() + "' into table " + readTblName).getResponseCode());
+      dataDir.getPath().replaceAll("\\\\", "/") + "' into table " + readTblName).getResponseCode());
 
     PigServer server = new PigServer(ExecType.LOCAL);
     server.registerQuery(
@@ -104,7 +104,7 @@ public class TestHCatLoaderStorer extend
       String.format("%d\t%d", Short.MIN_VALUE, Byte.MIN_VALUE),
       String.format("%d\t%d", Short.MAX_VALUE, Byte.MAX_VALUE)
     });
-    smallTinyIntBoundsCheckHelper(writeDataFile.getAbsolutePath(), ExecJob.JOB_STATUS.COMPLETED);
+    smallTinyIntBoundsCheckHelper(writeDataFile.getPath().replaceAll("\\\\", "/"), ExecJob.JOB_STATUS.COMPLETED);
 
     // Values outside the column type bounds will fail at runtime.
     HcatTestUtils.createTestDataFile(TEST_DATA_DIR + "/shortTooSmall.tsv", new String[]{

Modified: hive/branches/tez/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hcatalog/pig/TestHCatStorerMulti.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hcatalog/pig/TestHCatStorerMulti.java?rev=1552848&r1=1552847&r2=1552848&view=diff
==============================================================================
--- hive/branches/tez/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hcatalog/pig/TestHCatStorerMulti.java (original)
+++ hive/branches/tez/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hcatalog/pig/TestHCatStorerMulti.java Sat Dec 21 00:15:47 2013
@@ -41,8 +41,7 @@ import org.apache.pig.PigServer;
  * @deprecated Use/modify {@link org.apache.hive.hcatalog.pig.TestHCatStorerMulti} instead
  */
 public class TestHCatStorerMulti extends TestCase {
-  private static final String TEST_DATA_DIR = System.getProperty("user.dir") +
-    "/build/test/data/" + TestHCatStorerMulti.class.getCanonicalName();
+  private static final String TEST_DATA_DIR = org.apache.hive.hcatalog.pig.TestHCatStorerMulti.TEST_DATA_DIR;
   private static final String TEST_WAREHOUSE_DIR = TEST_DATA_DIR + "/warehouse";
   private static final String INPUT_FILE_NAME = TEST_DATA_DIR + "/input.data";
 

Modified: hive/branches/tez/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hcatalog/pig/TestHCatStorerWrapper.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hcatalog/pig/TestHCatStorerWrapper.java?rev=1552848&r1=1552847&r2=1552848&view=diff
==============================================================================
--- hive/branches/tez/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hcatalog/pig/TestHCatStorerWrapper.java (original)
+++ hive/branches/tez/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hcatalog/pig/TestHCatStorerWrapper.java Sat Dec 21 00:15:47 2013
@@ -48,7 +48,7 @@ public class TestHCatStorerWrapper exten
   @Test
   public void testStoreExternalTableWithExternalDir() throws IOException, CommandNeedRetryException{
 
-    File tmpExternalDir = new File(SystemUtils.getJavaIoTmpDir(), UUID.randomUUID().toString());
+    File tmpExternalDir = new File(TEST_DATA_DIR, UUID.randomUUID().toString());
     tmpExternalDir.deleteOnExit();
 
     String part_val = "100";
@@ -71,11 +71,11 @@ public class TestHCatStorerWrapper exten
     server.setBatchOn();
     logAndRegister(server, "A = load '"+INPUT_FILE_NAME+"' as (a:int, b:chararray);");
     logAndRegister(server, "store A into 'default.junit_external' using " + HCatStorerWrapper.class.getName()
-        + "('c=" + part_val + "','" + tmpExternalDir.getAbsolutePath() + "');");
+        + "('c=" + part_val + "','" + tmpExternalDir.getPath().replaceAll("\\\\", "/") + "');");
     server.executeBatch();
 
     Assert.assertTrue(tmpExternalDir.exists());
-    Assert.assertTrue(new File(tmpExternalDir.getAbsoluteFile() + "/" + "part-m-00000").exists());
+    Assert.assertTrue(new File(tmpExternalDir.getPath().replaceAll("\\\\", "/") + "/" + "part-m-00000").exists());
 
     driver.run("select * from junit_external");
     ArrayList<String> res = new ArrayList<String>();

Modified: hive/branches/tez/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoader.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoader.java?rev=1552848&r1=1552847&r2=1552848&view=diff
==============================================================================
--- hive/branches/tez/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoader.java (original)
+++ hive/branches/tez/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoader.java Sat Dec 21 00:15:47 2013
@@ -43,6 +43,7 @@ import org.apache.hadoop.hive.ql.Driver;
 import org.apache.hadoop.hive.ql.session.SessionState;
 import org.apache.hadoop.mapreduce.Job;
 import org.apache.hive.hcatalog.HcatTestUtils;
+import org.apache.hive.hcatalog.common.HCatUtil;
 import org.apache.hive.hcatalog.common.HCatConstants;
 import org.apache.hive.hcatalog.data.Pair;
 import org.apache.pig.ExecType;
@@ -57,8 +58,8 @@ import org.junit.Before;
 import org.junit.Test;
 
 public class TestHCatLoader {
-  private static final String TEST_DATA_DIR = System.getProperty("java.io.tmpdir") + File.separator
-      + TestHCatLoader.class.getCanonicalName() + "-" + System.currentTimeMillis();
+  private static final String TEST_DATA_DIR = HCatUtil.makePathASafeFileName(System.getProperty("java.io.tmpdir") +
+          File.separator + TestHCatLoader.class.getCanonicalName() + "-" + System.currentTimeMillis());
   private static final String TEST_WAREHOUSE_DIR = TEST_DATA_DIR + "/warehouse";
   private static final String BASIC_FILE_NAME = TEST_DATA_DIR + "/basic.input.data";
   private static final String COMPLEX_FILE_NAME = TEST_DATA_DIR + "/complex.input.data";
@@ -420,7 +421,7 @@ public class TestHCatLoader {
     assertEquals(0, driver.run("drop table if exists " + tbl).getResponseCode());
     assertEquals(0, driver.run("create external table " + tbl +
       " (a string, b boolean) row format delimited fields terminated by '\t'" +
-      " stored as textfile location 'file://" +
+      " stored as textfile location 'file:///" +
       inputDataDir.getPath().replaceAll("\\\\", "/") + "'").getResponseCode());
 
     Properties properties = new Properties();

Modified: hive/branches/tez/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatStorerMulti.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatStorerMulti.java?rev=1552848&r1=1552847&r2=1552848&view=diff
==============================================================================
--- hive/branches/tez/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatStorerMulti.java (original)
+++ hive/branches/tez/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatStorerMulti.java Sat Dec 21 00:15:47 2013
@@ -33,13 +33,15 @@ import org.apache.hadoop.hive.conf.HiveC
 import org.apache.hadoop.hive.ql.CommandNeedRetryException;
 import org.apache.hadoop.hive.ql.Driver;
 import org.apache.hadoop.hive.ql.session.SessionState;
+import org.apache.hive.hcatalog.common.HCatUtil;
 import org.apache.hive.hcatalog.data.Pair;
 import org.apache.pig.ExecType;
 import org.apache.pig.PigServer;
 
 public class TestHCatStorerMulti extends TestCase {
-  private static final String TEST_DATA_DIR =
-    "/tmp/build/test/data/" + TestHCatStorerMulti.class.getCanonicalName();
+  public static final String TEST_DATA_DIR = HCatUtil.makePathASafeFileName(
+          System.getProperty("user.dir") + "/build/test/data/" +
+                  TestHCatStorerMulti.class.getCanonicalName() + "-" + System.currentTimeMillis());
   private static final String TEST_WAREHOUSE_DIR = TEST_DATA_DIR + "/warehouse";
   private static final String INPUT_FILE_NAME = TEST_DATA_DIR + "/input.data";
 

Modified: hive/branches/tez/hcatalog/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/SkeletonHBaseTest.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/hcatalog/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/SkeletonHBaseTest.java?rev=1552848&r1=1552847&r2=1552848&view=diff
==============================================================================
--- hive/branches/tez/hcatalog/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/SkeletonHBaseTest.java (original)
+++ hive/branches/tez/hcatalog/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/SkeletonHBaseTest.java Sat Dec 21 00:15:47 2013
@@ -165,12 +165,8 @@ public abstract class SkeletonHBaseTest 
     protected int usageCount = 0;
 
     public Context(String handle) {
-      try {
-        testDir = new File(TEST_DIR + "/test_" + handle + "_" + Math.abs(new Random().nextLong()) + "/").getCanonicalPath();
-        System.out.println("Cluster work directory: " + testDir);
-      } catch (IOException e) {
-        throw new IllegalStateException("Failed to generate testDir", e);
-      }
+      testDir = new File(TEST_DIR + "/test_" + handle + "_" + Math.abs(new Random().nextLong()) + "/").getPath();
+      System.out.println("Cluster work directory: " + testDir);
     }
 
     public void start() {

Modified: hive/branches/tez/hcatalog/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/TestHCatHBaseInputFormat.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/hcatalog/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/TestHCatHBaseInputFormat.java?rev=1552848&r1=1552847&r2=1552848&view=diff
==============================================================================
--- hive/branches/tez/hcatalog/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/TestHCatHBaseInputFormat.java (original)
+++ hive/branches/tez/hcatalog/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/TestHCatHBaseInputFormat.java Sat Dec 21 00:15:47 2013
@@ -64,7 +64,7 @@ import org.apache.hcatalog.cli.HCatDrive
 import org.apache.hcatalog.cli.SemanticAnalysis.HCatSemanticAnalyzer;
 import org.apache.hcatalog.common.HCatConstants;
 import org.apache.hcatalog.common.HCatException;
-import org.apache.hcatalog.common.HCatUtil;
+import org.apache.hive.hcatalog.common.HCatUtil;
 import org.apache.hcatalog.data.HCatRecord;
 import org.apache.hcatalog.data.schema.HCatFieldSchema;
 import org.apache.hcatalog.data.schema.HCatSchema;
@@ -188,7 +188,7 @@ public class TestHCatHBaseInputFormat ex
     String databaseName = newTableName("MyDatabase");
     //Table name will be lower case unless specified by hbase.table.name property
     String hbaseTableName = (databaseName + "." + tableName).toLowerCase();
-    String db_dir = getTestDir() + "/hbasedb";
+    String db_dir = HCatUtil.makePathASafeFileName(getTestDir() + "/hbasedb");
 
     String dbquery = "CREATE DATABASE IF NOT EXISTS " + databaseName + " LOCATION '"
         + db_dir + "'";

Modified: hive/branches/tez/hcatalog/storage-handlers/hbase/src/test/org/apache/hive/hcatalog/hbase/SkeletonHBaseTest.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/hcatalog/storage-handlers/hbase/src/test/org/apache/hive/hcatalog/hbase/SkeletonHBaseTest.java?rev=1552848&r1=1552847&r2=1552848&view=diff
==============================================================================
--- hive/branches/tez/hcatalog/storage-handlers/hbase/src/test/org/apache/hive/hcatalog/hbase/SkeletonHBaseTest.java (original)
+++ hive/branches/tez/hcatalog/storage-handlers/hbase/src/test/org/apache/hive/hcatalog/hbase/SkeletonHBaseTest.java Sat Dec 21 00:15:47 2013
@@ -172,12 +172,8 @@ public abstract class SkeletonHBaseTest 
     protected int usageCount = 0;
 
     public Context(String handle) {
-      try {
-        testDir = new File(TEST_DIR + "/test_" + handle + "_" + Math.abs(new Random().nextLong()) + "/").getCanonicalPath();
-        System.out.println("Cluster work directory: " + testDir);
-      } catch (IOException e) {
-        throw new IllegalStateException("Failed to generate testDir", e);
-      }
+      testDir = new File(TEST_DIR + "/test_" + handle + "_" + Math.abs(new Random().nextLong()) + "/").getPath();
+      System.out.println("Cluster work directory: " + testDir);
     }
 
     public void start() {

Modified: hive/branches/tez/hcatalog/webhcat/java-client/src/test/java/org/apache/hcatalog/api/TestHCatClient.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/hcatalog/webhcat/java-client/src/test/java/org/apache/hcatalog/api/TestHCatClient.java?rev=1552848&r1=1552847&r2=1552848&view=diff
==============================================================================
--- hive/branches/tez/hcatalog/webhcat/java-client/src/test/java/org/apache/hcatalog/api/TestHCatClient.java (original)
+++ hive/branches/tez/hcatalog/webhcat/java-client/src/test/java/org/apache/hcatalog/api/TestHCatClient.java Sat Dec 21 00:15:47 2013
@@ -128,7 +128,8 @@ public class TestHCatClient {
     assertTrue(testDb.getProperties().size() == 0);
     String warehouseDir = System
       .getProperty("test.warehouse.dir", "/user/hive/warehouse");
-    String expectedDir = warehouseDir.replaceAll("\\\\", "/").replaceFirst("pfile:///", "pfile:/");
+    String expectedDir = org.apache.hive.hcatalog.api.TestHCatClient.fixPath(warehouseDir).
+            replaceFirst("pfile:///", "pfile:/");
     assertEquals(expectedDir + "/" + db + ".db", testDb.getLocation());
     ArrayList<HCatFieldSchema> cols = new ArrayList<HCatFieldSchema>();
     cols.add(new HCatFieldSchema("id", Type.INT, "id comment"));

Modified: hive/branches/tez/hcatalog/webhcat/java-client/src/test/java/org/apache/hive/hcatalog/api/TestHCatClient.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/hcatalog/webhcat/java-client/src/test/java/org/apache/hive/hcatalog/api/TestHCatClient.java?rev=1552848&r1=1552847&r2=1552848&view=diff
==============================================================================
--- hive/branches/tez/hcatalog/webhcat/java-client/src/test/java/org/apache/hive/hcatalog/api/TestHCatClient.java (original)
+++ hive/branches/tez/hcatalog/webhcat/java-client/src/test/java/org/apache/hive/hcatalog/api/TestHCatClient.java Sat Dec 21 00:15:47 2013
@@ -53,6 +53,8 @@ import static org.junit.Assert.assertFal
 import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.assertArrayEquals;
 
+import org.apache.hadoop.util.Shell;
+
 public class TestHCatClient {
   private static final Logger LOG = LoggerFactory.getLogger(TestHCatClient.class);
   private static final String msPort = "20101";
@@ -100,7 +102,16 @@ public class TestHCatClient {
     System.setProperty(HiveConf.ConfVars.PREEXECHOOKS.varname, " ");
     System.setProperty(HiveConf.ConfVars.POSTEXECHOOKS.varname, " ");
   }
-
+  public static String fixPath(String path) {
+    if(!Shell.WINDOWS) {
+      return path;
+    }
+    String expectedDir = path.replaceAll("\\\\", "/");
+    if (!expectedDir.startsWith("/")) {
+      expectedDir = "/" + expectedDir;
+    }
+    return expectedDir;
+  }
   @Test
   public void testBasicDDLCommands() throws Exception {
     String db = "testdb";
@@ -121,7 +132,7 @@ public class TestHCatClient {
     assertTrue(testDb.getProperties().size() == 0);
     String warehouseDir = System
       .getProperty("test.warehouse.dir", "/user/hive/warehouse");
-    String expectedDir = warehouseDir.replaceAll("\\\\", "/").replaceFirst("pfile:///", "pfile:/");
+    String expectedDir = fixPath(warehouseDir).replaceFirst("pfile:///", "pfile:/");
     assertEquals(expectedDir + "/" + db + ".db", testDb.getLocation());
     ArrayList<HCatFieldSchema> cols = new ArrayList<HCatFieldSchema>();
     cols.add(new HCatFieldSchema("id", Type.INT, "id comment"));

Modified: hive/branches/tez/itests/hcatalog-unit/src/test/java/org/apache/hcatalog/mapreduce/TestSequenceFileReadWrite.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/itests/hcatalog-unit/src/test/java/org/apache/hcatalog/mapreduce/TestSequenceFileReadWrite.java?rev=1552848&r1=1552847&r2=1552848&view=diff
==============================================================================
--- hive/branches/tez/itests/hcatalog-unit/src/test/java/org/apache/hcatalog/mapreduce/TestSequenceFileReadWrite.java (original)
+++ hive/branches/tez/itests/hcatalog-unit/src/test/java/org/apache/hcatalog/mapreduce/TestSequenceFileReadWrite.java Sat Dec 21 00:15:47 2013
@@ -43,7 +43,7 @@ import org.apache.hadoop.mapreduce.lib.i
 import org.apache.hcatalog.HcatTestUtils;
 import org.apache.hcatalog.common.HCatConstants;
 import org.apache.hcatalog.common.HCatException;
-import org.apache.hcatalog.common.HCatUtil;
+import org.apache.hive.hcatalog.common.HCatUtil;
 import org.apache.hcatalog.data.DefaultHCatRecord;
 import org.apache.hcatalog.data.schema.HCatFieldSchema;
 import org.apache.hcatalog.data.schema.HCatSchema;
@@ -72,8 +72,8 @@ public class TestSequenceFileReadWrite {
     dataDir = new File(System.getProperty("java.io.tmpdir") + File.separator +
         TestSequenceFileReadWrite.class.getCanonicalName() + "-" + System.currentTimeMillis());
     hiveConf = new HiveConf(this.getClass());
-    warehouseDir = new File(dataDir, "warehouse").getAbsolutePath();
-    inputFileName = new File(dataDir, "input.data").getAbsolutePath();
+    warehouseDir = HCatUtil.makePathASafeFileName(dataDir + File.separator + "warehouse");
+    inputFileName = HCatUtil.makePathASafeFileName(dataDir + File.separator + "input.data");
     hiveConf.set(HiveConf.ConfVars.PREEXECHOOKS.varname, "");
     hiveConf.set(HiveConf.ConfVars.POSTEXECHOOKS.varname, "");
     hiveConf.set(HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY.varname, "false");

Modified: hive/branches/tez/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/hbase/TestPigHBaseStorageHandler.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/hbase/TestPigHBaseStorageHandler.java?rev=1552848&r1=1552847&r2=1552848&view=diff
==============================================================================
--- hive/branches/tez/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/hbase/TestPigHBaseStorageHandler.java (original)
+++ hive/branches/tez/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/hbase/TestPigHBaseStorageHandler.java Sat Dec 21 00:15:47 2013
@@ -47,6 +47,7 @@ import org.apache.hadoop.hive.conf.HiveC
 import org.apache.hadoop.hive.ql.Driver;
 import org.apache.hadoop.hive.ql.processors.CommandProcessorResponse;
 import org.apache.hadoop.hive.ql.session.SessionState;
+import org.apache.hive.hcatalog.common.HCatUtil;
 import org.apache.pig.ExecType;
 import org.apache.pig.PigServer;
 import org.apache.pig.data.DataType;
@@ -138,7 +139,7 @@ public class TestPigHBaseStorageHandler 
     String databaseName = newTableName("MyDatabase");
     //Table name will be lower case unless specified by hbase.table.name property
     String hbaseTableName = "testTable";
-    String db_dir = getTestDir() + "/hbasedb";
+    String db_dir = HCatUtil.makePathASafeFileName(getTestDir() + "/hbasedb");
 
     String dbQuery = "CREATE DATABASE IF NOT EXISTS " + databaseName + " LOCATION '"
         + db_dir + "'";
@@ -193,7 +194,7 @@ public class TestPigHBaseStorageHandler 
     String databaseName = newTableName("MyDatabase");
     //Table name will be lower case unless specified by hbase.table.name property
     String hbaseTableName = (databaseName + "." + tableName).toLowerCase();
-    String db_dir = getTestDir() + "/hbasedb";
+    String db_dir = HCatUtil.makePathASafeFileName(getTestDir() + "/hbasedb");
 
     String dbQuery = "CREATE DATABASE IF NOT EXISTS " + databaseName + " LOCATION '"
         + db_dir + "'";
@@ -255,7 +256,7 @@ public class TestPigHBaseStorageHandler 
     String databaseName = newTableName("MyDatabase");
     //Table name will be lower case unless specified by hbase.table.name property
     String hbaseTableName = (databaseName + "." + tableName).toLowerCase();
-    String db_dir = getTestDir() + "/hbasedb";
+    String db_dir = HCatUtil.makePathASafeFileName(getTestDir() + "/hbasedb");
     String POPTXT_FILE_NAME = db_dir+"testfile.txt";
     float f = -100.1f;
 

Modified: hive/branches/tez/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/mapreduce/TestSequenceFileReadWrite.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/mapreduce/TestSequenceFileReadWrite.java?rev=1552848&r1=1552847&r2=1552848&view=diff
==============================================================================
--- hive/branches/tez/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/mapreduce/TestSequenceFileReadWrite.java (original)
+++ hive/branches/tez/itests/hcatalog-unit/src/test/java/org/apache/hive/hcatalog/mapreduce/TestSequenceFileReadWrite.java Sat Dec 21 00:15:47 2013
@@ -69,8 +69,8 @@ public class TestSequenceFileReadWrite {
     dataDir = new File(System.getProperty("java.io.tmpdir") + File.separator +
         TestSequenceFileReadWrite.class.getCanonicalName() + "-" + System.currentTimeMillis());
     hiveConf = new HiveConf(this.getClass());
-    warehouseDir = new File(dataDir, "warehouse").getAbsolutePath();
-    inputFileName = new File(dataDir, "input.data").getAbsolutePath();
+    warehouseDir = HCatUtil.makePathASafeFileName(dataDir + File.separator + "warehouse");
+    inputFileName = HCatUtil.makePathASafeFileName(dataDir + File.separator + "input.data");
     hiveConf = new HiveConf(this.getClass());
     hiveConf.set(HiveConf.ConfVars.PREEXECHOOKS.varname, "");
     hiveConf.set(HiveConf.ConfVars.POSTEXECHOOKS.varname, "");

Modified: hive/branches/tez/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java?rev=1552848&r1=1552847&r2=1552848&view=diff
==============================================================================
--- hive/branches/tez/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java (original)
+++ hive/branches/tez/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java Sat Dec 21 00:15:47 2013
@@ -31,6 +31,7 @@ import java.io.FileReader;
 import java.io.FileWriter;
 import java.io.IOException;
 import java.io.InputStreamReader;
+import java.io.OutputStreamWriter;
 import java.io.PrintStream;
 import java.io.Serializable;
 import java.io.StringWriter;
@@ -48,6 +49,7 @@ import java.util.TreeMap;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
+import org.apache.commons.io.FileUtils;
 import org.apache.commons.io.IOUtils;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
@@ -1150,17 +1152,12 @@ public class QTestUtil {
     BufferedReader in;
     BufferedWriter out;
 
-    in = new BufferedReader(new FileReader(fname));
-    out = new BufferedWriter(new FileWriter(fname + ".orig"));
-    while (null != (line = in.readLine())) {
-      out.write(line);
-      out.write('\n');
-    }
-    in.close();
-    out.close();
+    File file = new File(fname);
+    File fileOrig = new File(fname + ".orig");
+    FileUtils.copyFile(file, fileOrig);
 
-    in = new BufferedReader(new FileReader(fname + ".orig"));
-    out = new BufferedWriter(new FileWriter(fname));
+    in = new BufferedReader(new InputStreamReader(new FileInputStream(fileOrig), "UTF-8"));
+    out = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(file), "UTF-8"));
 
     boolean lastWasMasked = false;
     while (null != (line = in.readLine())) {
@@ -1413,9 +1410,12 @@ public class QTestUtil {
     public void preTest(HiveConf conf) throws Exception {
 
       if (zooKeeperCluster == null) {
-        String tmpdir =  System.getProperty("test.tmp.dir");
+        //create temp dir
+        String tmpBaseDir =  System.getProperty("test.tmp.dir");
+        File tmpDir = Utilities.createTempDir(tmpBaseDir);
+
         zooKeeperCluster = new MiniZooKeeperCluster();
-        zkPort = zooKeeperCluster.startup(new File(tmpdir, "zookeeper"));
+        zkPort = zooKeeperCluster.startup(tmpDir);
       }
 
       if (zooKeeper != null) {

Modified: hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java?rev=1552848&r1=1552847&r2=1552848&view=diff
==============================================================================
--- hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java (original)
+++ hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java Sat Dec 21 00:15:47 2013
@@ -3166,5 +3166,30 @@ public final class Utilities {
   public static void clearWorkMap() {
     gWorkMap.clear();
   }
-}
 
+  /**
+   * Create a temp dir in specified baseDir
+   * This can go away once hive moves to support only JDK 7
+   *  and can use Files.createTempDirectory
+   *  Guava Files.createTempDir() does not take a base dir
+   * @param baseDir - directory under which new temp dir will be created
+   * @return File object for new temp dir
+   */
+  public static File createTempDir(String baseDir){
+    //try creating the temp dir MAX_ATTEMPTS times
+    final int MAX_ATTEMPS = 30;
+    for(int i = 0; i < MAX_ATTEMPS; i++){
+      //pick a random file name
+      String tempDirName = "tmp_" + ((int)(100000 * Math.random()));
+
+      //return if dir could successfully be created with that file name
+      File tempDir = new File(baseDir, tempDirName);
+      if(tempDir.mkdir()){
+        return tempDir;
+      }
+    }
+    throw new IllegalStateException("Failed to create a temp dir under "
+    + baseDir + " Giving up after " + MAX_ATTEMPS + " attemps");
+
+  }
+}

Modified: hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/metadata/formatting/TextMetaDataFormatter.java
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/metadata/formatting/TextMetaDataFormatter.java?rev=1552848&r1=1552847&r2=1552848&view=diff
==============================================================================
--- hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/metadata/formatting/TextMetaDataFormatter.java (original)
+++ hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/metadata/formatting/TextMetaDataFormatter.java Sat Dec 21 00:15:47 2013
@@ -130,7 +130,7 @@ class TextMetaDataFormatter implements M
           } else {
             output = MetaDataFormatUtils.getAllColumnsInformation(cols, isFormatted);
           }
-          outStream.write(output.getBytes());
+          outStream.write(output.getBytes("UTF-8"));
 
           if (tableName.equals(colPath)) {
             if (isFormatted) {
@@ -139,7 +139,7 @@ class TextMetaDataFormatter implements M
               } else {
                 output = MetaDataFormatUtils.getTableInformation(tbl);
               }
-              outStream.write(output.getBytes());
+              outStream.write(output.getBytes("UTF-8"));
             }
 
           // if extended desc table then show the complete details of the table
@@ -150,7 +150,7 @@ class TextMetaDataFormatter implements M
                 // show partition information
                 outStream.writeBytes("Detailed Partition Information");
                 outStream.write(separator);
-                outStream.write(part.getTPartition().toString().getBytes());
+                outStream.write(part.getTPartition().toString().getBytes("UTF-8"));
                 outStream.write(separator);
                 // comment column is empty
                 outStream.write(terminator);
@@ -158,7 +158,7 @@ class TextMetaDataFormatter implements M
                 // show table information
                 outStream.writeBytes("Detailed Table Information");
                 outStream.write(separator);
-                outStream.write(tbl.getTTable().toString().getBytes());
+                outStream.write(tbl.getTTable().toString().getBytes("UTF-8"));
                 outStream.write(separator);
                 outStream.write(terminator);
               }
@@ -443,7 +443,7 @@ class TextMetaDataFormatter implements M
             outStream.writeBytes(database);
             outStream.write(separator);
             if (comment != null) {
-              outStream.write(comment.getBytes());
+              outStream.write(comment.getBytes("UTF-8"));
             }
             outStream.write(separator);
             if (location != null) {

Modified: hive/branches/tez/ql/src/test/queries/clientpositive/input20.q
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/test/queries/clientpositive/input20.q?rev=1552848&r1=1552847&r2=1552848&view=diff
==============================================================================
--- hive/branches/tez/ql/src/test/queries/clientpositive/input20.q (original)
+++ hive/branches/tez/ql/src/test/queries/clientpositive/input20.q Sat Dec 21 00:15:47 2013
@@ -1,6 +1,6 @@
 CREATE TABLE dest1(key INT, value STRING) STORED AS TEXTFILE;
 
-ADD FILE ../../data/scripts/input20_script;
+ADD FILE ../../data/scripts/input20_script.py;
 
 EXPLAIN
 FROM (
@@ -12,7 +12,7 @@ FROM (
 ) tmap
 INSERT OVERWRITE TABLE dest1
 REDUCE tmap.key, tmap.value
-USING 'input20_script'
+USING 'python input20_script.py'
 AS key, value;
 
 FROM (
@@ -24,7 +24,7 @@ FROM (
 ) tmap
 INSERT OVERWRITE TABLE dest1
 REDUCE tmap.key, tmap.value
-USING 'input20_script'
+USING 'python input20_script.py'
 AS key, value;
 
 SELECT * FROM dest1 SORT BY key, value;

Modified: hive/branches/tez/ql/src/test/queries/clientpositive/input33.q
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/test/queries/clientpositive/input33.q?rev=1552848&r1=1552847&r2=1552848&view=diff
==============================================================================
--- hive/branches/tez/ql/src/test/queries/clientpositive/input33.q (original)
+++ hive/branches/tez/ql/src/test/queries/clientpositive/input33.q Sat Dec 21 00:15:47 2013
@@ -1,6 +1,6 @@
 CREATE TABLE dest1(key INT, value STRING) STORED AS TEXTFILE;
 
-ADD FILE ../../data/scripts/input20_script;
+ADD FILE ../../data/scripts/input20_script.py;
 
 EXPLAIN
 FROM (
@@ -12,7 +12,7 @@ FROM (
 ) tmap
 INSERT OVERWRITE TABLE dest1
 REDUCE tmap.key, tmap.value
-USING 'input20_script'
+USING 'python input20_script.py'
 AS (key STRING, value STRING);
 
 FROM (
@@ -24,7 +24,7 @@ FROM (
 ) tmap
 INSERT OVERWRITE TABLE dest1
 REDUCE tmap.key, tmap.value
-USING 'input20_script'
+USING 'python input20_script.py'
 AS (key STRING, value STRING);
 
 SELECT * FROM dest1 SORT BY key, value;

Modified: hive/branches/tez/ql/src/test/results/beelinepositive/input20.q.out
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/test/results/beelinepositive/input20.q.out?rev=1552848&r1=1552847&r2=1552848&view=diff
==============================================================================
--- hive/branches/tez/ql/src/test/results/beelinepositive/input20.q.out (original)
+++ hive/branches/tez/ql/src/test/results/beelinepositive/input20.q.out Sat Dec 21 00:15:47 2013
@@ -3,7 +3,7 @@ Saving all output to "!!{outputDirectory
 >>>  CREATE TABLE dest1(key INT, value STRING) STORED AS TEXTFILE;
 No rows affected 
 >>>  
->>>  ADD FILE ../data/scripts/input20_script;
+>>>  ADD FILE ../data/scripts/input20_script.py;
 No rows affected 
 >>>  
 >>>  EXPLAIN 
@@ -16,11 +16,11 @@ SORT BY key, value 
 ) tmap 
 INSERT OVERWRITE TABLE dest1 
 REDUCE tmap.key, tmap.value 
-USING 'input20_script' 
+USING 'python input20_script.py' 
 AS key, value;
 'Explain'
 'ABSTRACT SYNTAX TREE:'
-'  (TOK_QUERY (TOK_FROM (TOK_SUBQUERY (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME src))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_TRANSFORM (TOK_EXPLIST (. (TOK_TABLE_OR_COL src) key) (. (TOK_TABLE_OR_COL src) key)) TOK_SERDE TOK_RECORDWRITER 'cat' TOK_SERDE TOK_RECORDREADER))) (TOK_DISTRIBUTEBY (TOK_TABLE_OR_COL key)) (TOK_SORTBY (TOK_TABSORTCOLNAMEASC (TOK_TABLE_OR_COL key)) (TOK_TABSORTCOLNAMEASC (TOK_TABLE_OR_COL value))))) tmap)) (TOK_INSERT (TOK_DESTINATION (TOK_TAB (TOK_TABNAME dest1))) (TOK_SELECT (TOK_SELEXPR (TOK_TRANSFORM (TOK_EXPLIST (. (TOK_TABLE_OR_COL tmap) key) (. (TOK_TABLE_OR_COL tmap) value)) TOK_SERDE TOK_RECORDWRITER 'input20_script' TOK_SERDE TOK_RECORDREADER (TOK_ALIASLIST key value))))))'
+'  (TOK_QUERY (TOK_FROM (TOK_SUBQUERY (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME src))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_TRANSFORM (TOK_EXPLIST (. (TOK_TABLE_OR_COL src) key) (. (TOK_TABLE_OR_COL src) key)) TOK_SERDE TOK_RECORDWRITER 'cat' TOK_SERDE TOK_RECORDREADER))) (TOK_DISTRIBUTEBY (TOK_TABLE_OR_COL key)) (TOK_SORTBY (TOK_TABSORTCOLNAMEASC (TOK_TABLE_OR_COL key)) (TOK_TABSORTCOLNAMEASC (TOK_TABLE_OR_COL value))))) tmap)) (TOK_INSERT (TOK_DESTINATION (TOK_TAB (TOK_TABNAME dest1))) (TOK_SELECT (TOK_SELEXPR (TOK_TRANSFORM (TOK_EXPLIST (. (TOK_TABLE_OR_COL tmap) key) (. (TOK_TABLE_OR_COL tmap) value)) TOK_SERDE TOK_RECORDWRITER 'python input20_script.py' TOK_SERDE TOK_RECORDREADER (TOK_ALIASLIST key value))))))'
 ''
 'STAGE DEPENDENCIES:'
 '  Stage-1 is a root stage'
@@ -72,7 +72,7 @@ AS key, value;
 '                  type: string'
 '            outputColumnNames: _col0, _col1'
 '            Transform Operator'
-'              command: input20_script'
+'              command: python input20_script.py'
 '              output info:'
 '                  input format: org.apache.hadoop.mapred.TextInputFormat'
 '                  output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat'
@@ -117,7 +117,7 @@ SORT BY key, value 
 ) tmap 
 INSERT OVERWRITE TABLE dest1 
 REDUCE tmap.key, tmap.value 
-USING 'input20_script' 
+USING 'python input20_script.py' 
 AS key, value;
 '_col0','_col1'
 No rows selected 

Modified: hive/branches/tez/ql/src/test/results/beelinepositive/input33.q.out
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/test/results/beelinepositive/input33.q.out?rev=1552848&r1=1552847&r2=1552848&view=diff
==============================================================================
--- hive/branches/tez/ql/src/test/results/beelinepositive/input33.q.out (original)
+++ hive/branches/tez/ql/src/test/results/beelinepositive/input33.q.out Sat Dec 21 00:15:47 2013
@@ -3,7 +3,7 @@ Saving all output to "!!{outputDirectory
 >>>  CREATE TABLE dest1(key INT, value STRING) STORED AS TEXTFILE;
 No rows affected 
 >>>  
->>>  ADD FILE ../data/scripts/input20_script;
+>>>  ADD FILE ../data/scripts/input20_script.py;
 No rows affected 
 >>>  
 >>>  EXPLAIN 
@@ -16,11 +16,11 @@ SORT BY key, value 
 ) tmap 
 INSERT OVERWRITE TABLE dest1 
 REDUCE tmap.key, tmap.value 
-USING 'input20_script' 
+USING 'python input20_script.py' 
 AS (key STRING, value STRING);
 'Explain'
 'ABSTRACT SYNTAX TREE:'
-'  (TOK_QUERY (TOK_FROM (TOK_SUBQUERY (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME src))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_TRANSFORM (TOK_EXPLIST (. (TOK_TABLE_OR_COL src) key) (. (TOK_TABLE_OR_COL src) key)) TOK_SERDE TOK_RECORDWRITER 'cat' TOK_SERDE TOK_RECORDREADER))) (TOK_DISTRIBUTEBY (TOK_TABLE_OR_COL key)) (TOK_SORTBY (TOK_TABSORTCOLNAMEASC (TOK_TABLE_OR_COL key)) (TOK_TABSORTCOLNAMEASC (TOK_TABLE_OR_COL value))))) tmap)) (TOK_INSERT (TOK_DESTINATION (TOK_TAB (TOK_TABNAME dest1))) (TOK_SELECT (TOK_SELEXPR (TOK_TRANSFORM (TOK_EXPLIST (. (TOK_TABLE_OR_COL tmap) key) (. (TOK_TABLE_OR_COL tmap) value)) TOK_SERDE TOK_RECORDWRITER 'input20_script' TOK_SERDE TOK_RECORDREADER (TOK_TABCOLLIST (TOK_TABCOL key TOK_STRING) (TOK_TABCOL value TOK_STRING)))))))'
+'  (TOK_QUERY (TOK_FROM (TOK_SUBQUERY (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME src))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_TRANSFORM (TOK_EXPLIST (. (TOK_TABLE_OR_COL src) key) (. (TOK_TABLE_OR_COL src) key)) TOK_SERDE TOK_RECORDWRITER 'cat' TOK_SERDE TOK_RECORDREADER))) (TOK_DISTRIBUTEBY (TOK_TABLE_OR_COL key)) (TOK_SORTBY (TOK_TABSORTCOLNAMEASC (TOK_TABLE_OR_COL key)) (TOK_TABSORTCOLNAMEASC (TOK_TABLE_OR_COL value))))) tmap)) (TOK_INSERT (TOK_DESTINATION (TOK_TAB (TOK_TABNAME dest1))) (TOK_SELECT (TOK_SELEXPR (TOK_TRANSFORM (TOK_EXPLIST (. (TOK_TABLE_OR_COL tmap) key) (. (TOK_TABLE_OR_COL tmap) value)) TOK_SERDE TOK_RECORDWRITER 'python input20_script.py' TOK_SERDE TOK_RECORDREADER (TOK_TABCOLLIST (TOK_TABCOL key TOK_STRING) (TOK_TABCOL value TOK_STRING)))))))'
 ''
 'STAGE DEPENDENCIES:'
 '  Stage-1 is a root stage'
@@ -72,7 +72,7 @@ AS (key STRING, value STRING);
 '                  type: string'
 '            outputColumnNames: _col0, _col1'
 '            Transform Operator'
-'              command: input20_script'
+'              command: python input20_script.py'
 '              output info:'
 '                  input format: org.apache.hadoop.mapred.TextInputFormat'
 '                  output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat'
@@ -117,7 +117,7 @@ SORT BY key, value 
 ) tmap 
 INSERT OVERWRITE TABLE dest1 
 REDUCE tmap.key, tmap.value 
-USING 'input20_script' 
+USING 'python input20_script.py' 
 AS (key STRING, value STRING);
 '_col0','_col1'
 No rows selected 

Modified: hive/branches/tez/ql/src/test/results/beelinepositive/newline.q.out
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/test/results/beelinepositive/newline.q.out?rev=1552848&r1=1552847&r2=1552848&view=diff
==============================================================================
--- hive/branches/tez/ql/src/test/results/beelinepositive/newline.q.out (original)
+++ hive/branches/tez/ql/src/test/results/beelinepositive/newline.q.out Sat Dec 21 00:15:47 2013
@@ -141,12 +141,12 @@ No rows selected 
 '1\	2',''
 5 rows selected 
 >>>  
->>>  SELECT key FROM (SELECT TRANSFORM ('a\tb', 'c') USING '/bin/cat' AS (key, value) FROM src limit 1)a ORDER BY key ASC;
+>>>  SELECT key FROM (SELECT TRANSFORM ('a\tb', 'c') USING 'cat' AS (key, value) FROM src limit 1)a ORDER BY key ASC;
 'key'
 'a	b'
 1 row selected 
 >>>  
->>>  SELECT value FROM (SELECT TRANSFORM ('a\tb', 'c') USING '/bin/cat' AS (key, value) FROM src limit 1)a ORDER BY value ASC;
+>>>  SELECT value FROM (SELECT TRANSFORM ('a\tb', 'c') USING 'cat' AS (key, value) FROM src limit 1)a ORDER BY value ASC;
 'value'
 'c'
 1 row selected 

Modified: hive/branches/tez/ql/src/test/results/clientpositive/input20.q.out
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/test/results/clientpositive/input20.q.out?rev=1552848&r1=1552847&r2=1552848&view=diff
==============================================================================
--- hive/branches/tez/ql/src/test/results/clientpositive/input20.q.out (original)
+++ hive/branches/tez/ql/src/test/results/clientpositive/input20.q.out Sat Dec 21 00:15:47 2013
@@ -13,7 +13,7 @@ FROM (
 ) tmap
 INSERT OVERWRITE TABLE dest1
 REDUCE tmap.key, tmap.value
-USING 'input20_script'
+USING 'python input20_script.py'
 AS key, value
 PREHOOK: type: QUERY
 POSTHOOK: query: EXPLAIN
@@ -26,11 +26,11 @@ FROM (
 ) tmap
 INSERT OVERWRITE TABLE dest1
 REDUCE tmap.key, tmap.value
-USING 'input20_script'
+USING 'python input20_script.py'
 AS key, value
 POSTHOOK: type: QUERY
 ABSTRACT SYNTAX TREE:
-  (TOK_QUERY (TOK_FROM (TOK_SUBQUERY (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME src))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_TRANSFORM (TOK_EXPLIST (. (TOK_TABLE_OR_COL src) key) (. (TOK_TABLE_OR_COL src) key)) TOK_SERDE TOK_RECORDWRITER 'cat' TOK_SERDE TOK_RECORDREADER))) (TOK_DISTRIBUTEBY (TOK_TABLE_OR_COL key)) (TOK_SORTBY (TOK_TABSORTCOLNAMEASC (TOK_TABLE_OR_COL key)) (TOK_TABSORTCOLNAMEASC (TOK_TABLE_OR_COL value))))) tmap)) (TOK_INSERT (TOK_DESTINATION (TOK_TAB (TOK_TABNAME dest1))) (TOK_SELECT (TOK_SELEXPR (TOK_TRANSFORM (TOK_EXPLIST (. (TOK_TABLE_OR_COL tmap) key) (. (TOK_TABLE_OR_COL tmap) value)) TOK_SERDE TOK_RECORDWRITER 'input20_script' TOK_SERDE TOK_RECORDREADER (TOK_ALIASLIST key value))))))
+  (TOK_QUERY (TOK_FROM (TOK_SUBQUERY (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME src))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_TRANSFORM (TOK_EXPLIST (. (TOK_TABLE_OR_COL src) key) (. (TOK_TABLE_OR_COL src) key)) TOK_SERDE TOK_RECORDWRITER 'cat' TOK_SERDE TOK_RECORDREADER))) (TOK_DISTRIBUTEBY (TOK_TABLE_OR_COL key)) (TOK_SORTBY (TOK_TABSORTCOLNAMEASC (TOK_TABLE_OR_COL key)) (TOK_TABSORTCOLNAMEASC (TOK_TABLE_OR_COL value))))) tmap)) (TOK_INSERT (TOK_DESTINATION (TOK_TAB (TOK_TABNAME dest1))) (TOK_SELECT (TOK_SELEXPR (TOK_TRANSFORM (TOK_EXPLIST (. (TOK_TABLE_OR_COL tmap) key) (. (TOK_TABLE_OR_COL tmap) value)) TOK_SERDE TOK_RECORDWRITER 'python input20_script.py' TOK_SERDE TOK_RECORDREADER (TOK_ALIASLIST key value))))))
 
 STAGE DEPENDENCIES:
   Stage-1 is a root stage
@@ -83,7 +83,7 @@ STAGE PLANS:
                   type: string
             outputColumnNames: _col0, _col1
             Transform Operator
-              command: input20_script
+              command: python input20_script.py
               output info:
                   input format: org.apache.hadoop.mapred.TextInputFormat
                   output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
@@ -126,7 +126,7 @@ PREHOOK: query: FROM (
 ) tmap
 INSERT OVERWRITE TABLE dest1
 REDUCE tmap.key, tmap.value
-USING 'input20_script'
+USING 'python input20_script.py'
 AS key, value
 PREHOOK: type: QUERY
 PREHOOK: Input: default@src
@@ -140,7 +140,7 @@ POSTHOOK: query: FROM (
 ) tmap
 INSERT OVERWRITE TABLE dest1
 REDUCE tmap.key, tmap.value
-USING 'input20_script'
+USING 'python input20_script.py'
 AS key, value
 POSTHOOK: type: QUERY
 POSTHOOK: Input: default@src

Modified: hive/branches/tez/ql/src/test/results/clientpositive/input33.q.out
URL: http://svn.apache.org/viewvc/hive/branches/tez/ql/src/test/results/clientpositive/input33.q.out?rev=1552848&r1=1552847&r2=1552848&view=diff
==============================================================================
--- hive/branches/tez/ql/src/test/results/clientpositive/input33.q.out (original)
+++ hive/branches/tez/ql/src/test/results/clientpositive/input33.q.out Sat Dec 21 00:15:47 2013
@@ -13,7 +13,7 @@ FROM (
 ) tmap
 INSERT OVERWRITE TABLE dest1
 REDUCE tmap.key, tmap.value
-USING 'input20_script'
+USING 'python input20_script.py'
 AS (key STRING, value STRING)
 PREHOOK: type: QUERY
 POSTHOOK: query: EXPLAIN
@@ -26,11 +26,11 @@ FROM (
 ) tmap
 INSERT OVERWRITE TABLE dest1
 REDUCE tmap.key, tmap.value
-USING 'input20_script'
+USING 'python input20_script.py'
 AS (key STRING, value STRING)
 POSTHOOK: type: QUERY
 ABSTRACT SYNTAX TREE:
-  (TOK_QUERY (TOK_FROM (TOK_SUBQUERY (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME src))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_TRANSFORM (TOK_EXPLIST (. (TOK_TABLE_OR_COL src) key) (. (TOK_TABLE_OR_COL src) key)) TOK_SERDE TOK_RECORDWRITER 'cat' TOK_SERDE TOK_RECORDREADER))) (TOK_DISTRIBUTEBY (TOK_TABLE_OR_COL key)) (TOK_SORTBY (TOK_TABSORTCOLNAMEASC (TOK_TABLE_OR_COL key)) (TOK_TABSORTCOLNAMEASC (TOK_TABLE_OR_COL value))))) tmap)) (TOK_INSERT (TOK_DESTINATION (TOK_TAB (TOK_TABNAME dest1))) (TOK_SELECT (TOK_SELEXPR (TOK_TRANSFORM (TOK_EXPLIST (. (TOK_TABLE_OR_COL tmap) key) (. (TOK_TABLE_OR_COL tmap) value)) TOK_SERDE TOK_RECORDWRITER 'input20_script' TOK_SERDE TOK_RECORDREADER (TOK_TABCOLLIST (TOK_TABCOL key TOK_STRING) (TOK_TABCOL value TOK_STRING)))))))
+  (TOK_QUERY (TOK_FROM (TOK_SUBQUERY (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME src))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_TRANSFORM (TOK_EXPLIST (. (TOK_TABLE_OR_COL src) key) (. (TOK_TABLE_OR_COL src) key)) TOK_SERDE TOK_RECORDWRITER 'cat' TOK_SERDE TOK_RECORDREADER))) (TOK_DISTRIBUTEBY (TOK_TABLE_OR_COL key)) (TOK_SORTBY (TOK_TABSORTCOLNAMEASC (TOK_TABLE_OR_COL key)) (TOK_TABSORTCOLNAMEASC (TOK_TABLE_OR_COL value))))) tmap)) (TOK_INSERT (TOK_DESTINATION (TOK_TAB (TOK_TABNAME dest1))) (TOK_SELECT (TOK_SELEXPR (TOK_TRANSFORM (TOK_EXPLIST (. (TOK_TABLE_OR_COL tmap) key) (. (TOK_TABLE_OR_COL tmap) value)) TOK_SERDE TOK_RECORDWRITER 'python input20_script.py' TOK_SERDE TOK_RECORDREADER (TOK_TABCOLLIST (TOK_TABCOL key TOK_STRING) (TOK_TABCOL value TOK_STRING)))))))
 
 STAGE DEPENDENCIES:
   Stage-1 is a root stage
@@ -83,7 +83,7 @@ STAGE PLANS:
                   type: string
             outputColumnNames: _col0, _col1
             Transform Operator
-              command: input20_script
+              command: python input20_script.py
               output info:
                   input format: org.apache.hadoop.mapred.TextInputFormat
                   output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
@@ -126,7 +126,7 @@ PREHOOK: query: FROM (
 ) tmap
 INSERT OVERWRITE TABLE dest1
 REDUCE tmap.key, tmap.value
-USING 'input20_script'
+USING 'python input20_script.py'
 AS (key STRING, value STRING)
 PREHOOK: type: QUERY
 PREHOOK: Input: default@src
@@ -140,7 +140,7 @@ POSTHOOK: query: FROM (
 ) tmap
 INSERT OVERWRITE TABLE dest1
 REDUCE tmap.key, tmap.value
-USING 'input20_script'
+USING 'python input20_script.py'
 AS (key STRING, value STRING)
 POSTHOOK: type: QUERY
 POSTHOOK: Input: default@src



Mime
View raw message