incubator-hcatalog-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From da...@apache.org
Subject svn commit: r1458574 - in /incubator/hcatalog/trunk: ./ core/ core/src/main/java/org/apache/hcatalog/common/ core/src/main/java/org/apache/hcatalog/mapreduce/ core/src/test/java/org/apache/hcatalog/ core/src/test/java/org/apache/hcatalog/mapreduce/ hca...
Date Tue, 19 Mar 2013 23:42:59 GMT
Author: daijy
Date: Tue Mar 19 23:42:58 2013
New Revision: 1458574

URL: http://svn.apache.org/r1458574
Log:
HCATALOG-624: Fix HCat for hadoop 2 & HCATALOG-637: Upgrade Pig from 0.8 to 0.10

Added:
    incubator/hcatalog/trunk/shims/src/20/java/org/apache/hadoop/
    incubator/hcatalog/trunk/shims/src/20/java/org/apache/hadoop/mapred/
    incubator/hcatalog/trunk/shims/src/20/java/org/apache/hadoop/mapred/TempletonJobTracker.java
    incubator/hcatalog/trunk/shims/src/23/java/org/apache/hadoop/
    incubator/hcatalog/trunk/shims/src/23/java/org/apache/hadoop/mapred/
    incubator/hcatalog/trunk/shims/src/23/java/org/apache/hadoop/mapred/TempletonJobTracker.java
Removed:
    incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hadoop/mapred/TempletonJobTracker.java
Modified:
    incubator/hcatalog/trunk/build.xml
    incubator/hcatalog/trunk/core/pom.xml
    incubator/hcatalog/trunk/core/src/main/java/org/apache/hcatalog/common/HCatUtil.java
    incubator/hcatalog/trunk/core/src/main/java/org/apache/hcatalog/mapreduce/MultiOutputFormat.java
    incubator/hcatalog/trunk/core/src/test/java/org/apache/hcatalog/HcatTestUtils.java
    incubator/hcatalog/trunk/core/src/test/java/org/apache/hcatalog/mapreduce/HCatMapReduceTest.java
    incubator/hcatalog/trunk/core/src/test/java/org/apache/hcatalog/mapreduce/TestHCatDynamicPartitioned.java
    incubator/hcatalog/trunk/core/src/test/java/org/apache/hcatalog/mapreduce/TestHCatMultiOutputFormat.java
    incubator/hcatalog/trunk/core/src/test/java/org/apache/hcatalog/mapreduce/TestHCatPartitionPublish.java
    incubator/hcatalog/trunk/core/src/test/java/org/apache/hcatalog/mapreduce/TestMultiOutputFormat.java
    incubator/hcatalog/trunk/core/src/test/java/org/apache/hcatalog/mapreduce/TestSequenceFileReadWrite.java
    incubator/hcatalog/trunk/hcatalog-pig-adapter/pom.xml
    incubator/hcatalog/trunk/hcatalog-pig-adapter/src/test/java/org/apache/hcatalog/pig/TestHCatLoaderComplexSchema.java
    incubator/hcatalog/trunk/hcatalog-pig-adapter/src/test/java/org/apache/hcatalog/pig/TestPigHCatUtil.java
    incubator/hcatalog/trunk/pom.xml
    incubator/hcatalog/trunk/storage-handlers/hbase/pom.xml
    incubator/hcatalog/trunk/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/HBaseBulkOutputFormat.java
    incubator/hcatalog/trunk/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/ManyMiniCluster.java
    incubator/hcatalog/trunk/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/SkeletonHBaseTest.java
    incubator/hcatalog/trunk/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/TestHBaseBulkOutputFormat.java
    incubator/hcatalog/trunk/webhcat/svr/pom.xml
    incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/DeleteDelegator.java
    incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/ListDelegator.java
    incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/StatusDelegator.java
    incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/TempletonDelegator.java

Modified: incubator/hcatalog/trunk/build.xml
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/build.xml?rev=1458574&r1=1458573&r2=1458574&view=diff
==============================================================================
--- incubator/hcatalog/trunk/build.xml (original)
+++ incubator/hcatalog/trunk/build.xml Tue Mar 19 23:42:58 2013
@@ -69,13 +69,13 @@
           <property name="_mvn.hadoop.profile" value="${mvn.hadoop.profile}"/>
         </ant>
         <ant target="jar" dir="webhcat/svr" inheritAll="false">
-          <property name="_mvn.hadoop.profile" value="hadoop20"/>
+          <property name="_mvn.hadoop.profile" value="${mvn.hadoop.profile}"/>
         </ant>
         <ant target="jar" dir="webhcat/java-client" inheritAll="false">
           <property name="_mvn.hadoop.profile" value="${mvn.hadoop.profile}"/>
         </ant>
         <ant target="jar" dir="storage-handlers/hbase" inheritAll="false">
-          <property name="_mvn.hadoop.profile" value="hadoop20"/>
+          <property name="_mvn.hadoop.profile" value="${mvn.hadoop.profile}"/>
         </ant>
     </target>
 
@@ -192,7 +192,7 @@
             <packageset dir="webhcat/svr/src/main/java"/>
             <packageset dir="webhcat/java-client/src/main/java"/>
             <classpath>
-                <fileset dir="core/build/lib/compile"/>
+                <fileset dir="core/build/lib/test"/>
                 <fileset dir="hcatalog-pig-adapter/build/lib/compile"/>
                 <fileset dir="hcatalog-pig-adapter/build/lib/provided"/>
                 <fileset dir="server-extensions/build/lib/compile"/>

Modified: incubator/hcatalog/trunk/core/pom.xml
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/core/pom.xml?rev=1458574&r1=1458573&r2=1458574&view=diff
==============================================================================
--- incubator/hcatalog/trunk/core/pom.xml (original)
+++ incubator/hcatalog/trunk/core/pom.xml Tue Mar 19 23:42:58 2013
@@ -89,13 +89,5 @@
             <version>${hive.version}</version>
             <scope>compile</scope>
         </dependency>
-
-        <!-- test scope -->
-        <dependency>
-            <groupId>org.apache.pig</groupId>
-            <artifactId>pig</artifactId>
-            <version>${pig.version}</version>
-            <scope>test</scope>
-        </dependency>
     </dependencies>
 </project>

Modified: incubator/hcatalog/trunk/core/src/main/java/org/apache/hcatalog/common/HCatUtil.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/core/src/main/java/org/apache/hcatalog/common/HCatUtil.java?rev=1458574&r1=1458573&r2=1458574&view=diff
==============================================================================
--- incubator/hcatalog/trunk/core/src/main/java/org/apache/hcatalog/common/HCatUtil.java (original)
+++ incubator/hcatalog/trunk/core/src/main/java/org/apache/hcatalog/common/HCatUtil.java Tue Mar 19 23:42:58 2013
@@ -615,4 +615,12 @@ public class HCatUtil {
             jobConf.set(entry.getKey(), entry.getValue());
         }
     }
+    
+
+    public static boolean isHadoop23() {
+        String version = org.apache.hadoop.util.VersionInfo.getVersion();
+        if (version.matches("\\b0\\.23\\..+\\b")||version.matches("\\b2\\..*"))
+            return true;
+        return false;
+    }
 }

Modified: incubator/hcatalog/trunk/core/src/main/java/org/apache/hcatalog/mapreduce/MultiOutputFormat.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/core/src/main/java/org/apache/hcatalog/mapreduce/MultiOutputFormat.java?rev=1458574&r1=1458573&r2=1458574&view=diff
==============================================================================
--- incubator/hcatalog/trunk/core/src/main/java/org/apache/hcatalog/mapreduce/MultiOutputFormat.java (original)
+++ incubator/hcatalog/trunk/core/src/main/java/org/apache/hcatalog/mapreduce/MultiOutputFormat.java Tue Mar 19 23:42:58 2013
@@ -47,6 +47,7 @@ import org.apache.hadoop.mapreduce.Recor
 import org.apache.hadoop.mapreduce.TaskAttemptContext;
 import org.apache.hadoop.mapreduce.TaskInputOutputContext;
 import org.apache.hadoop.util.ReflectionUtils;
+import org.apache.hcatalog.common.HCatUtil;
 import org.apache.hcatalog.shims.HCatHadoopShims;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -152,8 +153,14 @@ public class MultiOutputFormat extends O
         configsToMerge.put("tmparchives", COMMA_DELIM);
         configsToMerge.put(HCatHadoopShims.Instance.get().getPropertyName(HCatHadoopShims.PropertyName.CACHE_ARCHIVES), COMMA_DELIM);
         configsToMerge.put(HCatHadoopShims.Instance.get().getPropertyName(HCatHadoopShims.PropertyName.CACHE_FILES), COMMA_DELIM);
-        configsToMerge.put("mapred.job.classpath.archives", System.getProperty("path.separator"));
-        configsToMerge.put("mapred.job.classpath.files", System.getProperty("path.separator"));
+        String fileSep;
+        if (HCatUtil.isHadoop23()) {
+            fileSep = ",";
+        } else {
+            fileSep = System.getProperty("path.separator");
+        }
+        configsToMerge.put("mapred.job.classpath.archives", fileSep);
+        configsToMerge.put("mapred.job.classpath.files", fileSep);
     }
 
     /**

Modified: incubator/hcatalog/trunk/core/src/test/java/org/apache/hcatalog/HcatTestUtils.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/core/src/test/java/org/apache/hcatalog/HcatTestUtils.java?rev=1458574&r1=1458573&r2=1458574&view=diff
==============================================================================
--- incubator/hcatalog/trunk/core/src/test/java/org/apache/hcatalog/HcatTestUtils.java (original)
+++ incubator/hcatalog/trunk/core/src/test/java/org/apache/hcatalog/HcatTestUtils.java Tue Mar 19 23:42:58 2013
@@ -97,11 +97,4 @@ public class HcatTestUtils {
         }
 
     }
-
-    public static boolean isHadoop23() {
-        String version = org.apache.hadoop.util.VersionInfo.getVersion();
-        if (version.matches("\\b0\\.23\\..+\\b"))
-            return true;
-        return false;
-    }
 }

Modified: incubator/hcatalog/trunk/core/src/test/java/org/apache/hcatalog/mapreduce/HCatMapReduceTest.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/core/src/test/java/org/apache/hcatalog/mapreduce/HCatMapReduceTest.java?rev=1458574&r1=1458573&r2=1458574&view=diff
==============================================================================
--- incubator/hcatalog/trunk/core/src/test/java/org/apache/hcatalog/mapreduce/HCatMapReduceTest.java (original)
+++ incubator/hcatalog/trunk/core/src/test/java/org/apache/hcatalog/mapreduce/HCatMapReduceTest.java Tue Mar 19 23:42:58 2013
@@ -51,7 +51,6 @@ import org.apache.hadoop.mapreduce.JobSt
 import org.apache.hadoop.mapreduce.Mapper;
 import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;
 import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;
-import org.apache.hcatalog.HcatTestUtils;
 import org.apache.hcatalog.common.HCatConstants;
 import org.apache.hcatalog.common.HCatUtil;
 import org.apache.hcatalog.data.DefaultHCatRecord;
@@ -281,7 +280,7 @@ public abstract class HCatMapReduceTest 
                     .findCounter("FILE_BYTES_READ").getValue() > 0);
         }
 
-        if (!HcatTestUtils.isHadoop23()) {
+        if (!HCatUtil.isHadoop23()) {
             // Local mode outputcommitter hook is not invoked in Hadoop 1.x
             if (success) {
                 new FileOutputCommitterContainer(job, null).commitJob(job);

Modified: incubator/hcatalog/trunk/core/src/test/java/org/apache/hcatalog/mapreduce/TestHCatDynamicPartitioned.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/core/src/test/java/org/apache/hcatalog/mapreduce/TestHCatDynamicPartitioned.java?rev=1458574&r1=1458573&r2=1458574&view=diff
==============================================================================
--- incubator/hcatalog/trunk/core/src/test/java/org/apache/hcatalog/mapreduce/TestHCatDynamicPartitioned.java (original)
+++ incubator/hcatalog/trunk/core/src/test/java/org/apache/hcatalog/mapreduce/TestHCatDynamicPartitioned.java Tue Mar 19 23:42:58 2013
@@ -23,11 +23,13 @@ import java.io.IOException;
 import java.util.ArrayList;
 import java.util.List;
 
+import junit.framework.Assert;
+
 import org.apache.hadoop.hive.conf.HiveConf;
 import org.apache.hadoop.hive.metastore.api.FieldSchema;
 import org.apache.hadoop.hive.serde.serdeConstants;
 import org.apache.hadoop.mapreduce.Job;
-import org.apache.hcatalog.HcatTestUtils;
+import org.apache.hcatalog.common.HCatUtil;
 import org.apache.hcatalog.common.ErrorType;
 import org.apache.hcatalog.common.HCatConstants;
 import org.apache.hcatalog.common.HCatException;
@@ -141,20 +143,23 @@ public class TestHCatDynamicPartitioned 
         try {
             generateWriteRecords(NUM_RECORDS, NUM_PARTITIONS, 0);
             Job job = runMRCreate(null, dataColumns, writeRecords, NUM_RECORDS, false);
-            if (HcatTestUtils.isHadoop23()) {
-                new FileOutputCommitterContainer(job, null).cleanupJob(job);
+            
+            if (HCatUtil.isHadoop23()) {
+                Assert.assertTrue(job.isSuccessful()==false);
             }
         } catch (IOException e) {
             exc = e;
         }
 
-        assertTrue(exc != null);
-        assertTrue(exc instanceof HCatException);
-        assertTrue("Got exception of type [" + ((HCatException) exc).getErrorType().toString()
-                + "] Expected ERROR_PUBLISHING_PARTITION or ERROR_MOVE_FAILED",
-                (ErrorType.ERROR_PUBLISHING_PARTITION == ((HCatException) exc).getErrorType())
-                        || (ErrorType.ERROR_MOVE_FAILED == ((HCatException) exc).getErrorType())
-        );
+        if (!HCatUtil.isHadoop23()) {
+            assertTrue(exc != null);
+            assertTrue(exc instanceof HCatException);
+            assertTrue("Got exception of type [" + ((HCatException) exc).getErrorType().toString()
+                    + "] Expected ERROR_PUBLISHING_PARTITION or ERROR_MOVE_FAILED",
+                    (ErrorType.ERROR_PUBLISHING_PARTITION == ((HCatException) exc).getErrorType())
+                            || (ErrorType.ERROR_MOVE_FAILED == ((HCatException) exc).getErrorType())
+            );
+        }
     }
 
     //TODO 1.0 miniCluster is slow this test times out, make it work

Modified: incubator/hcatalog/trunk/core/src/test/java/org/apache/hcatalog/mapreduce/TestHCatMultiOutputFormat.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/core/src/test/java/org/apache/hcatalog/mapreduce/TestHCatMultiOutputFormat.java?rev=1458574&r1=1458573&r2=1458574&view=diff
==============================================================================
--- incubator/hcatalog/trunk/core/src/test/java/org/apache/hcatalog/mapreduce/TestHCatMultiOutputFormat.java (original)
+++ incubator/hcatalog/trunk/core/src/test/java/org/apache/hcatalog/mapreduce/TestHCatMultiOutputFormat.java Tue Mar 19 23:42:58 2013
@@ -175,6 +175,9 @@ public class TestHCatMultiOutputFormat {
         // LocalJobRunner does not work with mapreduce OutputCommitter. So need
         // to use MiniMRCluster. MAPREDUCE-2350
         Configuration conf = new Configuration(true);
+        conf.set("yarn.scheduler.capacity.root.queues", "default");
+        conf.set("yarn.scheduler.capacity.root.default.capacity", "100");
+
         FileSystem fs = FileSystem.get(conf);
         System.setProperty("hadoop.log.dir", new File(workDir, "/logs").getAbsolutePath());
         mrCluster = new MiniMRCluster(1, fs.getUri().toString(), 1, null, null,

Modified: incubator/hcatalog/trunk/core/src/test/java/org/apache/hcatalog/mapreduce/TestHCatPartitionPublish.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/core/src/test/java/org/apache/hcatalog/mapreduce/TestHCatPartitionPublish.java?rev=1458574&r1=1458573&r2=1458574&view=diff
==============================================================================
--- incubator/hcatalog/trunk/core/src/test/java/org/apache/hcatalog/mapreduce/TestHCatPartitionPublish.java (original)
+++ incubator/hcatalog/trunk/core/src/test/java/org/apache/hcatalog/mapreduce/TestHCatPartitionPublish.java Tue Mar 19 23:42:58 2013
@@ -75,6 +75,9 @@ public class TestHCatPartitionPublish {
     @BeforeClass
     public static void setup() throws Exception {
         Configuration conf = new Configuration(true);
+        conf.set("yarn.scheduler.capacity.root.queues", "default");
+        conf.set("yarn.scheduler.capacity.root.default.capacity", "100");
+        
         fs = FileSystem.get(conf);
         System.setProperty("hadoop.log.dir", new File(fs.getWorkingDirectory()
                 .toString(), "/logs").getAbsolutePath());

Modified: incubator/hcatalog/trunk/core/src/test/java/org/apache/hcatalog/mapreduce/TestMultiOutputFormat.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/core/src/test/java/org/apache/hcatalog/mapreduce/TestMultiOutputFormat.java?rev=1458574&r1=1458573&r2=1458574&view=diff
==============================================================================
--- incubator/hcatalog/trunk/core/src/test/java/org/apache/hcatalog/mapreduce/TestMultiOutputFormat.java (original)
+++ incubator/hcatalog/trunk/core/src/test/java/org/apache/hcatalog/mapreduce/TestMultiOutputFormat.java Tue Mar 19 23:42:58 2013
@@ -23,6 +23,8 @@ import java.io.File;
 import java.io.FileWriter;
 import java.io.IOException;
 import java.net.URI;
+import java.util.Arrays;
+import java.util.List;
 import java.util.Random;
 import java.util.StringTokenizer;
 
@@ -70,6 +72,9 @@ public class TestMultiOutputFormat {
     public static void setup() throws IOException {
         createWorkDir();
         Configuration conf = new Configuration(true);
+        conf.set("yarn.scheduler.capacity.root.queues", "default");
+        conf.set("yarn.scheduler.capacity.root.default.capacity", "100");
+        
         fs = FileSystem.get(conf);
         System.setProperty("hadoop.log.dir", new File(workDir, "/logs").getAbsolutePath());
         // LocalJobRunner does not work with mapreduce OutputCommitter. So need
@@ -134,12 +139,14 @@ public class TestMultiOutputFormat {
 
         // Verify if the configs are merged
         Path[] fileClassPaths = DistributedCache.getFileClassPaths(job.getConfiguration());
-        Assert.assertArrayEquals(new Path[]{new Path(inputFile), new Path(dummyFile)},
-            fileClassPaths);
-        URI[] expectedCacheFiles = new URI[]{new Path(inputFile).makeQualified(fs).toUri(),
-            new Path(dummyFile).makeQualified(fs).toUri()};
+        List<Path> fileClassPathsList = Arrays.asList(fileClassPaths);
+        Assert.assertTrue(fileClassPathsList.contains(new Path(inputFile)));
+        Assert.assertTrue(fileClassPathsList.contains(new Path(dummyFile)));
+
         URI[] cacheFiles = DistributedCache.getCacheFiles(job.getConfiguration());
-        Assert.assertArrayEquals(expectedCacheFiles, cacheFiles);
+        List<URI> cacheFilesList = Arrays.asList(cacheFiles);
+        Assert.assertTrue(cacheFilesList.contains(new Path(inputFile).makeQualified(fs).toUri()));
+        Assert.assertTrue(cacheFilesList.contains(new Path(dummyFile).makeQualified(fs).toUri()));
 
         Assert.assertTrue(job.waitForCompletion(true));
 

Modified: incubator/hcatalog/trunk/core/src/test/java/org/apache/hcatalog/mapreduce/TestSequenceFileReadWrite.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/core/src/test/java/org/apache/hcatalog/mapreduce/TestSequenceFileReadWrite.java?rev=1458574&r1=1458573&r2=1458574&view=diff
==============================================================================
--- incubator/hcatalog/trunk/core/src/test/java/org/apache/hcatalog/mapreduce/TestSequenceFileReadWrite.java (original)
+++ incubator/hcatalog/trunk/core/src/test/java/org/apache/hcatalog/mapreduce/TestSequenceFileReadWrite.java Tue Mar 19 23:42:58 2013
@@ -170,7 +170,7 @@ public class TestSequenceFileReadWrite e
         HCatOutputFormat.setSchema(job, getSchema());
         job.setNumReduceTasks(0);
         assertTrue(job.waitForCompletion(true));
-        if (!HcatTestUtils.isHadoop23()) {
+        if (!HCatUtil.isHadoop23()) {
             new FileOutputCommitterContainer(job, null).commitJob(job);
         }
         assertTrue(job.isSuccessful());
@@ -217,7 +217,7 @@ public class TestSequenceFileReadWrite e
         job.setOutputFormatClass(HCatOutputFormat.class);
         HCatOutputFormat.setSchema(job, getSchema());
         assertTrue(job.waitForCompletion(true));
-        if (!HcatTestUtils.isHadoop23()) {
+        if (!HCatUtil.isHadoop23()) {
             new FileOutputCommitterContainer(job, null).commitJob(job);
         }
         assertTrue(job.isSuccessful());

Modified: incubator/hcatalog/trunk/hcatalog-pig-adapter/pom.xml
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/hcatalog-pig-adapter/pom.xml?rev=1458574&r1=1458573&r2=1458574&view=diff
==============================================================================
--- incubator/hcatalog/trunk/hcatalog-pig-adapter/pom.xml (original)
+++ incubator/hcatalog/trunk/hcatalog-pig-adapter/pom.xml Tue Mar 19 23:42:58 2013
@@ -43,11 +43,5 @@
             <version>${hcatalog.version}</version>
             <scope>compile</scope>
         </dependency>
-        <dependency>
-            <groupId>org.apache.pig</groupId>
-            <artifactId>pig</artifactId>
-            <version>${pig.version}</version>
-            <scope>compile</scope>
-        </dependency>
     </dependencies>
 </project>

Modified: incubator/hcatalog/trunk/hcatalog-pig-adapter/src/test/java/org/apache/hcatalog/pig/TestHCatLoaderComplexSchema.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/hcatalog-pig-adapter/src/test/java/org/apache/hcatalog/pig/TestHCatLoaderComplexSchema.java?rev=1458574&r1=1458573&r2=1458574&view=diff
==============================================================================
--- incubator/hcatalog/trunk/hcatalog-pig-adapter/src/test/java/org/apache/hcatalog/pig/TestHCatLoaderComplexSchema.java (original)
+++ incubator/hcatalog/trunk/hcatalog-pig-adapter/src/test/java/org/apache/hcatalog/pig/TestHCatLoaderComplexSchema.java Tue Mar 19 23:42:58 2013
@@ -112,7 +112,6 @@ public class TestHCatLoaderComplexSchema
     @Test
     public void testSyntheticComplexSchema() throws Exception {
         String pigSchema =
-            "(" +
                 "a: " +
                 "(" +
                 "aa: chararray, " +
@@ -124,8 +123,7 @@ public class TestHCatLoaderComplexSchema
                 ")," +
                 "b: chararray, " +
                 "c: long, " +
-                "d:  { t: (da:long, db: ( dba: chararray, dbb: long), dc: { t: (dca: long) } ) } " +
-                ")";
+                "d:  { t: (da:long, db: ( dba: chararray, dbb: long), dc: { t: (dca: long) } ) } ";
 
         // with extra structs
         String tableSchema =
@@ -191,7 +189,7 @@ public class TestHCatLoaderComplexSchema
             createTable(tablename, tableSchema);
             PigServer server = new PigServer(ExecType.LOCAL);
             server.setBatchOn();
-            server.registerQuery("A = load '" + tablename + "Input' using org.apache.hcatalog.pig.MockLoader() AS " + pigSchema + ";");
+            server.registerQuery("A = load '" + tablename + "Input' using org.apache.hcatalog.pig.MockLoader() AS (" + pigSchema + ");");
             Schema dumpedASchema = server.dumpSchema("A");
             server.registerQuery("STORE A into '" + tablename + "' using org.apache.hcatalog.pig.HCatStorer("
                 + (provideSchemaToStorer ? "'', '" + pigSchema + "'" : "")
@@ -255,7 +253,7 @@ public class TestHCatLoaderComplexSchema
      */
     @Test
     public void testTupleInBagInTupleInBag() throws Exception {
-        String pigSchema = "(a: { b : ( c: { d: (i : long) } ) })";
+        String pigSchema = "a: { b : ( c: { d: (i : long) } ) }";
 
         String tableSchema = "a array< array< bigint > >";
 
@@ -279,7 +277,7 @@ public class TestHCatLoaderComplexSchema
 
     @Test
     public void testMapWithComplexData() throws Exception {
-        String pigSchema = "(a: long, b: map[])";
+        String pigSchema = "a: long, b: map[]";
         String tableSchema = "a bigint, b map<string, struct<aa:bigint, ab:string>>";
 
         List<Tuple> data = new ArrayList<Tuple>();

Modified: incubator/hcatalog/trunk/hcatalog-pig-adapter/src/test/java/org/apache/hcatalog/pig/TestPigHCatUtil.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/hcatalog-pig-adapter/src/test/java/org/apache/hcatalog/pig/TestPigHCatUtil.java?rev=1458574&r1=1458573&r2=1458574&view=diff
==============================================================================
--- incubator/hcatalog/trunk/hcatalog-pig-adapter/src/test/java/org/apache/hcatalog/pig/TestPigHCatUtil.java (original)
+++ incubator/hcatalog/trunk/hcatalog-pig-adapter/src/test/java/org/apache/hcatalog/pig/TestPigHCatUtil.java Tue Mar 19 23:42:58 2013
@@ -65,7 +65,7 @@ public class TestPigHCatUtil {
         // When updating our pig dependency this will need updated.
         System.setProperty(HCatConstants.HCAT_PIG_INNER_TUPLE_NAME, "t");
         System.setProperty(HCatConstants.HCAT_PIG_INNER_FIELD_NAME, "FIELDNAME_tuple");
-        UDFContext.getUDFContext().setClientSystemProps();
+        UDFContext.getUDFContext().setClientSystemProps(System.getProperties());
 
         // Define the expected schema.
         ResourceFieldSchema[] bagSubFieldSchemas = new ResourceFieldSchema[1];

Modified: incubator/hcatalog/trunk/pom.xml
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/pom.xml?rev=1458574&r1=1458573&r2=1458574&view=diff
==============================================================================
--- incubator/hcatalog/trunk/pom.xml (original)
+++ incubator/hcatalog/trunk/pom.xml Tue Mar 19 23:42:58 2013
@@ -27,18 +27,19 @@
       <commons-io.version>2.4</commons-io.version>
       <guava.version>11.0.2</guava.version>
       <hadoop20.version>1.0.3</hadoop20.version>
-      <hadoop23.version>0.23.3</hadoop23.version>
-      <hbase.version>0.92.0</hbase.version>
+      <hadoop23.version>2.0.3-alpha</hadoop23.version>
+      <hbase.version>0.94.5</hbase.version>
       <hcatalog.version>${project.version}</hcatalog.version>
-      <hive.version>0.10.0</hive.version>
+      <hive.version>0.11.0-SNAPSHOT</hive.version>
       <jackson.version>1.8.3</jackson.version>
       <jersey.version>1.14</jersey.version>
       <jetty.webhcat.version>7.6.0.v20120127</jetty.webhcat.version>
       <jms.version>1.1</jms.version>
-      <pig.version>0.8.0</pig.version>
+      <pig.version>0.10.1</pig.version>
       <slf4j.version>1.6.1</slf4j.version>
       <wadl-resourcedoc-doclet.version>1.4</wadl-resourcedoc-doclet.version>
       <zookeeper.version>3.4.3</zookeeper.version>
+      <metrics-core.version>2.1.2</metrics-core.version>
   </properties>
 
   <parent>
@@ -83,6 +84,12 @@
           <version>${hadoop20.version}</version>
           <scope>test</scope>
         </dependency>
+        <dependency>
+          <groupId>org.apache.pig</groupId>
+          <artifactId>pig</artifactId>
+          <version>${pig.version}</version>
+          <scope>compile</scope>
+        </dependency>
       </dependencies>
     </profile>
     <profile>
@@ -108,10 +115,63 @@
         </dependency>
         <dependency>
           <groupId>org.apache.hadoop</groupId>
+          <artifactId>hadoop-hdfs</artifactId>
+          <version>${hadoop23.version}</version>
+          <classifier>tests</classifier>
+          <scope>compile</scope>
+        </dependency>
+        <dependency>
+          <groupId>org.apache.hadoop</groupId>
           <artifactId>hadoop-mapreduce-client-core</artifactId>
           <version>${hadoop23.version}</version>
           <scope>compile</scope>
         </dependency>
+        <dependency>
+          <groupId>org.apache.hadoop</groupId>
+          <artifactId>hadoop-mapreduce-client-jobclient</artifactId>
+          <version>${hadoop23.version}</version>
+          <classifier>tests</classifier>
+          <scope>compile</scope>
+        </dependency>
+        <dependency>
+          <groupId>org.apache.hadoop</groupId>
+          <artifactId>hadoop-yarn-server-tests</artifactId>
+          <version>${hadoop23.version}</version>
+          <classifier>tests</classifier>
+          <scope>compile</scope>
+        </dependency>
+        <dependency>
+          <groupId>org.apache.hadoop</groupId>
+          <artifactId>hadoop-mapreduce-client-app</artifactId>
+          <version>${hadoop23.version}</version>
+          <scope>compile</scope>
+        </dependency>
+        <dependency>
+          <groupId>org.apache.hadoop</groupId>
+          <artifactId>hadoop-common</artifactId>
+          <version>${hadoop23.version}</version>
+          <classifier>tests</classifier>
+          <scope>compile</scope>
+        </dependency>
+        <dependency>
+          <groupId>org.apache.hadoop</groupId>
+          <artifactId>hadoop-mapreduce-client-hs</artifactId>
+          <version>${hadoop23.version}</version>
+          <scope>compile</scope>
+        </dependency>
+        <dependency>
+          <groupId>org.apache.hadoop</groupId>
+          <artifactId>hadoop-minicluster</artifactId>
+          <version>${hadoop23.version}</version>
+          <scope>compile</scope>
+        </dependency>
+        <dependency>
+          <groupId>org.apache.pig</groupId>
+          <artifactId>pig</artifactId>
+          <version>${pig.version}</version>
+          <classifier>h2</classifier>
+          <scope>compile</scope>
+        </dependency>
       </dependencies>
     </profile>
   </profiles>

Added: incubator/hcatalog/trunk/shims/src/20/java/org/apache/hadoop/mapred/TempletonJobTracker.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/shims/src/20/java/org/apache/hadoop/mapred/TempletonJobTracker.java?rev=1458574&view=auto
==============================================================================
--- incubator/hcatalog/trunk/shims/src/20/java/org/apache/hadoop/mapred/TempletonJobTracker.java (added)
+++ incubator/hcatalog/trunk/shims/src/20/java/org/apache/hadoop/mapred/TempletonJobTracker.java Tue Mar 19 23:42:58 2013
@@ -0,0 +1,95 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hadoop.mapred;
+
+import java.io.IOException;
+import java.net.InetSocketAddress;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.ipc.RPC;
+import org.apache.hadoop.net.NetUtils;
+import org.apache.hadoop.security.UserGroupInformation;
+
+/*
+ * Communicate with the JobTracker as a specific user.
+ */
+public class TempletonJobTracker {
+    private JobSubmissionProtocol cnx;
+
+    /**
+     * Create a connection to the Job Tracker.
+     */
+    public TempletonJobTracker(InetSocketAddress addr,
+                               Configuration conf)
+        throws IOException {
+        UserGroupInformation ugi = UserGroupInformation.getLoginUser();
+        cnx = (JobSubmissionProtocol)
+            RPC.getProxy(JobSubmissionProtocol.class,
+                JobSubmissionProtocol.versionID,
+                addr,
+                ugi,
+                conf,
+                NetUtils.getSocketFactory(conf,
+                    JobSubmissionProtocol.class));
+    }
+
+    /**
+     * Grab a handle to a job that is already known to the JobTracker.
+     *
+     * @return Profile of the job, or null if not found.
+     */
+    public JobProfile getJobProfile(JobID jobid)
+        throws IOException {
+        return cnx.getJobProfile(jobid);
+    }
+
+    /**
+     * Grab a handle to a job that is already known to the JobTracker.
+     *
+     * @return Status of the job, or null if not found.
+     */
+    public JobStatus getJobStatus(JobID jobid)
+        throws IOException {
+        return cnx.getJobStatus(jobid);
+    }
+
+
+    /**
+     * Kill a job.
+     */
+    public void killJob(JobID jobid)
+        throws IOException {
+        cnx.killJob(jobid);
+    }
+
+    /**
+     * Get all the jobs submitted.
+     */
+    public JobStatus[] getAllJobs()
+        throws IOException {
+        return cnx.getAllJobs();
+    }
+
+    /**
+     * Close the connection to the Job Tracker.
+     */
+    public void close() {
+        RPC.stopProxy(cnx);
+    }
+}

Added: incubator/hcatalog/trunk/shims/src/23/java/org/apache/hadoop/mapred/TempletonJobTracker.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/shims/src/23/java/org/apache/hadoop/mapred/TempletonJobTracker.java?rev=1458574&view=auto
==============================================================================
--- incubator/hcatalog/trunk/shims/src/23/java/org/apache/hadoop/mapred/TempletonJobTracker.java (added)
+++ incubator/hcatalog/trunk/shims/src/23/java/org/apache/hadoop/mapred/TempletonJobTracker.java Tue Mar 19 23:42:58 2013
@@ -0,0 +1,96 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hadoop.mapred;
+
+import java.io.IOException;
+import java.net.InetSocketAddress;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.security.UserGroupInformation;
+
+/*
+ * Communicate with the JobTracker as a specific user.
+ */
+public class TempletonJobTracker {
+    private JobClient jc;
+
+    /**
+     * Create a connection to the Job Tracker.
+     */
+    public TempletonJobTracker(InetSocketAddress addr,
+                               Configuration conf)
+        throws IOException {
+        
+        jc = new JobClient(conf);
+    }
+
+    /**
+     * Grab a handle to a job that is already known to the JobTracker.
+     *
+     * @return Profile of the job, or null if not found.
+     */
+    public JobProfile getJobProfile(JobID jobid)
+        throws IOException {
+        RunningJob rj = jc.getJob(jobid);
+        JobStatus jobStatus = rj.getJobStatus();
+        JobProfile jobProfile = new JobProfile(jobStatus.getUsername(), jobStatus.getJobID(), 
+                jobStatus.getJobFile(), jobStatus.getTrackingUrl(), jobStatus.getJobName());
+        return jobProfile;
+    }
+
+    /**
+     * Grab a handle to a job that is already known to the JobTracker.
+     *
+     * @return Status of the job, or null if not found.
+     */
+    public JobStatus getJobStatus(JobID jobid)
+        throws IOException {
+        RunningJob rj = jc.getJob(jobid);
+        JobStatus jobStatus = rj.getJobStatus();
+        return jobStatus;
+    }
+
+
+    /**
+     * Kill a job.
+     */
+    public void killJob(JobID jobid)
+        throws IOException {
+        RunningJob rj = jc.getJob(jobid);
+        rj.killJob();
+    }
+
+    /**
+     * Get all the jobs submitted.
+     */
+    public JobStatus[] getAllJobs()
+        throws IOException {
+        return jc.getAllJobs();
+    }
+
+    /**
+     * Close the connection to the Job Tracker.
+     */
+    public void close() {
+        try {
+            jc.close();
+        } catch (IOException e) {
+        }
+    }
+}

Modified: incubator/hcatalog/trunk/storage-handlers/hbase/pom.xml
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/storage-handlers/hbase/pom.xml?rev=1458574&r1=1458573&r2=1458574&view=diff
==============================================================================
--- incubator/hcatalog/trunk/storage-handlers/hbase/pom.xml (original)
+++ incubator/hcatalog/trunk/storage-handlers/hbase/pom.xml Tue Mar 19 23:42:58 2013
@@ -61,6 +61,12 @@
       <version>${zookeeper.version}</version>
       <scope>compile</scope>
     </dependency>
+    <dependency>
+      <groupId>org.apache.hbase</groupId>
+      <artifactId>hbase</artifactId>
+      <version>${hbase.version}</version>
+      <scope>compile</scope>
+    </dependency>
 
     <!-- test scope -->
     <dependency>
@@ -89,5 +95,11 @@
       <classifier>tests</classifier>
       <scope>test</scope>
     </dependency>
+    <dependency>
+      <groupId>com.yammer.metrics</groupId>
+      <artifactId>metrics-core</artifactId>
+      <version>${metrics-core.version}</version>
+      <scope>test</scope>
+    </dependency>
   </dependencies>
 </project>

Modified: incubator/hcatalog/trunk/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/HBaseBulkOutputFormat.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/HBaseBulkOutputFormat.java?rev=1458574&r1=1458573&r2=1458574&view=diff
==============================================================================
--- incubator/hcatalog/trunk/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/HBaseBulkOutputFormat.java (original)
+++ incubator/hcatalog/trunk/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/HBaseBulkOutputFormat.java Tue Mar 19 23:42:58 2013
@@ -144,7 +144,7 @@ class HBaseBulkOutputFormat extends HBas
         @Override
         public void commitTask(TaskAttemptContext taskContext)
             throws IOException {
-            baseOutputCommitter.commitTask(taskContext);
+            // baseOutputCommitter.commitTask(taskContext);
         }
 
         @Override

Modified: incubator/hcatalog/trunk/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/ManyMiniCluster.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/ManyMiniCluster.java?rev=1458574&r1=1458573&r2=1458574&view=diff
==============================================================================
--- incubator/hcatalog/trunk/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/ManyMiniCluster.java (original)
+++ incubator/hcatalog/trunk/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/ManyMiniCluster.java Tue Mar 19 23:42:58 2013
@@ -210,6 +210,8 @@ public class ManyMiniCluster {
                 jobConf = new JobConf();
 
             jobConf.setInt("mapred.submit.replication", 1);
+            jobConf.set("yarn.scheduler.capacity.root.queues", "default");
+            jobConf.set("yarn.scheduler.capacity.root.default.capacity", "100");
             //conf.set("hadoop.job.history.location",new File(workDir).getAbsolutePath()+"/history");
             System.setProperty("hadoop.log.dir", new File(workDir, "/logs").getAbsolutePath());
 

Modified: incubator/hcatalog/trunk/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/SkeletonHBaseTest.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/SkeletonHBaseTest.java?rev=1458574&r1=1458573&r2=1458574&view=diff
==============================================================================
--- incubator/hcatalog/trunk/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/SkeletonHBaseTest.java (original)
+++ incubator/hcatalog/trunk/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/SkeletonHBaseTest.java Tue Mar 19 23:42:58 2013
@@ -30,6 +30,7 @@ import java.util.Set;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.FileUtil;
+import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hbase.HBaseConfiguration;
 import org.apache.hadoop.hbase.HColumnDescriptor;
 import org.apache.hadoop.hbase.HTableDescriptor;
@@ -203,10 +204,12 @@ public abstract class SkeletonHBaseTest 
                 } finally {
                     System.out.println("Trying to cleanup: " + testDir);
                     try {
-                        FileUtil.fullyDelete(new File(testDir));
+                        FileSystem fs = FileSystem.get(jobConf);
+                        fs.delete(new Path(testDir), true);
                     } catch (IOException e) {
                         throw new IllegalStateException("Failed to cleanup test dir", e);
                     }
+                    
                 }
             }
         }

Modified: incubator/hcatalog/trunk/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/TestHBaseBulkOutputFormat.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/TestHBaseBulkOutputFormat.java?rev=1458574&r1=1458573&r2=1458574&view=diff
==============================================================================
--- incubator/hcatalog/trunk/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/TestHBaseBulkOutputFormat.java (original)
+++ incubator/hcatalog/trunk/storage-handlers/hbase/src/test/org/apache/hcatalog/hbase/TestHBaseBulkOutputFormat.java Tue Mar 19 23:42:58 2013
@@ -183,6 +183,8 @@ public class TestHBaseBulkOutputFormat e
 
         //create table
         conf.set(HBaseConstants.PROPERTY_OUTPUT_TABLE_NAME_KEY, tableName);
+        conf.set("yarn.scheduler.capacity.root.queues", "default");
+        conf.set("yarn.scheduler.capacity.root.default.capacity", "100");
         createTable(tableName, new String[]{familyName});
 
         String data[] = {"1,english:one,spanish:uno",

Modified: incubator/hcatalog/trunk/webhcat/svr/pom.xml
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/webhcat/svr/pom.xml?rev=1458574&r1=1458573&r2=1458574&view=diff
==============================================================================
--- incubator/hcatalog/trunk/webhcat/svr/pom.xml (original)
+++ incubator/hcatalog/trunk/webhcat/svr/pom.xml Tue Mar 19 23:42:58 2013
@@ -48,12 +48,6 @@
             <version>${hcatalog.version}</version>
             <scope>provided</scope>
         </dependency>
-        <dependency>
-            <groupId>org.apache.hadoop</groupId>
-            <artifactId>hadoop-core</artifactId>
-            <version>${hadoop20.version}</version>
-            <scope>provided</scope>
-        </dependency>
 
         <!-- compile scope - this is packaged -->
         <dependency>

Modified: incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/DeleteDelegator.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/DeleteDelegator.java?rev=1458574&r1=1458573&r2=1458574&view=diff
==============================================================================
--- incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/DeleteDelegator.java (original)
+++ incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/DeleteDelegator.java Tue Mar 19 23:42:58 2013
@@ -20,7 +20,6 @@ package org.apache.hcatalog.templeton;
 
 import java.io.IOException;
 import org.apache.hadoop.mapred.JobID;
-import org.apache.hadoop.mapred.JobTracker;
 import org.apache.hadoop.mapred.TempletonJobTracker;
 import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hcatalog.templeton.tool.JobState;
@@ -40,7 +39,7 @@ public class DeleteDelegator extends Tem
         TempletonJobTracker tracker = null;
         JobState state = null;
         try {
-            tracker = new TempletonJobTracker(JobTracker.getAddress(appConf),
+            tracker = new TempletonJobTracker(getAddress(appConf),
                                               appConf);
             JobID jobid = StatusDelegator.StringToJobID(id);
             if (jobid == null)

Modified: incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/ListDelegator.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/ListDelegator.java?rev=1458574&r1=1458573&r2=1458574&view=diff
==============================================================================
--- incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/ListDelegator.java (original)
+++ incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/ListDelegator.java Tue Mar 19 23:42:58 2013
@@ -23,7 +23,6 @@ import java.util.List;
 import java.util.ArrayList;
 
 import org.apache.hadoop.mapred.JobStatus;
-import org.apache.hadoop.mapred.JobTracker;
 import org.apache.hadoop.mapred.TempletonJobTracker;
 import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hcatalog.templeton.tool.JobState;
@@ -42,7 +41,7 @@ public class ListDelegator extends Templ
         UserGroupInformation ugi = UserGroupInformation.createRemoteUser(user);
         TempletonJobTracker tracker = null;
         try {
-            tracker = new TempletonJobTracker(JobTracker.getAddress(appConf),
+            tracker = new TempletonJobTracker(getAddress(appConf),
                     appConf);
 
             ArrayList<String> ids = new ArrayList<String>();

Modified: incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/StatusDelegator.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/StatusDelegator.java?rev=1458574&r1=1458573&r2=1458574&view=diff
==============================================================================
--- incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/StatusDelegator.java (original)
+++ incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/StatusDelegator.java Tue Mar 19 23:42:58 2013
@@ -25,7 +25,6 @@ import org.apache.commons.logging.LogFac
 import org.apache.hadoop.mapred.JobID;
 import org.apache.hadoop.mapred.JobProfile;
 import org.apache.hadoop.mapred.JobStatus;
-import org.apache.hadoop.mapred.JobTracker;
 import org.apache.hadoop.mapred.TempletonJobTracker;
 import org.apache.hcatalog.templeton.tool.JobState;
 
@@ -45,7 +44,7 @@ public class StatusDelegator extends Tem
         TempletonJobTracker tracker = null;
         JobState state = null;
         try {
-            tracker = new TempletonJobTracker(JobTracker.getAddress(appConf),
+            tracker = new TempletonJobTracker(getAddress(appConf),
                                               appConf);
             JobID jobid = StatusDelegator.StringToJobID(id);
             if (jobid == null)

Modified: incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/TempletonDelegator.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/TempletonDelegator.java?rev=1458574&r1=1458573&r2=1458574&view=diff
==============================================================================
--- incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/TempletonDelegator.java (original)
+++ incubator/hcatalog/trunk/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/TempletonDelegator.java Tue Mar 19 23:42:58 2013
@@ -18,6 +18,11 @@
  */
 package org.apache.hcatalog.templeton;
 
+import java.net.InetSocketAddress;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.net.NetUtils;
+
 /**
  * The helper class for all the Templeton delegator classes. A
  * delegator will call the underlying Templeton service such as hcat
@@ -29,4 +34,10 @@ public class TempletonDelegator {
     public TempletonDelegator(AppConfig appConf) {
         this.appConf = appConf;
     }
+    
+    public static InetSocketAddress getAddress(Configuration conf) {
+        String jobTrackerStr =
+                conf.get("mapred.job.tracker", "localhost:8012");
+              return NetUtils.createSocketAddr(jobTrackerStr);
+            }
 }



Mime
View raw message