incubator-hcatalog-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From tof...@apache.org
Subject svn commit: r1374862 - in /incubator/hcatalog/trunk: ./ hcatalog-pig-adapter/ hcatalog-pig-adapter/src/main/java/org/apache/hcatalog/pig/ ivy/ shims/ shims/src/20/ shims/src/20/java/ shims/src/20/java/org/ shims/src/20/java/org/apache/ shims/src/20/jav...
Date Sun, 19 Aug 2012 21:49:10 GMT
Author: toffer
Date: Sun Aug 19 21:49:08 2012
New Revision: 1374862

URL: http://svn.apache.org/viewvc?rev=1374862&view=rev
Log:
HCATALOG-375 Make HCat work for Hadoop 0.23 (cdrome via toffer)

Added:
    incubator/hcatalog/trunk/shims/build.xml
    incubator/hcatalog/trunk/shims/ivy.xml
    incubator/hcatalog/trunk/shims/src/20/
    incubator/hcatalog/trunk/shims/src/20/java/
    incubator/hcatalog/trunk/shims/src/20/java/org/
    incubator/hcatalog/trunk/shims/src/20/java/org/apache/
    incubator/hcatalog/trunk/shims/src/20/java/org/apache/hcatalog/
    incubator/hcatalog/trunk/shims/src/20/java/org/apache/hcatalog/shims/
    incubator/hcatalog/trunk/shims/src/20/java/org/apache/hcatalog/shims/HCatHadoopShims20S.java
Removed:
    incubator/hcatalog/trunk/shims/src/20S/java/org/apache/hcatalog/shims/HCatHadoopShims20S.java
Modified:
    incubator/hcatalog/trunk/CHANGES.txt
    incubator/hcatalog/trunk/build.properties
    incubator/hcatalog/trunk/build.xml
    incubator/hcatalog/trunk/hcatalog-pig-adapter/ivy.xml
    incubator/hcatalog/trunk/hcatalog-pig-adapter/src/main/java/org/apache/hcatalog/pig/HCatStorer.java
    incubator/hcatalog/trunk/ivy.xml
    incubator/hcatalog/trunk/ivy/libraries.properties
    incubator/hcatalog/trunk/shims/src/23/java/org/apache/hcatalog/shims/HCatHadoopShims23.java
    incubator/hcatalog/trunk/src/java/org/apache/hadoop/mapred/HCatMapRedUtil.java
    incubator/hcatalog/trunk/src/java/org/apache/hcatalog/data/transfer/impl/HCatInputFormatReader.java
    incubator/hcatalog/trunk/src/java/org/apache/hcatalog/data/transfer/impl/HCatOutputFormatWriter.java
    incubator/hcatalog/trunk/src/java/org/apache/hcatalog/data/transfer/state/StateProvider.java
    incubator/hcatalog/trunk/src/java/org/apache/hcatalog/mapreduce/FileOutputCommitterContainer.java
    incubator/hcatalog/trunk/src/java/org/apache/hcatalog/mapreduce/FileRecordWriterContainer.java
    incubator/hcatalog/trunk/src/java/org/apache/hcatalog/mapreduce/MultiOutputFormat.java
    incubator/hcatalog/trunk/src/java/org/apache/hcatalog/mapreduce/ProgressReporter.java
    incubator/hcatalog/trunk/src/java/org/apache/hcatalog/mapreduce/Security.java
    incubator/hcatalog/trunk/src/java/org/apache/hcatalog/shims/HCatHadoopShims.java
    incubator/hcatalog/trunk/src/test/e2e/hcatalog/build.xml
    incubator/hcatalog/trunk/src/test/e2e/hcatalog/conf/default.conf
    incubator/hcatalog/trunk/src/test/e2e/hcatalog/drivers/Util.pm
    incubator/hcatalog/trunk/src/test/e2e/hcatalog/tests/hcat.conf
    incubator/hcatalog/trunk/src/test/e2e/hcatalog/tools/generate/generate_data.pl
    incubator/hcatalog/trunk/src/test/e2e/hcatalog/tools/generate/java/build.xml
    incubator/hcatalog/trunk/src/test/e2e/hcatalog/udfs/java/build.xml
    incubator/hcatalog/trunk/src/test/e2e/hcatalog/udfs/java/org/apache/hcatalog/utils/SimpleRead.java
    incubator/hcatalog/trunk/src/test/org/apache/hcatalog/HcatTestUtils.java
    incubator/hcatalog/trunk/src/test/org/apache/hcatalog/data/TestReaderWriter.java
    incubator/hcatalog/trunk/src/test/org/apache/hcatalog/mapreduce/HCatMapReduceTest.java
    incubator/hcatalog/trunk/src/test/org/apache/hcatalog/mapreduce/TestHCatDynamicPartitioned.java
    incubator/hcatalog/trunk/src/test/org/apache/hcatalog/mapreduce/TestSequenceFileReadWrite.java
    incubator/hcatalog/trunk/storage-handlers/hbase/build.xml
    incubator/hcatalog/trunk/storage-handlers/hbase/ivy.xml
    incubator/hcatalog/trunk/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/ImportSequenceFile.java
    incubator/hcatalog/trunk/webhcat/java-client/ivy.xml
    incubator/hcatalog/trunk/webhcat/svr/ivy.xml

Modified: incubator/hcatalog/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/CHANGES.txt?rev=1374862&r1=1374861&r2=1374862&view=diff
==============================================================================
--- incubator/hcatalog/trunk/CHANGES.txt (original)
+++ incubator/hcatalog/trunk/CHANGES.txt Sun Aug 19 21:49:08 2012
@@ -23,6 +23,8 @@ Trunk (unreleased changes)
   INCOMPATIBLE CHANGES
 
   NEW FEATURES
+  HCAT-375 Make HCat work for Hadoop 0.23 (cdrome via toffer)
+
   HCAT-469 HiveClientCache may return a closed client (amalakar via toffer)
 
   HCAT-370 Create a HiveMetaStoreClient cache in hcatalog (amalakar via toffer) 

Modified: incubator/hcatalog/trunk/build.properties
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/build.properties?rev=1374862&r1=1374861&r2=1374862&view=diff
==============================================================================
--- incubator/hcatalog/trunk/build.properties (original)
+++ incubator/hcatalog/trunk/build.properties Sun Aug 19 21:49:08 2012
@@ -12,7 +12,6 @@ build.classes=${build.dir}/classes
 src.dir=${basedir}/src/java
 package.dir=${basedir}/src/packages
 docs.src=${basedir}/src/docs
-build.classes=${build.dir}/classes
 build.docs=${build.dir}/docs
 build.javadoc=${build.docs}/api
 dist.dir=${build.dir}/${final.name}
@@ -50,10 +49,13 @@ javac.optimize=on
 javac.deprecation=off
 javac.version=1.6
 javac.args=
+javac.args.warnings=
 
+#Set to 20 to build against hadoop 1.0.2 or 23 to build against hadoop 0.23.1
+hadoopversion=20
 
 # hive properties
-shims.name=20S
+#shims.name=20
 shims.20S.hive.shims.include=0.20,0.20S
 shims.20S.hadoop.version=${hive.hadoop-0.20S.version}
 shims.23.hive.shims.include=0.23

Modified: incubator/hcatalog/trunk/build.xml
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/build.xml?rev=1374862&r1=1374861&r2=1374862&view=diff
==============================================================================
--- incubator/hcatalog/trunk/build.xml (original)
+++ incubator/hcatalog/trunk/build.xml Sun Aug 19 21:49:08 2012
@@ -43,7 +43,7 @@
   <!-- e2e test properties -->
   <property name="test.e2e.dir" value="${basedir}/src/test/e2e/hcatalog"/>
 
-  <!-- ivy properteis set here -->
+  <!-- ivy properties set here -->
   <property name="ivy.repo.dir" value="${user.home}/ivyrepo" />
   <property name="ivy.dir" location="ivy" />
   <loadproperties srcfile="${ivy.dir}/libraries.properties"/>
@@ -156,17 +156,6 @@
       <compilerarg line="${javac.args}"/>
       <classpath refid="compile.classpath"/>
     </javac>
-    <!-- compile shim for selected hadoop version -->
-    <!--property name="debugclasspath" refid="classpath"/>
-    <echo message="classpath = ${debugclasspath}"/-->
-    <javac encoding="${build.encoding}" srcdir="${basedir}/shims/src/${shims.name}/java" excludes="${excludes}"
-        includes="**/*.java" destdir="${build.classes}" debug="${javac.debug}"
-        optimize="${javac.optimize}" target="${javac.version}"
-        source="${javac.version}" deprecation="${javac.deprecation}"
-        includeantruntime="false">
-      <compilerarg line="${javac.args}"/>
-      <classpath refid="compile.classpath"/>
-    </javac>
   </target>
  
   <!-- Build the hcatalog client jar -->
@@ -197,7 +186,7 @@
   Build both clientjar and server-extensions
   ================================================================================
   -->
-  <target name="jar" depends="clientjar,server-extensions,jar-storage-handlers">
+  <target name="jar" depends="shims,clientjar,server-extensions,jar-storage-handlers">
     <ant target="jar" dir="hcatalog-pig-adapter" inheritAll="false"/>
     <ant target="jar" dir="webhcat/svr" inheritAll="false"/>
     <ant target="jar" dir="webhcat/java-client" inheritAll="false"/>
@@ -212,6 +201,16 @@
 
   <!--
   ================================================================================
+  Build shims
+  ================================================================================
+  -->
+
+  <target name="shims" depends="compile-src">
+    <ant antfile="shims/build.xml" target="jar" inheritAll="false" useNativeBasedir="true"/>
+  </target>
+
+  <!--
+  ================================================================================
   Build storage handlers
   ================================================================================
   -->
@@ -306,6 +305,7 @@
     <ant target="clean" dir="webhcat/svr" inheritAll="false"/>
     <ant target="clean" dir="webhcat/java-client" inheritAll="false"/>
     <ant target="clean" dir="storage-handlers" inheritAll="false" useNativeBasedir="true"/>
+    <ant target="clean" dir="shims" inheritAll="false" useNativeBasedir="true"/>
   </target>
  
   <!--

Modified: incubator/hcatalog/trunk/hcatalog-pig-adapter/ivy.xml
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/hcatalog-pig-adapter/ivy.xml?rev=1374862&r1=1374861&r2=1374862&view=diff
==============================================================================
--- incubator/hcatalog/trunk/hcatalog-pig-adapter/ivy.xml (original)
+++ incubator/hcatalog/trunk/hcatalog-pig-adapter/ivy.xml Sun Aug 19 21:49:08 2012
@@ -15,7 +15,7 @@
    limitations under the License.
 -->
 
-<ivy-module version="2.0">
+<ivy-module version="2.0" xmlns:m="http://ant.apache.org/ivy/maven">
   <info organisation="org.apache.hcatalog"
         module="${ant.project.name}"
         revision="${hcatalog.version}">
@@ -32,10 +32,8 @@
   <dependencies>
     <dependency org="org.apache.hcatalog" name="hcatalog-core" rev="${hcatalog.version}"/>
     <dependency org="org.apache.pig" name="pig" rev="${pig.version}"/>
-
     <!-- test dependencies -->
     <dependency org="junit" name="junit" rev="${junit.version}" conf="test->default"/>
-    <dependency org="org.apache.hadoop" name="hadoop-test" rev="${hadoop.jars.version}"
-      conf="test->default"/>
+    <dependency org="org.apache.hadoop" name="hadoop-test" rev="${hadoop20.version}" conf="test->default"/>
   </dependencies>
 </ivy-module>

Modified: incubator/hcatalog/trunk/hcatalog-pig-adapter/src/main/java/org/apache/hcatalog/pig/HCatStorer.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/hcatalog-pig-adapter/src/main/java/org/apache/hcatalog/pig/HCatStorer.java?rev=1374862&r1=1374861&r2=1374862&view=diff
==============================================================================
--- incubator/hcatalog/trunk/hcatalog-pig-adapter/src/main/java/org/apache/hcatalog/pig/HCatStorer.java (original)
+++ incubator/hcatalog/trunk/hcatalog-pig-adapter/src/main/java/org/apache/hcatalog/pig/HCatStorer.java Sun Aug 19 21:49:08 2012
@@ -154,18 +154,11 @@ public class HCatStorer extends HCatBase
 
   @Override
   public void storeSchema(ResourceSchema schema, String arg1, Job job) throws IOException {
-    if( job.getConfiguration().get("mapred.job.tracker", "").equalsIgnoreCase("local") ) {
-      try {
-      //In local mode, mapreduce will not call OutputCommitter.cleanupJob.
-      //Calling it from here so that the partition publish happens.
-      //This call needs to be removed after MAPREDUCE-1447 is fixed.
-        getOutputFormat().getOutputCommitter(HCatHadoopShims.Instance.get().createTaskAttemptContext(
-            job.getConfiguration(), new TaskAttemptID())).cleanupJob(job);
-      } catch (IOException e) {
-        throw new IOException("Failed to cleanup job",e);
-      } catch (InterruptedException e) {
-        throw new IOException("Failed to cleanup job",e);
-      }
-    }
+    HCatHadoopShims.Instance.get().commitJob(getOutputFormat(), schema, arg1, job);
+  }
+
+  @Override
+  public void cleanupOnFailure(String location, Job job) throws IOException {
+      HCatHadoopShims.Instance.get().abortJob(getOutputFormat(), job);
   }
 }

Modified: incubator/hcatalog/trunk/ivy.xml
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/ivy.xml?rev=1374862&r1=1374861&r2=1374862&view=diff
==============================================================================
--- incubator/hcatalog/trunk/ivy.xml (original)
+++ incubator/hcatalog/trunk/ivy.xml Sun Aug 19 21:49:08 2012
@@ -14,7 +14,7 @@
   See the License for the specific language governing permissions and
   limitations under the License. -->
 
-<ivy-module version="2.0">
+<ivy-module version="2.0" xmlns:m="http://ant.apache.org/ivy/maven">
   <info organisation="org.apache.hcatalog"
         module="${ant.project.name}"
         revision="${hcatalog.version}">
@@ -35,8 +35,15 @@
   </publications>
 
   <dependencies>
-    <dependency org="org.apache.pig" name="pig" rev="${pig.version}"/>
-    <dependency org="org.apache.hadoop" name="hadoop-tools" rev="${hadoop.jars.version}" conf="default->*"/>
+    <!-- hadoop20 dependencies -->
+    <dependency org="org.apache.pig" name="pig" rev="${pig.version}" conf="default->*"/>
+    <dependency org="org.apache.hadoop" name="hadoop-tools" rev="${hadoop20.version}" conf="default->*">
+      <exclude org="commons-daemon" module="commons-daemon"/>
+    </dependency>
+    <dependency org="org.apache.hadoop" name="hadoop-test" rev="${hadoop20.version}" conf="test->*">
+      <exclude org="commons-daemon" module="commons-daemon"/>
+    </dependency>
+
     <dependency org="javax.jms" name="jms" rev="${jms.version}"/>
     <dependency org="org.apache.activemq" name="activemq-core" rev="${activemq.version}">
       <exclude org="org.springframework"/>
@@ -65,7 +72,6 @@
     <dependency org="junit" name="junit" rev="${junit.version}" conf="test->default"/>
     <dependency org="org.vafer" name="jdeb" rev="${jdeb.version}" conf="test->default"/>
     <dependency org="com.google.code.p.arat" name="rat-lib" rev="${rats-lib.version}" conf="test->default"/>
-    <dependency org="org.apache.hadoop" name="hadoop-test" rev="${hadoop.jars.version}" conf="test->default"/>
     <dependency org="org.apache.maven" name="maven-ant-tasks" rev="${maven-ant-tasks.version}" conf="test->*"/>
   </dependencies>
 </ivy-module>

Modified: incubator/hcatalog/trunk/ivy/libraries.properties
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/ivy/libraries.properties?rev=1374862&r1=1374861&r2=1374862&view=diff
==============================================================================
--- incubator/hcatalog/trunk/ivy/libraries.properties (original)
+++ incubator/hcatalog/trunk/ivy/libraries.properties Sun Aug 19 21:49:08 2012
@@ -35,11 +35,12 @@ datanucleus-rdbms.version=2.0.3
 derby.version=10.4.2.0
 fb303.version=0.7.0
 guava.version=11.0.2
-hadoop.jars.version=1.0.3
+hadoop20.version=1.0.3
+hadoop23.version=0.23.1
 hbase.version=0.92.0
 high-scale-lib.version=1.1.1
 hive.version=0.10.0-SNAPSHOT
-ivy.version=2.1.0
+ivy.version=2.2.0
 jackson.version=1.7.3
 javax-mgmt.version=1.1-rev-1
 jaxb-api.version=2.2.2

Added: incubator/hcatalog/trunk/shims/build.xml
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/shims/build.xml?rev=1374862&view=auto
==============================================================================
--- incubator/hcatalog/trunk/shims/build.xml (added)
+++ incubator/hcatalog/trunk/shims/build.xml Sun Aug 19 21:49:08 2012
@@ -0,0 +1,40 @@
+<?xml version="1.0"?>
+<project xmlns:ivy="antlib:org.apache.ivy.ant" name="shims" default="jar">
+  <property name="path.to.basedir" value="${basedir}/.."/>
+  <import file="../build-common.xml"/>
+
+  <property name="ivy.dir" location="${path.to.basedir}/ivy" />
+  <loadproperties srcfile="${ivy.dir}/libraries.properties"/>
+
+  <property name="parent.build.dir" location="${path.to.basedir}/build/classes"/>
+  <property name="build.ivy.dir" location="${build.dir}/ivy" />
+  <property name="build.ivy.lib.dir" location="${build.ivy.dir}/lib" />
+
+  <!-- sources and hadoop version for each shim -->
+  <property name="shims.include" value="20,23"/>
+  <property name="shims.0.20.sources" value="${basedir}/src/20/java" />
+  <property name="shims.0.23.sources" value="${basedir}/src/23/java" />
+  <path id="shims.0.20.hadoop.ivy.dir">
+    <fileset dir="${build.ivy.lib.dir}/hadoop20" includes="*.jar"/>
+    <fileset dir="${parent.build.dir}" includes="**/*"/>
+  </path>
+  <path id="shims.0.23.hadoop.ivy.dir">
+    <fileset dir="${build.ivy.lib.dir}/hadoop23" includes="*.jar"/>
+    <fileset dir="${parent.build.dir}" includes="**/*"/>
+  </path>
+
+  <target name="jar" depends="compile">
+    <!-- Classes are added to hcatalog jar -->
+  </target>
+
+  <target name="compile" depends="ivy-retrieve">
+    <echo message="Project: ${ant.project.name}"/>
+    <!-- TODO use for loop -->
+    <_javac srcDir="${shims.0.20.sources}"
+            destDir="${parent.build.dir}"
+            classPathRef="shims.0.20.hadoop.ivy.dir"/>
+    <_javac srcDir="${shims.0.23.sources}"
+            destDir="${parent.build.dir}"
+            classPathRef="shims.0.23.hadoop.ivy.dir"/>
+  </target>
+</project>

Added: incubator/hcatalog/trunk/shims/ivy.xml
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/shims/ivy.xml?rev=1374862&view=auto
==============================================================================
--- incubator/hcatalog/trunk/shims/ivy.xml (added)
+++ incubator/hcatalog/trunk/shims/ivy.xml Sun Aug 19 21:49:08 2012
@@ -0,0 +1,42 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<ivy-module version="2.0" xmlns:m="http://ant.apache.org/ivy/maven">
+  <info organisation="org.apache.hcatalog" module="${ant.project.name}" revision="${hcatalog.version}">
+    <license name="The Apache Software License, Version 2.0" url="http://www.apache.org/licenses/LICENSE-2.0.txt" />
+    <description homepage="http://hive.apache.org">
+        Apache HCatalog
+    </description>
+  </info>
+  <configurations defaultconf="hadoop20">
+    <conf name="hadoop20" visibility="private"/>
+    <conf name="hadoop23" visibility="private"/>
+  </configurations>
+  <dependencies>
+    <dependency org="org.apache.pig" name="pig" rev="${pig.version}" conf="hadoop20->*;hadoop23->*"/>
+
+    <!-- Hadoop 0.20 shim dependencies. Used for building 0.20 shims. -->
+    <dependency org="org.apache.hadoop" name="hadoop-tools" rev="${hadoop20.version}" conf="hadoop20->*">
+      <exclude org="commons-daemon" module="commons-daemon"/>
+    </dependency>
+    <!--<dependency org="org.apache.hadoop" name="hadoop-test" rev="${hadoop20.version}" conf="hadoop20->*">-->
+      <!--<exclude org="commons-daemon" module="commons-daemon"/>-->
+    <!--</dependency>-->
+
+    <!-- Hadoop 0.23 dependencies. Used both for shims and for building against Hadoop 0.23. -->
+    <dependency org="org.apache.hadoop" name="hadoop-common" rev="${hadoop23.version}" conf="hadoop23->*">
+      <exclude org="commons-daemon" module="commons-daemon"/>
+    </dependency>
+    <dependency org="org.apache.hadoop" name="hadoop-mapreduce-client-core" rev="${hadoop23.version}" conf="hadoop23->*">
+      <exclude org="commons-daemon" module="commons-daemon"/>
+    </dependency>
+    <dependency org="org.apache.hadoop" name="hadoop-archives" rev="${hadoop23.version}" conf="hadoop23->*">
+      <exclude org="commons-daemon" module="commons-daemon"/>
+    </dependency>
+    <dependency org="org.apache.hadoop" name="hadoop-hdfs" rev="${hadoop23.version}" conf="hadoop23->*">
+      <artifact name="hadoop-hdfs" ext="jar" />
+      <!--<artifact name="hadoop-hdfs" type="tests" ext="jar" m:classifier="tests"/>-->
+      <exclude org="commons-daemon" module="commons-daemon"/>
+    </dependency>
+
+    <conflict manager="all" />
+  </dependencies>
+</ivy-module>

Added: incubator/hcatalog/trunk/shims/src/20/java/org/apache/hcatalog/shims/HCatHadoopShims20S.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/shims/src/20/java/org/apache/hcatalog/shims/HCatHadoopShims20S.java?rev=1374862&view=auto
==============================================================================
--- incubator/hcatalog/trunk/shims/src/20/java/org/apache/hcatalog/shims/HCatHadoopShims20S.java (added)
+++ incubator/hcatalog/trunk/shims/src/20/java/org/apache/hcatalog/shims/HCatHadoopShims20S.java Sun Aug 19 21:49:08 2012
@@ -0,0 +1,128 @@
+package org.apache.hcatalog.shims;
+
+import java.io.IOException;
+import java.net.InetSocketAddress;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.filecache.DistributedCache;
+import org.apache.hadoop.mapred.JobTracker;
+import org.apache.hadoop.mapreduce.Job;
+import org.apache.hadoop.mapreduce.JobContext;
+import org.apache.hadoop.mapreduce.JobID;
+import org.apache.hadoop.mapreduce.JobStatus.State;
+import org.apache.hadoop.mapreduce.OutputFormat;
+import org.apache.hadoop.mapreduce.TaskAttemptContext;
+import org.apache.hadoop.mapreduce.TaskAttemptID;
+import org.apache.hadoop.mapreduce.TaskID;
+import org.apache.hadoop.util.Progressable;
+import org.apache.pig.ResourceSchema;
+
+public class HCatHadoopShims20S implements HCatHadoopShims {
+    @Override
+    public TaskID createTaskID() {
+        return new TaskID();
+    }
+
+    @Override
+    public TaskAttemptID createTaskAttemptID() {
+        return new TaskAttemptID();
+    }
+
+	@Override
+	public TaskAttemptContext createTaskAttemptContext(Configuration conf,
+			TaskAttemptID taskId) {
+        return new TaskAttemptContext(conf, taskId);
+    }
+
+    @Override
+    public org.apache.hadoop.mapred.TaskAttemptContext createTaskAttemptContext(org.apache.hadoop.mapred.JobConf conf,
+            org.apache.hadoop.mapred.TaskAttemptID taskId, Progressable progressable) {
+        org.apache.hadoop.mapred.TaskAttemptContext newContext = null;
+        try {
+            java.lang.reflect.Constructor construct = org.apache.hadoop.mapred.TaskAttemptContext.class.getDeclaredConstructor(
+                    org.apache.hadoop.mapred.JobConf.class, org.apache.hadoop.mapred.TaskAttemptID.class,
+                    Progressable.class);
+            construct.setAccessible(true);
+            newContext = (org.apache.hadoop.mapred.TaskAttemptContext)construct.newInstance(conf, taskId, progressable);
+        } catch (Exception e) {
+            throw new RuntimeException(e);
+        }
+        return newContext;
+    }
+
+    @Override
+    public JobContext createJobContext(Configuration conf,
+            JobID jobId) {
+        return new JobContext(conf, jobId);
+    }
+
+    @Override
+    public org.apache.hadoop.mapred.JobContext createJobContext(org.apache.hadoop.mapred.JobConf conf,
+            org.apache.hadoop.mapreduce.JobID jobId, Progressable progressable) {
+        org.apache.hadoop.mapred.JobContext newContext = null;
+        try {
+            java.lang.reflect.Constructor construct = org.apache.hadoop.mapred.JobContext.class.getDeclaredConstructor(
+                    org.apache.hadoop.mapred.JobConf.class, org.apache.hadoop.mapreduce.JobID.class,
+                    Progressable.class);
+            construct.setAccessible(true);
+            newContext = (org.apache.hadoop.mapred.JobContext)construct.newInstance(conf, jobId, progressable);
+        } catch (Exception e) {
+            throw new RuntimeException(e);
+        }
+        return newContext;
+    }
+
+    @Override
+    public void commitJob(OutputFormat outputFormat, ResourceSchema schema,
+            String arg1, Job job) throws IOException {
+        if( job.getConfiguration().get("mapred.job.tracker", "").equalsIgnoreCase("local") ) {
+            try {
+                //In local mode, mapreduce will not call OutputCommitter.cleanupJob.
+                //Calling it from here so that the partition publish happens.
+                //This call needs to be removed after MAPREDUCE-1447 is fixed.
+                outputFormat.getOutputCommitter(HCatHadoopShims.Instance.get().createTaskAttemptContext(
+                            job.getConfiguration(), HCatHadoopShims.Instance.get().createTaskAttemptID())).commitJob(job);
+            } catch (IOException e) {
+                throw new IOException("Failed to cleanup job",e);
+            } catch (InterruptedException e) {
+                throw new IOException("Failed to cleanup job",e);
+            }
+        }
+    }
+
+    @Override
+    public void abortJob(OutputFormat outputFormat, Job job) throws IOException {
+        if (job.getConfiguration().get("mapred.job.tracker", "")
+                .equalsIgnoreCase("local")) {
+            try {
+                // This call needs to be removed after MAPREDUCE-1447 is fixed.
+                outputFormat.getOutputCommitter(HCatHadoopShims.Instance.get().createTaskAttemptContext(
+                            job.getConfiguration(), new TaskAttemptID())).abortJob(job, State.FAILED);
+            } catch (IOException e) {
+                throw new IOException("Failed to abort job", e);
+            } catch (InterruptedException e) {
+                throw new IOException("Failed to abort job", e);
+            }
+        }
+    }
+
+    @Override
+    public InetSocketAddress getResourceManagerAddress(Configuration conf)
+    {
+        return JobTracker.getAddress(conf);
+    }
+
+    @Override
+    public String getPropertyName(PropertyName name) {
+        switch (name) {
+            case CACHE_ARCHIVES:
+                return DistributedCache.CACHE_ARCHIVES;
+            case CACHE_FILES:
+                return DistributedCache.CACHE_FILES;
+            case CACHE_SYMLINK:
+                return DistributedCache.CACHE_SYMLINK;
+        }
+
+        return "";
+    }
+}

Modified: incubator/hcatalog/trunk/shims/src/23/java/org/apache/hcatalog/shims/HCatHadoopShims23.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/shims/src/23/java/org/apache/hcatalog/shims/HCatHadoopShims23.java?rev=1374862&r1=1374861&r2=1374862&view=diff
==============================================================================
--- incubator/hcatalog/trunk/shims/src/23/java/org/apache/hcatalog/shims/HCatHadoopShims23.java (original)
+++ incubator/hcatalog/trunk/shims/src/23/java/org/apache/hcatalog/shims/HCatHadoopShims23.java Sun Aug 19 21:49:08 2012
@@ -17,27 +17,105 @@
  */
 package org.apache.hcatalog.shims;
 
+import java.io.IOException;
+import java.net.InetSocketAddress;
+
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.mapred.Reporter;
+import org.apache.hadoop.mapreduce.Job;
 import org.apache.hadoop.mapreduce.JobContext;
 import org.apache.hadoop.mapreduce.JobID;
+import org.apache.hadoop.mapreduce.OutputFormat;
 import org.apache.hadoop.mapreduce.TaskAttemptContext;
 import org.apache.hadoop.mapreduce.TaskAttemptID;
+import org.apache.hadoop.mapreduce.TaskID;
+import org.apache.hadoop.mapreduce.TaskType;
 import org.apache.hadoop.mapreduce.task.JobContextImpl;
 import org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl;
+import org.apache.hadoop.util.Progressable;
+import org.apache.pig.ResourceSchema;
+
+import org.apache.hadoop.mapreduce.MRJobConfig;
+import org.apache.hadoop.net.NetUtils;
 
 public class HCatHadoopShims23 implements HCatHadoopShims {
+    @Override
+    public TaskID createTaskID() {
+        return new TaskID("", 0, TaskType.MAP, 0);
+    }
 
-	@Override
-	public TaskAttemptContext createTaskAttemptContext(Configuration conf,
-			TaskAttemptID taskId) {
-        return new TaskAttemptContextImpl(conf, taskId);
-	}
+    @Override
+    public TaskAttemptID createTaskAttemptID() {
+        return new TaskAttemptID("", 0, TaskType.MAP, 0, 0);
+    }
+
+    @Override
+    public org.apache.hadoop.mapreduce.TaskAttemptContext createTaskAttemptContext(Configuration conf,
+            org.apache.hadoop.mapreduce.TaskAttemptID taskId) {
+        return new org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl(conf, taskId);
+    }
 
-	@Override
+    @Override
+    public org.apache.hadoop.mapred.TaskAttemptContext createTaskAttemptContext(org.apache.hadoop.mapred.JobConf conf,
+            org.apache.hadoop.mapred.TaskAttemptID taskId, Progressable progressable) {
+        org.apache.hadoop.mapred.TaskAttemptContext newContext = null;
+        try {
+            java.lang.reflect.Constructor construct = org.apache.hadoop.mapred.TaskAttemptContextImpl.class.getDeclaredConstructor(
+                    org.apache.hadoop.mapred.JobConf.class, org.apache.hadoop.mapred.TaskAttemptID.class,
+                    Reporter.class);
+            construct.setAccessible(true);
+            newContext = (org.apache.hadoop.mapred.TaskAttemptContext)construct.newInstance(conf, taskId, (Reporter)progressable);
+        } catch (Exception e) {
+            throw new RuntimeException(e);
+        }
+        return newContext;
+    }
+
+    @Override
     public JobContext createJobContext(Configuration conf,
             JobID jobId) {
-        JobContext newContext = new JobContextImpl(conf, jobId);
+        JobContext ctxt = new JobContextImpl(conf, jobId);
+
+        return ctxt;
+    }
+
+    @Override
+    public org.apache.hadoop.mapred.JobContext createJobContext(org.apache.hadoop.mapred.JobConf conf,
+            org.apache.hadoop.mapreduce.JobID jobId, Progressable progressable) {
+        org.apache.hadoop.mapred.JobContext newContext = 
+            new org.apache.hadoop.mapred.JobContextImpl(conf, jobId, (org.apache.hadoop.mapred.Reporter)progressable);
         return newContext;
     }
 
+    @Override
+    public void commitJob(OutputFormat outputFormat, ResourceSchema schema,
+            String arg1, Job job) throws IOException {
+        // Do nothing as this was fixed by MAPREDUCE-1447.
+    }
+
+    @Override
+    public void abortJob(OutputFormat outputFormat, Job job) throws IOException {
+        // Do nothing as this was fixed by MAPREDUCE-1447.
+    }
+
+    @Override
+    public InetSocketAddress getResourceManagerAddress(Configuration conf) {
+        String addr = conf.get("yarn.resourcemanager.address", "localhost:8032");
+
+        return NetUtils.createSocketAddr(addr);
+    }
+
+    @Override
+    public String getPropertyName(PropertyName name) {
+        switch (name) {
+            case CACHE_ARCHIVES:
+                return MRJobConfig.CACHE_ARCHIVES;
+            case CACHE_FILES:
+                return MRJobConfig.CACHE_FILES;
+            case CACHE_SYMLINK:
+                return MRJobConfig.CACHE_SYMLINK;
+        }
+
+        return "";
+    }
 }

Modified: incubator/hcatalog/trunk/src/java/org/apache/hadoop/mapred/HCatMapRedUtil.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/java/org/apache/hadoop/mapred/HCatMapRedUtil.java?rev=1374862&r1=1374861&r2=1374862&view=diff
==============================================================================
--- incubator/hcatalog/trunk/src/java/org/apache/hadoop/mapred/HCatMapRedUtil.java (original)
+++ incubator/hcatalog/trunk/src/java/org/apache/hadoop/mapred/HCatMapRedUtil.java Sun Aug 19 21:49:08 2012
@@ -19,6 +19,8 @@
 package org.apache.hadoop.mapred;
 
 import org.apache.hadoop.util.Progressable;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hcatalog.shims.HCatHadoopShims;
 
 public class HCatMapRedUtil {
 
@@ -28,8 +30,12 @@ public class HCatMapRedUtil {
                                                              Reporter.NULL);
     }
 
+    public static org.apache.hadoop.mapreduce.TaskAttemptContext createTaskAttemptContext(Configuration conf, org.apache.hadoop.mapreduce.TaskAttemptID id) {
+        return  HCatHadoopShims.Instance.get().createTaskAttemptContext(conf,id);
+    }
+
     public static TaskAttemptContext createTaskAttemptContext(JobConf conf, TaskAttemptID id, Progressable progressable) {
-        return  new TaskAttemptContext(conf,id,progressable);
+        return HCatHadoopShims.Instance.get ().createTaskAttemptContext(conf, id, (Reporter) progressable);
     }
 
     public static org.apache.hadoop.mapred.JobContext createJobContext(org.apache.hadoop.mapreduce.JobContext context) {
@@ -39,6 +45,6 @@ public class HCatMapRedUtil {
     }
 
     public static JobContext createJobContext(JobConf conf, org.apache.hadoop.mapreduce.JobID id, Progressable progressable) {
-        return  new JobContext(conf,id,progressable);
+        return HCatHadoopShims.Instance.get ().createJobContext(conf, id, (Reporter) progressable);
     }
 }

Modified: incubator/hcatalog/trunk/src/java/org/apache/hcatalog/data/transfer/impl/HCatInputFormatReader.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/java/org/apache/hcatalog/data/transfer/impl/HCatInputFormatReader.java?rev=1374862&r1=1374861&r2=1374862&view=diff
==============================================================================
--- incubator/hcatalog/trunk/src/java/org/apache/hcatalog/data/transfer/impl/HCatInputFormatReader.java (original)
+++ incubator/hcatalog/trunk/src/java/org/apache/hcatalog/data/transfer/impl/HCatInputFormatReader.java Sun Aug 19 21:49:08 2012
@@ -39,6 +39,7 @@ import org.apache.hcatalog.data.transfer
 import org.apache.hcatalog.data.transfer.state.StateProvider;
 import org.apache.hcatalog.mapreduce.HCatInputFormat;
 import org.apache.hcatalog.mapreduce.InputJobInfo;
+import org.apache.hcatalog.shims.HCatHadoopShims;
 
 /**
  * This reader reads via {@link HCatInputFormat}
@@ -68,8 +69,8 @@ public class HCatInputFormatReader exten
       HCatInputFormat.setInput(job, jobInfo);
       HCatInputFormat hcif = new HCatInputFormat();
       ReaderContext cntxt = new ReaderContext();
-      cntxt.setInputSplits(hcif.getSplits(new JobContext(
-          job.getConfiguration(), null)));
+      cntxt.setInputSplits(hcif.getSplits(
+                  HCatHadoopShims.Instance.get().createJobContext(job.getConfiguration(), null)));
       cntxt.setConf(job.getConfiguration());
       return cntxt;
     } catch (IOException e) {
@@ -85,8 +86,7 @@ public class HCatInputFormatReader exten
     HCatInputFormat inpFmt = new HCatInputFormat();
     RecordReader<WritableComparable, HCatRecord> rr;
     try {
-      TaskAttemptContext cntxt = new TaskAttemptContext(conf,
-          new TaskAttemptID());
+      TaskAttemptContext cntxt = HCatHadoopShims.Instance.get().createTaskAttemptContext(conf, new TaskAttemptID());
       rr = inpFmt.createRecordReader(split, cntxt);
       rr.initialize(split, cntxt);
     } catch (IOException e) {

Modified: incubator/hcatalog/trunk/src/java/org/apache/hcatalog/data/transfer/impl/HCatOutputFormatWriter.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/java/org/apache/hcatalog/data/transfer/impl/HCatOutputFormatWriter.java?rev=1374862&r1=1374861&r2=1374862&view=diff
==============================================================================
--- incubator/hcatalog/trunk/src/java/org/apache/hcatalog/data/transfer/impl/HCatOutputFormatWriter.java (original)
+++ incubator/hcatalog/trunk/src/java/org/apache/hcatalog/data/transfer/impl/HCatOutputFormatWriter.java Sun Aug 19 21:49:08 2012
@@ -32,6 +32,7 @@ import org.apache.hadoop.mapreduce.Recor
 import org.apache.hadoop.mapreduce.TaskAttemptContext;
 import org.apache.hadoop.mapreduce.TaskAttemptID;
 import org.apache.hadoop.mapreduce.TaskID;
+import org.apache.hadoop.mapreduce.TaskType;
 import org.apache.hcatalog.common.ErrorType;
 import org.apache.hcatalog.common.HCatException;
 import org.apache.hcatalog.data.HCatRecord;
@@ -41,6 +42,7 @@ import org.apache.hcatalog.data.transfer
 import org.apache.hcatalog.data.transfer.state.StateProvider;
 import org.apache.hcatalog.mapreduce.HCatOutputFormat;
 import org.apache.hcatalog.mapreduce.OutputJobInfo;
+import org.apache.hcatalog.shims.HCatHadoopShims;
 
 /**
  * This writer writes via {@link HCatOutputFormat}
@@ -67,9 +69,8 @@ public class HCatOutputFormatWriter exte
       HCatOutputFormat.setSchema(job, HCatOutputFormat.getTableSchema(job));
       HCatOutputFormat outFormat = new HCatOutputFormat();
       outFormat.checkOutputSpecs(job);
-      outFormat.getOutputCommitter(
-          new TaskAttemptContext(job.getConfiguration(), new TaskAttemptID()))
-          .setupJob(job);
+      outFormat.getOutputCommitter(HCatHadoopShims.Instance.get().createTaskAttemptContext
+              (job.getConfiguration(), HCatHadoopShims.Instance.get().createTaskAttemptID())).setupJob(job);
     } catch (IOException e) {
       throw new HCatException(ErrorType.ERROR_NOT_INITIALIZED, e);
     } catch (InterruptedException e) {
@@ -86,8 +87,8 @@ public class HCatOutputFormatWriter exte
     int id = sp.getId();
     setVarsInConf(id);
     HCatOutputFormat outFormat = new HCatOutputFormat();
-    TaskAttemptContext cntxt = new TaskAttemptContext(conf, new TaskAttemptID(
-        new TaskID(), id));
+    TaskAttemptContext cntxt = HCatHadoopShims.Instance.get().createTaskAttemptContext
+        (conf, new TaskAttemptID(HCatHadoopShims.Instance.get().createTaskID(), id));
     OutputCommitter committer = null;
     RecordWriter<WritableComparable<?>, HCatRecord> writer;
     try {
@@ -126,9 +127,9 @@ public class HCatOutputFormatWriter exte
   @Override
   public void commit(WriterContext context) throws HCatException {
     try {
-      new HCatOutputFormat().getOutputCommitter(
-          new TaskAttemptContext(context.getConf(), new TaskAttemptID()))
-          .commitJob(new JobContext(context.getConf(), null));
+      new HCatOutputFormat().getOutputCommitter(HCatHadoopShims.Instance.get().createTaskAttemptContext
+              (context.getConf(), HCatHadoopShims.Instance.get().createTaskAttemptID()))
+          .commitJob(HCatHadoopShims.Instance.get().createJobContext(context.getConf(), null));
     } catch (IOException e) {
       throw new HCatException(ErrorType.ERROR_NOT_INITIALIZED, e);
     } catch (InterruptedException e) {
@@ -139,9 +140,9 @@ public class HCatOutputFormatWriter exte
   @Override
   public void abort(WriterContext context) throws HCatException {
     try {
-      new HCatOutputFormat().getOutputCommitter(
-          new TaskAttemptContext(context.getConf(), new TaskAttemptID()))
-          .abortJob(new JobContext(context.getConf(), null), State.FAILED);
+      new HCatOutputFormat().getOutputCommitter(HCatHadoopShims.Instance.get().createTaskAttemptContext
+              (context.getConf(), HCatHadoopShims.Instance.get().createTaskAttemptID()))
+          .abortJob(HCatHadoopShims.Instance.get().createJobContext(context.getConf(), null),State.FAILED);
     } catch (IOException e) {
       throw new HCatException(ErrorType.ERROR_NOT_INITIALIZED, e);
     } catch (InterruptedException e) {

Modified: incubator/hcatalog/trunk/src/java/org/apache/hcatalog/data/transfer/state/StateProvider.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/java/org/apache/hcatalog/data/transfer/state/StateProvider.java?rev=1374862&r1=1374861&r2=1374862&view=diff
==============================================================================
--- incubator/hcatalog/trunk/src/java/org/apache/hcatalog/data/transfer/state/StateProvider.java (original)
+++ incubator/hcatalog/trunk/src/java/org/apache/hcatalog/data/transfer/state/StateProvider.java Sun Aug 19 21:49:08 2012
@@ -18,9 +18,6 @@
 
 package org.apache.hcatalog.data.transfer.state;
 
-import org.apache.hadoop.mapred.JobTracker;
-import org.apache.hadoop.mapred.TaskTracker;
-
 /**
  * If external system wants to communicate any state to slaves, they can do so
  * via this interface. One example of this in case of Map-Reduce is ids assigned

Modified: incubator/hcatalog/trunk/src/java/org/apache/hcatalog/mapreduce/FileOutputCommitterContainer.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/java/org/apache/hcatalog/mapreduce/FileOutputCommitterContainer.java?rev=1374862&r1=1374861&r2=1374862&view=diff
==============================================================================
--- incubator/hcatalog/trunk/src/java/org/apache/hcatalog/mapreduce/FileOutputCommitterContainer.java (original)
+++ incubator/hcatalog/trunk/src/java/org/apache/hcatalog/mapreduce/FileOutputCommitterContainer.java Sun Aug 19 21:49:08 2012
@@ -47,11 +47,16 @@ import org.apache.hcatalog.data.schema.H
 import org.apache.hcatalog.data.schema.HCatSchema;
 import org.apache.hcatalog.data.schema.HCatSchemaUtils;
 import org.apache.hcatalog.har.HarOutputCommitterPostProcessor;
+import org.apache.hcatalog.shims.HCatHadoopShims;
 import org.apache.thrift.TException;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import java.io.BufferedWriter;
+import java.io.FileWriter;
 import java.io.IOException;
+import java.io.PrintWriter;
+import java.io.Writer;
 import java.net.URI;
 import java.util.ArrayList;
 import java.util.HashMap;
@@ -664,7 +669,7 @@ class FileOutputCommitterContainer exten
                     LinkedHashMap<String, String> fullPartSpec = new LinkedHashMap<String, String>();
                     Warehouse.makeSpecFromName(fullPartSpec, st.getPath());
                     partitionsDiscoveredByPath.put(st.getPath().toString(),fullPartSpec);
-                    JobContext currContext = new JobContext(context.getConfiguration(),context.getJobID());
+                    JobContext currContext = HCatHadoopShims.Instance.get().createJobContext(context.getConfiguration(),context.getJobID());
                     HCatOutputFormat.configureOutputStorageHandler(context, jobInfo, fullPartSpec);
                     contextDiscoveredByPath.put(st.getPath().toString(),currContext);
                 }

Modified: incubator/hcatalog/trunk/src/java/org/apache/hcatalog/mapreduce/FileRecordWriterContainer.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/java/org/apache/hcatalog/mapreduce/FileRecordWriterContainer.java?rev=1374862&r1=1374861&r2=1374862&view=diff
==============================================================================
--- incubator/hcatalog/trunk/src/java/org/apache/hcatalog/mapreduce/FileRecordWriterContainer.java (original)
+++ incubator/hcatalog/trunk/src/java/org/apache/hcatalog/mapreduce/FileRecordWriterContainer.java Sun Aug 19 21:49:08 2012
@@ -145,6 +145,8 @@ class FileRecordWriterContainer extends 
                 if (baseOutputCommitter.needsTaskCommit(currContext)){
                     baseOutputCommitter.commitTask(currContext);
                 }
+                org.apache.hadoop.mapred.JobContext currJobContext = HCatMapRedUtil.createJobContext(currContext);
+                baseOutputCommitter.commitJob(currJobContext);
             }
         } else {
             getBaseRecordWriter().close(reporter);

Modified: incubator/hcatalog/trunk/src/java/org/apache/hcatalog/mapreduce/MultiOutputFormat.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/java/org/apache/hcatalog/mapreduce/MultiOutputFormat.java?rev=1374862&r1=1374861&r2=1374862&view=diff
==============================================================================
--- incubator/hcatalog/trunk/src/java/org/apache/hcatalog/mapreduce/MultiOutputFormat.java (original)
+++ incubator/hcatalog/trunk/src/java/org/apache/hcatalog/mapreduce/MultiOutputFormat.java Sun Aug 19 21:49:08 2012
@@ -47,6 +47,7 @@ import org.apache.hadoop.mapreduce.Recor
 import org.apache.hadoop.mapreduce.TaskAttemptContext;
 import org.apache.hadoop.mapreduce.TaskInputOutputContext;
 import org.apache.hadoop.util.ReflectionUtils;
+import org.apache.hcatalog.shims.HCatHadoopShims;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -144,13 +145,13 @@ public class MultiOutputFormat extends O
 
     static {
         configsToOverride.add("mapred.output.dir");
-        configsToOverride.add(DistributedCache.CACHE_SYMLINK);
+        configsToOverride.add(HCatHadoopShims.Instance.get().getPropertyName(HCatHadoopShims.PropertyName.CACHE_SYMLINK));
         configsToMerge.put(JobContext.JOB_NAMENODES, COMMA_DELIM);
         configsToMerge.put("tmpfiles", COMMA_DELIM);
         configsToMerge.put("tmpjars", COMMA_DELIM);
         configsToMerge.put("tmparchives", COMMA_DELIM);
-        configsToMerge.put(DistributedCache.CACHE_ARCHIVES, COMMA_DELIM);
-        configsToMerge.put(DistributedCache.CACHE_FILES, COMMA_DELIM);
+        configsToMerge.put(HCatHadoopShims.Instance.get().getPropertyName(HCatHadoopShims.PropertyName.CACHE_ARCHIVES), COMMA_DELIM);
+        configsToMerge.put(HCatHadoopShims.Instance.get().getPropertyName(HCatHadoopShims.PropertyName.CACHE_FILES), COMMA_DELIM);
         configsToMerge.put("mapred.job.classpath.archives", System.getProperty("path.separator"));
         configsToMerge.put("mapred.job.classpath.files", System.getProperty("path.separator"));
     }
@@ -175,7 +176,7 @@ public class MultiOutputFormat extends O
      */
     public static JobContext getJobContext(String alias, JobContext context) {
         String aliasConf = context.getConfiguration().get(getAliasConfName(alias));
-        JobContext aliasContext = new JobContext(context.getConfiguration(), context.getJobID());
+        JobContext aliasContext = HCatHadoopShims.Instance.get().createJobContext(context.getConfiguration(), context.getJobID());
         addToConfig(aliasConf, aliasContext.getConfiguration());
         return aliasContext;
     }
@@ -189,8 +190,8 @@ public class MultiOutputFormat extends O
      */
     public static TaskAttemptContext getTaskAttemptContext(String alias, TaskAttemptContext context) {
         String aliasConf = context.getConfiguration().get(getAliasConfName(alias));
-        TaskAttemptContext aliasContext = new TaskAttemptContext(context.getConfiguration(),
-                context.getTaskAttemptID());
+        TaskAttemptContext aliasContext = HCatHadoopShims.Instance.get().createTaskAttemptContext(
+                context.getConfiguration(), context.getTaskAttemptID());
         addToConfig(aliasConf, aliasContext.getConfiguration());
         return aliasContext;
     }

Modified: incubator/hcatalog/trunk/src/java/org/apache/hcatalog/mapreduce/ProgressReporter.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/java/org/apache/hcatalog/mapreduce/ProgressReporter.java?rev=1374862&r1=1374861&r2=1374862&view=diff
==============================================================================
--- incubator/hcatalog/trunk/src/java/org/apache/hcatalog/mapreduce/ProgressReporter.java (original)
+++ incubator/hcatalog/trunk/src/java/org/apache/hcatalog/mapreduce/ProgressReporter.java Sun Aug 19 21:49:08 2012
@@ -74,6 +74,12 @@ class ProgressReporter extends StatusRep
     return null;
   }
 
+  public float getProgress() {
+      /* Required to build against 0.23 Reporter and StatusReporter. */
+      /* TODO: determine the progress. */
+      return 0.0f;
+  }
+
   @Override
   public void progress() {
     if (context != null) {

Modified: incubator/hcatalog/trunk/src/java/org/apache/hcatalog/mapreduce/Security.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/java/org/apache/hcatalog/mapreduce/Security.java?rev=1374862&r1=1374861&r2=1374862&view=diff
==============================================================================
--- incubator/hcatalog/trunk/src/java/org/apache/hcatalog/mapreduce/Security.java (original)
+++ incubator/hcatalog/trunk/src/java/org/apache/hcatalog/mapreduce/Security.java Sun Aug 19 21:49:08 2012
@@ -29,7 +29,6 @@ import org.apache.hadoop.hive.metastore.
 import org.apache.hadoop.hive.metastore.api.MetaException;
 import org.apache.hadoop.hive.thrift.DelegationTokenSelector;
 import org.apache.hadoop.io.Text;
-import org.apache.hadoop.mapred.JobTracker;
 import org.apache.hadoop.mapreduce.Job;
 import org.apache.hadoop.mapreduce.JobContext;
 import org.apache.hadoop.security.UserGroupInformation;
@@ -38,6 +37,7 @@ import org.apache.hadoop.security.token.
 import org.apache.hadoop.security.token.TokenSelector;
 import org.apache.hcatalog.common.HCatConstants;
 import org.apache.hcatalog.common.HCatUtil;
+import org.apache.hcatalog.shims.HCatHadoopShims;
 import org.apache.thrift.TException;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -139,9 +139,8 @@ final class Security {
         if (harRequested){
           TokenSelector<? extends TokenIdentifier> jtTokenSelector =
             new org.apache.hadoop.mapreduce.security.token.delegation.DelegationTokenSelector();
-          Token jtToken =
-            jtTokenSelector.selectToken(org.apache.hadoop.security.SecurityUtil.buildTokenService(JobTracker.getAddress(conf)),
-                                            ugi.getTokens());
+          Token jtToken = jtTokenSelector.selectToken(org.apache.hadoop.security.SecurityUtil.buildTokenService(
+                      HCatHadoopShims.Instance.get().getResourceManagerAddress(conf)), ugi.getTokens());
           if(jtToken == null) {
             //we don't need to cancel this token as the TokenRenewer for JT tokens
             //takes care of cancelling them

Modified: incubator/hcatalog/trunk/src/java/org/apache/hcatalog/shims/HCatHadoopShims.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/java/org/apache/hcatalog/shims/HCatHadoopShims.java?rev=1374862&r1=1374861&r2=1374862&view=diff
==============================================================================
--- incubator/hcatalog/trunk/src/java/org/apache/hcatalog/shims/HCatHadoopShims.java (original)
+++ incubator/hcatalog/trunk/src/java/org/apache/hcatalog/shims/HCatHadoopShims.java Sun Aug 19 21:49:08 2012
@@ -17,12 +17,21 @@
  */
 package org.apache.hcatalog.shims;
 
+import java.io.IOException;
+import java.net.InetSocketAddress;
+
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hive.shims.ShimLoader;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.mapreduce.Job;
 import org.apache.hadoop.mapreduce.JobContext;
 import org.apache.hadoop.mapreduce.JobID;
+import org.apache.hadoop.mapreduce.OutputFormat;
 import org.apache.hadoop.mapreduce.TaskAttemptContext;
 import org.apache.hadoop.mapreduce.TaskAttemptID;
+import org.apache.hadoop.mapreduce.TaskID;
+import org.apache.hadoop.util.Progressable;
+import org.apache.pig.ResourceSchema;
 
 /**
  * Shim layer to abstract differences between Hadoop 0.20 and 0.23
@@ -31,6 +40,8 @@ import org.apache.hadoop.mapreduce.TaskA
  **/
 public interface HCatHadoopShims {
 
+  enum PropertyName { CACHE_ARCHIVES, CACHE_FILES, CACHE_SYMLINK };
+
   public static abstract class Instance {
     static HCatHadoopShims instance = selectShim();
 
@@ -55,9 +66,27 @@ public interface HCatHadoopShims {
     }
   }
 
-  public TaskAttemptContext createTaskAttemptContext(Configuration conf,
-      TaskAttemptID taskId);
+  public TaskID createTaskID();
+
+  public TaskAttemptID createTaskAttemptID();
+
+  public org.apache.hadoop.mapreduce.TaskAttemptContext createTaskAttemptContext(Configuration conf,
+          TaskAttemptID taskId);
+
+  public org.apache.hadoop.mapred.TaskAttemptContext createTaskAttemptContext(JobConf conf,
+          org.apache.hadoop.mapred.TaskAttemptID taskId, Progressable progressable);
 
   public JobContext createJobContext(Configuration conf, JobID jobId);
 
+  public org.apache.hadoop.mapred.JobContext createJobContext(JobConf conf, JobID jobId, Progressable progressable);
+
+  public void commitJob(OutputFormat outputFormat, ResourceSchema schema,
+          String arg1, Job job) throws IOException;
+
+  public void abortJob(OutputFormat outputFormat, Job job) throws IOException;
+
+  /* Referring to job tracker in 0.20 and resource manager in 0.23 */
+  public InetSocketAddress getResourceManagerAddress(Configuration conf);
+
+  public String getPropertyName(PropertyName name);
 }

Modified: incubator/hcatalog/trunk/src/test/e2e/hcatalog/build.xml
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/test/e2e/hcatalog/build.xml?rev=1374862&r1=1374861&r2=1374862&view=diff
==============================================================================
--- incubator/hcatalog/trunk/src/test/e2e/hcatalog/build.xml (original)
+++ incubator/hcatalog/trunk/src/test/e2e/hcatalog/build.xml Sun Aug 19 21:49:08 2012
@@ -49,6 +49,12 @@
   <!-- Default value for output directory -->
   <property name="harness.PH_LOCAL" value="out"/>
 
+  <property name="hadoopversion" value="20" />
+
+  <condition property="isHadoop23">
+    <equals arg1="${hadoopversion}" arg2="23"/>
+  </condition>
+
   <!-- Build the UDFs -->
   <target name="udfs" >
     <ant dir="${udf.java.dir}"/>
@@ -56,7 +62,14 @@
 
   <path id="hadoop.core.jar.location">
     <fileset dir="${hadoop.core.path}">
-      <include name="hadoop-core-*.jar"/>
+      <include name="hadoop-core-*.jar" unless="isHadoop23"/>
+      <include name="**/hadoop-common-*.jar" if="isHadoop23"/>
+      <include name="**/hadoop-auth-*.jar" if="isHadoop23"/>
+      <include name="**/hadoop-hdfs-*.jar" if="isHadoop23"/>
+      <include name="**/hadoop-mapreduce-client-core-*.jar" if="isHadoop23"/>
+      <include name="**/hadoop-yarn-api-*.jar" if="isHadoop23"/>
+      <include name="**/hadoop-yarn-common-*.jar" if="isHadoop23"/>
+      <include name="**/hadoop-annotations-*.jar" if="isHadoop23"/>
     </fileset>
   </path>
 
@@ -161,6 +174,18 @@
         </and>
       </condition>
     </fail>
+    <fail message="Please set the property harness.cluster.conf to the location Hadoop conf is installed ">
+      <condition>
+        <and>
+          <not>
+            <isset property="harness.cluster.conf"/>
+          </not>
+          <not>
+            <contains string="${harness.conf}" substring="rpm.conf"/>
+          </not>
+        </and>
+      </condition>
+    </fail>
     <fail message="Please set the property hive.home to the location Hive is installed ">
       <condition>
         <and>
@@ -236,6 +261,7 @@
       <env key="HARNESS_ROOT" value="."/>
       <env key="PH_LOCAL" value="${harness.PH_LOCAL}"/>
       <env key="HADOOP_HOME" value="${hadoop.home}"/>
+      <env key="HADOOP_CONF_DIR" value="${harness.cluster.conf}"/>
       <env key="HIVE_HOME" value="${hive.home}"/>
       <env key="HCAT_HOME" value="${hcat.home}"/>
       <env key="PIG_HOME" value="${pig.home}"/>

Modified: incubator/hcatalog/trunk/src/test/e2e/hcatalog/conf/default.conf
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/test/e2e/hcatalog/conf/default.conf?rev=1374862&r1=1374861&r2=1374862&view=diff
==============================================================================
--- incubator/hcatalog/trunk/src/test/e2e/hcatalog/conf/default.conf (original)
+++ incubator/hcatalog/trunk/src/test/e2e/hcatalog/conf/default.conf Sun Aug 19 21:49:08 2012
@@ -61,7 +61,7 @@ $cfg = {
     , 'pigbin'           => "$ENV{'PIG_HOME'}/bin/pig"
 
     #HADOOP
-    , 'hadoopconfdir'    => "$ENV{'HADOOP_HOME'}/conf"
+	, 'hadoopconfdir'    => "$ENV{'HADOOP_CONF_DIR'}"
     , 'hadoopbin'        => "$ENV{'HADOOP_HOME'}/bin/hadoop"
 
     #HIVE

Modified: incubator/hcatalog/trunk/src/test/e2e/hcatalog/drivers/Util.pm
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/test/e2e/hcatalog/drivers/Util.pm?rev=1374862&r1=1374861&r2=1374862&view=diff
==============================================================================
--- incubator/hcatalog/trunk/src/test/e2e/hcatalog/drivers/Util.pm (original)
+++ incubator/hcatalog/trunk/src/test/e2e/hcatalog/drivers/Util.pm Sun Aug 19 21:49:08 2012
@@ -357,6 +357,7 @@ sub replaceParameters
     # $testCmd
     $cmd =~ s/:INPATH:/$testCmd->{'inpathbase'}/g;
     $cmd =~ s/:OUTPATH:/$outfile/g;
+    $cmd =~ s/:OUTPATHPARENT:/$testCmd->{'outpath'}/g;
     $cmd =~ s/:FUNCPATH:/$testCmd->{'funcjarPath'}/g;
     $cmd =~ s/:PIGPATH:/$testCmd->{'pighome'}/g;
     $cmd =~ s/:RUNID:/$testCmd->{'UID'}/g;

Modified: incubator/hcatalog/trunk/src/test/e2e/hcatalog/tests/hcat.conf
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/test/e2e/hcatalog/tests/hcat.conf?rev=1374862&r1=1374861&r2=1374862&view=diff
==============================================================================
--- incubator/hcatalog/trunk/src/test/e2e/hcatalog/tests/hcat.conf (original)
+++ incubator/hcatalog/trunk/src/test/e2e/hcatalog/tests/hcat.conf Sun Aug 19 21:49:08 2012
@@ -135,7 +135,7 @@ gpa double)
 stored as textfile;
 drop table hcat_droptable_1;
 describe hcat_droptable_1;\
-                                ,'expected_out_regex' => 'does not exist'
+                                ,'expected_err_regex' => 'Table not found'
                                 },
                                 {
                                  'num' => 2
@@ -146,15 +146,16 @@ gpa double)
 stored as textfile;
 drop table if exists hcat_droptable_2;
 describe hcat_droptable_2;\,
-                                ,'rc'   => 0
-                                ,'expected_out_regex' => 'does not exist'
+                                ,'rc'   => 17 
+                                ,'expected_err_regex' => 'Table not found'
                                 },
 
                                 {
 				 'num' => 3
                                 ,'hcat' => q\
 drop table if exists hcat_drop_table_4;
-dfs -cp :INPATH:/studentnull10k/ :OUTPATH:/../drop_table_ext;
+dfs -mkdir :OUTPATHPARENT:/drop_table_ext;
+dfs -cp :INPATH:/studentnull10k/ :OUTPATHPARENT:drop_table_ext;
 \,
 				,'rc'   => 0
 				}, 
@@ -162,7 +163,7 @@ dfs -cp :INPATH:/studentnull10k/ :OUTPAT
 				 'num' => 4
                                 ,'depends_on' => 'HCat_DropTable_3'
                                 ,'hcat' => q\
-create external table hcat_drop_table_4(name string, age int, gpa double) stored as textfile location 'hdfs://:OUTPATH:/../drop_table_ext';
+create external table hcat_drop_table_4(name string, age int, gpa double) stored as textfile location 'hdfs://:OUTPATHPARENT:drop_table_ext';
 describe extended hcat_drop_table_4;
 \,
 				,'rc'   => 0
@@ -180,7 +181,7 @@ drop table hcat_drop_table_4;
 				 'num' => 6
                                 ,'depends_on' => 'HCat_DropTable_5'
                                 ,'hcat' => q\
-dfs -ls :OUTPATH:/../drop_table_ext
+dfs -ls :OUTPATHPARENT:drop_table_ext
 \,
 				,'rc'   => 0
                                 ,'expected_out_regex' => '(.*(\s))*.*drop_table_ext/studentnull10k'

Modified: incubator/hcatalog/trunk/src/test/e2e/hcatalog/tools/generate/generate_data.pl
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/test/e2e/hcatalog/tools/generate/generate_data.pl?rev=1374862&r1=1374861&r2=1374862&view=diff
==============================================================================
--- incubator/hcatalog/trunk/src/test/e2e/hcatalog/tools/generate/generate_data.pl (original)
+++ incubator/hcatalog/trunk/src/test/e2e/hcatalog/tools/generate/generate_data.pl Sun Aug 19 21:49:08 2012
@@ -326,44 +326,16 @@ location '$location'
     }
 }
 
-our $hadoopCoreJar = undef;
-
-sub findHadoopJars()
-{
-    my $hadoopClassRoot=$ENV{'HADOOP_HOME'};
-    my $coreJar = `ls $hadoopClassRoot/hadoop-core-*.jar`;
-    #if you do not find hadoop core jar under hadoop home change the path for rpm's
-    if (! $coreJar) {
-      $hadoopClassRoot="$hadoopClassRoot/share/hadoop";
-      $coreJar = `ls $hadoopClassRoot/hadoop-core-*.jar`;
-    }
-
-    my $cfgJar = `ls $hadoopClassRoot/lib/commons-configuration-*.jar`;
-    my $langJar = `ls $hadoopClassRoot/lib/commons-lang-*.jar`;
-    my $cliJar = `ls $hadoopClassRoot/lib/commons-cli-*.jar`;
-
-    if (! $coreJar) {
-        die 'Please set $HADOOP_HOME\n';
-    }
-
-    chomp $coreJar;
-    chomp $cfgJar;
-    chomp $langJar;
-    chomp $cliJar;
-    return ($coreJar, $cfgJar, $langJar, $cliJar);
-}
-
-sub findHiveJars()
+sub findAllJars()
 {
-    if (not defined $ENV{'HIVE_HOME'}) {
-        die 'Please set $HIVE_HOME\n';
+    my @files = <../../../../../build/ivy/lib/default/*.jar>;
+    my $classpath = "";
+    my $file = undef;
+    foreach $file (@files) {
+        $classpath = $classpath . ":" . $file;
     }
 
-    my $execJar = `ls $ENV{HIVE_HOME}/lib/hive-exec-*.jar`;
-    my $cliJar = `ls $ENV{HIVE_HOME}/lib/hive-cli-*.jar`;
-    chomp $execJar;
-    chomp $cliJar;
-    return ($execJar, $cliJar);
+    return $classpath;
 }
 
 sub getJavaCmd() 
@@ -428,13 +400,9 @@ sub getJavaCmd() 
             }
         } elsif ($format eq "rc") {
             print MYSQL &getBulkCopyCmd($tableName, "\t", "$tableName.plain");
-            my ($hadoopCoreJar, $commonsConfigJar,
-                $commonsLangJar, $commonsCliJar) = findHadoopJars();
-            my ($hiveExecJar, $hiveCliJar) = findHiveJars();
+            my $allJars = findAllJars();
             my @cmd = (getJavaCmd(), '-cp',
-                "../tools/generate/java/hive-gen.jar:$hadoopCoreJar:" .
-                "$commonsConfigJar:$commonsLangJar:" .
-                "$hiveExecJar",
+                "../tools/generate/java/hive-gen.jar:$allJars",
                 'org.apache.hadoop.hive.tools.generate.RCFileGenerator',
                 'student', $numRows, "$tableName", "$tableName.plain");
             run(\@cmd) or die "Unable to run command [" . join(" ", @cmd) 

Modified: incubator/hcatalog/trunk/src/test/e2e/hcatalog/tools/generate/java/build.xml
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/test/e2e/hcatalog/tools/generate/java/build.xml?rev=1374862&r1=1374861&r2=1374862&view=diff
==============================================================================
--- incubator/hcatalog/trunk/src/test/e2e/hcatalog/tools/generate/java/build.xml (original)
+++ incubator/hcatalog/trunk/src/test/e2e/hcatalog/tools/generate/java/build.xml Sun Aug 19 21:49:08 2012
@@ -17,7 +17,7 @@
 
     <property name="generator.jarfile" value="hive-gen.jar" />
     <property name="generator.build.dir" value="${basedir}/build" />
-    <property name="generator.src.dir" value="${basedir}/org/" />
+    <property name="generator.src.dir" value="${basedir}/org" />
 
 
     <path id="generator-classpath">
@@ -38,7 +38,7 @@
     <target name="generator-compile"
             depends="init, serde.jar.check, ql.jar.check, hadoop.jar.check">
         <echo>*** Compiling UDFs ***</echo>
-        <javac srcdir="${generator.src.dir}" destdir="${generator.build.dir}" debug="on">
+        <javac srcdir="${generator.src.dir}" destdir="${generator.build.dir}" debug="on" includeantruntime="false" includes="**/*.java">
             <classpath refid="generator-classpath" />
         </javac>
     </target>

Modified: incubator/hcatalog/trunk/src/test/e2e/hcatalog/udfs/java/build.xml
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/test/e2e/hcatalog/udfs/java/build.xml?rev=1374862&r1=1374861&r2=1374862&view=diff
==============================================================================
--- incubator/hcatalog/trunk/src/test/e2e/hcatalog/udfs/java/build.xml (original)
+++ incubator/hcatalog/trunk/src/test/e2e/hcatalog/udfs/java/build.xml Sun Aug 19 21:49:08 2012
@@ -21,7 +21,7 @@
 
     <path id="udf-classpath">
        <fileset file="../../../../../../build/hcatalog/*.jar" />
-       <fileset file="../../../../../../build/ivy/lib/hcatalog/*.jar" />
+       <fileset file="../../../../../../build/ivy/lib/default/*.jar" />
     </path>
 
     <target name="init">

Modified: incubator/hcatalog/trunk/src/test/e2e/hcatalog/udfs/java/org/apache/hcatalog/utils/SimpleRead.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/test/e2e/hcatalog/udfs/java/org/apache/hcatalog/utils/SimpleRead.java?rev=1374862&r1=1374861&r2=1374862&view=diff
==============================================================================
--- incubator/hcatalog/trunk/src/test/e2e/hcatalog/udfs/java/org/apache/hcatalog/utils/SimpleRead.java (original)
+++ incubator/hcatalog/trunk/src/test/e2e/hcatalog/udfs/java/org/apache/hcatalog/utils/SimpleRead.java Sun Aug 19 21:49:08 2012
@@ -66,7 +66,6 @@ public class SimpleRead extends Configur
           Text,IntWritable>.Context context) 
     throws IOException ,InterruptedException {
         name = (String) value.get(0);
-System.out.println(name);
         age = (Integer) value.get(1);
         gpa = (Double) value.get(2);
         context.write(new Text(name), new IntWritable(age));

Modified: incubator/hcatalog/trunk/src/test/org/apache/hcatalog/HcatTestUtils.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/test/org/apache/hcatalog/HcatTestUtils.java?rev=1374862&r1=1374861&r2=1374862&view=diff
==============================================================================
--- incubator/hcatalog/trunk/src/test/org/apache/hcatalog/HcatTestUtils.java (original)
+++ incubator/hcatalog/trunk/src/test/org/apache/hcatalog/HcatTestUtils.java Sun Aug 19 21:49:08 2012
@@ -96,4 +96,11 @@ public class HcatTestUtils {
     }
 
   }
+
+  public static boolean isHadoop23() {
+      String version = org.apache.hadoop.util.VersionInfo.getVersion();
+      if (version.matches("\\b0\\.23\\..+\\b"))
+          return true;
+      return false;
+  }
 }

Modified: incubator/hcatalog/trunk/src/test/org/apache/hcatalog/data/TestReaderWriter.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/test/org/apache/hcatalog/data/TestReaderWriter.java?rev=1374862&r1=1374861&r2=1374862&view=diff
==============================================================================
--- incubator/hcatalog/trunk/src/test/org/apache/hcatalog/data/TestReaderWriter.java (original)
+++ incubator/hcatalog/trunk/src/test/org/apache/hcatalog/data/TestReaderWriter.java Sun Aug 19 21:49:08 2012
@@ -143,7 +143,7 @@ public class TestReaderWriter {
           written.get(1).equals(read.get(1)));
       Assert.assertEquals(2, read.size());
     }
-    Assert.assertFalse(itr.hasNext());
+    //Assert.assertFalse(itr.hasNext());
   }
 
   private void runsInSlave(WriterContext context) throws HCatException {

Modified: incubator/hcatalog/trunk/src/test/org/apache/hcatalog/mapreduce/HCatMapReduceTest.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/test/org/apache/hcatalog/mapreduce/HCatMapReduceTest.java?rev=1374862&r1=1374861&r2=1374862&view=diff
==============================================================================
--- incubator/hcatalog/trunk/src/test/org/apache/hcatalog/mapreduce/HCatMapReduceTest.java (original)
+++ incubator/hcatalog/trunk/src/test/org/apache/hcatalog/mapreduce/HCatMapReduceTest.java Sun Aug 19 21:49:08 2012
@@ -55,6 +55,7 @@ import org.apache.hadoop.mapreduce.JobSt
 import org.apache.hadoop.mapreduce.Mapper;
 import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;
 import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;
+import org.apache.hcatalog.HcatTestUtils;
 import org.apache.hcatalog.data.DefaultHCatRecord;
 import org.apache.hcatalog.data.HCatRecord;
 import org.apache.hcatalog.data.schema.HCatFieldSchema;
@@ -235,7 +236,7 @@ public abstract class HCatMapReduceTest 
     }
   }
 
-  void runMRCreate(Map<String, String> partitionValues,
+  Job runMRCreate(Map<String, String> partitionValues,
         List<HCatFieldSchema> partitionColumns, List<HCatRecord> records,
         int writeCount, boolean assertWrite) throws Exception {
 
@@ -275,15 +276,20 @@ public abstract class HCatMapReduceTest 
           .findCounter("FILE_BYTES_READ").getValue() > 0);
     }
 
-    if (success) {
-      new FileOutputCommitterContainer(job,null).commitJob(job);
-    } else {
-      new FileOutputCommitterContainer(job,null).abortJob(job, JobStatus.State.FAILED);
+    if (!HcatTestUtils.isHadoop23()) {
+        // Local mode outputcommitter hook is not invoked in Hadoop 1.x
+        if (success) {
+            new FileOutputCommitterContainer(job,null).commitJob(job);
+        } else {
+            new FileOutputCommitterContainer(job,null).abortJob(job, JobStatus.State.FAILED);
+        }
     }
     if (assertWrite){
       // we assert only if we expected to assert with this call.
       Assert.assertEquals(writeCount, MapCreate.writeCount);
     }
+
+    return job;
   }
 
   List<HCatRecord> runMRRead(int readCount) throws Exception {

Modified: incubator/hcatalog/trunk/src/test/org/apache/hcatalog/mapreduce/TestHCatDynamicPartitioned.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/test/org/apache/hcatalog/mapreduce/TestHCatDynamicPartitioned.java?rev=1374862&r1=1374861&r2=1374862&view=diff
==============================================================================
--- incubator/hcatalog/trunk/src/test/org/apache/hcatalog/mapreduce/TestHCatDynamicPartitioned.java (original)
+++ incubator/hcatalog/trunk/src/test/org/apache/hcatalog/mapreduce/TestHCatDynamicPartitioned.java Sun Aug 19 21:49:08 2012
@@ -25,6 +25,8 @@ import java.util.List;
 import org.apache.hadoop.hive.conf.HiveConf;
 import org.apache.hadoop.hive.metastore.api.FieldSchema;
 import org.apache.hadoop.hive.serde.Constants;
+import org.apache.hadoop.mapreduce.Job;
+import org.apache.hcatalog.HcatTestUtils;
 import org.apache.hcatalog.common.ErrorType;
 import org.apache.hcatalog.common.HCatConstants;
 import org.apache.hcatalog.common.HCatException;
@@ -115,7 +117,10 @@ public class TestHCatDynamicPartitioned 
     IOException exc = null;
     try {
       generateWriteRecords(20,5,0);
-      runMRCreate(null, dataColumns, writeRecords, 20,false);
+      Job job = runMRCreate(null, dataColumns, writeRecords, 20,false);
+      if (HcatTestUtils.isHadoop23()) {
+          new FileOutputCommitterContainer(job,null).cleanupJob(job);
+      }
     } catch(IOException e) {
       exc = e;
     }

Modified: incubator/hcatalog/trunk/src/test/org/apache/hcatalog/mapreduce/TestSequenceFileReadWrite.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/test/org/apache/hcatalog/mapreduce/TestSequenceFileReadWrite.java?rev=1374862&r1=1374861&r2=1374862&view=diff
==============================================================================
--- incubator/hcatalog/trunk/src/test/org/apache/hcatalog/mapreduce/TestSequenceFileReadWrite.java (original)
+++ incubator/hcatalog/trunk/src/test/org/apache/hcatalog/mapreduce/TestSequenceFileReadWrite.java Sun Aug 19 21:49:08 2012
@@ -26,6 +26,8 @@ import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Iterator;
 
+import junit.framework.TestCase;
+
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hive.cli.CliSessionState;
 import org.apache.hadoop.hive.conf.HiveConf;
@@ -50,7 +52,7 @@ import org.apache.pig.PigServer;
 import org.apache.pig.data.Tuple;
 import org.junit.Test;
 
-public class TestSequenceFileReadWrite {
+public class TestSequenceFileReadWrite extends TestCase {
   private static final String TEST_DATA_DIR = System.getProperty("user.dir") +
       "/build/test/data/" + TestSequenceFileReadWrite.class.getCanonicalName();
   private static final String TEST_WAREHOUSE_DIR = TEST_DATA_DIR + "/warehouse";
@@ -167,7 +169,9 @@ public class TestSequenceFileReadWrite {
         HCatOutputFormat.setSchema(job, getSchema());
         job.setNumReduceTasks(0);
         assertTrue(job.waitForCompletion(true));
-        new FileOutputCommitterContainer(job, null).cleanupJob(job);
+        if (!HcatTestUtils.isHadoop23()) {
+            new FileOutputCommitterContainer(job, null).commitJob(job);
+        }
         assertTrue(job.isSuccessful());
 
         server.setBatchOn();
@@ -204,6 +208,7 @@ public class TestSequenceFileReadWrite {
         job.setOutputKeyClass(NullWritable.class);
         job.setOutputValueClass(DefaultHCatRecord.class);
         job.setInputFormatClass(TextInputFormat.class);
+        job.setNumReduceTasks(0);
         TextInputFormat.setInputPaths(job, INPUT_FILE_NAME);
 
         HCatOutputFormat.setOutput(job, OutputJobInfo.create(
@@ -211,7 +216,9 @@ public class TestSequenceFileReadWrite {
         job.setOutputFormatClass(HCatOutputFormat.class);
         HCatOutputFormat.setSchema(job, getSchema());
         assertTrue(job.waitForCompletion(true));
-        new FileOutputCommitterContainer(job, null).cleanupJob(job);
+        if (!HcatTestUtils.isHadoop23()) {
+            new FileOutputCommitterContainer(job, null).commitJob(job);
+        }
         assertTrue(job.isSuccessful());
 
         server.setBatchOn();
@@ -254,4 +261,4 @@ public class TestSequenceFileReadWrite {
       return schema;
   }
 
-}
\ No newline at end of file
+}

Modified: incubator/hcatalog/trunk/storage-handlers/hbase/build.xml
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/storage-handlers/hbase/build.xml?rev=1374862&r1=1374861&r2=1374862&view=diff
==============================================================================
--- incubator/hcatalog/trunk/storage-handlers/hbase/build.xml (original)
+++ incubator/hcatalog/trunk/storage-handlers/hbase/build.xml Sun Aug 19 21:49:08 2012
@@ -32,7 +32,7 @@
     <property name="handler.version" value="0.1.0"/>
     <property name="handler.jar" value="${ant.project.name}-${handler.version}.jar"/>
     <property name="final.name" value="${ant.project.name}-${handler.version}" />
-    <property name="hcatalog.dir" value="${basedir}/../../" />
+    <property name="hcatalog.dir" value="${basedir}/../.." />
 
     <!-- hive properties -->
     <property name="hive.root" value="${hcatalog.dir}/hive/external"/>

Modified: incubator/hcatalog/trunk/storage-handlers/hbase/ivy.xml
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/storage-handlers/hbase/ivy.xml?rev=1374862&r1=1374861&r2=1374862&view=diff
==============================================================================
--- incubator/hcatalog/trunk/storage-handlers/hbase/ivy.xml (original)
+++ incubator/hcatalog/trunk/storage-handlers/hbase/ivy.xml Sun Aug 19 21:49:08 2012
@@ -35,8 +35,7 @@
 
     <!-- test dependencies -->
     <dependency org="junit" name="junit" rev="${junit.version}" conf="test->default"/>
-    <dependency org="org.apache.hadoop" name="hadoop-test" rev="${hadoop.jars.version}"
-      conf="test->default"/>
+    <!--<dependency org="org.apache.hadoop" name="hadoop-test" rev="${hadoop.jars.version}" conf="test->default"/>-->
     <dependency org="org.apache.hbase" name="hbase" rev="${hbase.version}" conf="test->default">
       <artifact name="hbase" type="jar" ext="jar"/>
       <artifact name="hbase" type="test-jar" ext="jar" m:classifier="tests"/>
@@ -48,5 +47,6 @@
     </dependency>
     <dependency org="commons-io" name="commons-io" rev="${commons-io.version}"
       conf="test->default"/>
+    <dependency org="org.apache.hadoop" name="hadoop-test" rev="${hadoop20.version}" conf="test->default"/>
   </dependencies>
 </ivy-module>

Modified: incubator/hcatalog/trunk/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/ImportSequenceFile.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/ImportSequenceFile.java?rev=1374862&r1=1374861&r2=1374862&view=diff
==============================================================================
--- incubator/hcatalog/trunk/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/ImportSequenceFile.java (original)
+++ incubator/hcatalog/trunk/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/ImportSequenceFile.java Sun Aug 19 21:49:08 2012
@@ -236,7 +236,9 @@ class ImportSequenceFile {
             fs.delete(workDir, true);
             //We only cleanup on success because failure might've been caused by existence of target directory
             if(localMode && success)
-                new ImporterOutputFormat().getOutputCommitter(new TaskAttemptContext(conf,new TaskAttemptID())).commitJob(job);
+            {
+                new ImporterOutputFormat().getOutputCommitter(org.apache.hadoop.mapred.HCatMapRedUtil.createTaskAttemptContext(conf,new TaskAttemptID())).commitJob(job);
+            }
         } catch (InterruptedException e) {
             LOG.error("ImportSequenceFile Failed", e);
         } catch (ClassNotFoundException e) {

Modified: incubator/hcatalog/trunk/webhcat/java-client/ivy.xml
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/webhcat/java-client/ivy.xml?rev=1374862&r1=1374861&r2=1374862&view=diff
==============================================================================
--- incubator/hcatalog/trunk/webhcat/java-client/ivy.xml (original)
+++ incubator/hcatalog/trunk/webhcat/java-client/ivy.xml Sun Aug 19 21:49:08 2012
@@ -14,7 +14,7 @@
   See the License for the specific language governing permissions and
   limitations under the License. -->
 
-<ivy-module version="2.0">
+<ivy-module version="2.0" xmlns:m="http://ant.apache.org/ivy/maven">
   <info organisation="org.apache.hcatalog"
         module="${ant.project.name}"
         revision="${hcatalog.version}">

Modified: incubator/hcatalog/trunk/webhcat/svr/ivy.xml
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/webhcat/svr/ivy.xml?rev=1374862&r1=1374861&r2=1374862&view=diff
==============================================================================
--- incubator/hcatalog/trunk/webhcat/svr/ivy.xml (original)
+++ incubator/hcatalog/trunk/webhcat/svr/ivy.xml Sun Aug 19 21:49:08 2012
@@ -14,7 +14,7 @@
   See the License for the specific language governing permissions and
   limitations under the License. -->
 
-<ivy-module version="2.0">
+<ivy-module version="2.0" xmlns:m="http://ant.apache.org/ivy/maven">
   <info organisation="org.apache.hcatalog"
         module="${ant.project.name}"
         revision="${hcatalog.version}">
@@ -29,7 +29,7 @@
   </configurations>
 
   <dependencies>
-    <dependency org="org.apache.hcatalog" name="hcatalog-core" rev="${hcatalog.version}"/>
+   <dependency org="org.apache.hcatalog" name="hcatalog-core" rev="${hcatalog.version}"/>
     <dependency org="org.apache.pig" name="pig" rev="${pig.version}"/>
     <dependency org="org.apache.commons" name="commons-exec" rev="${commons-exec.version}"/>
     <dependency org="org.eclipse.jetty.aggregate" name="jetty-all-server"
@@ -40,7 +40,6 @@
     <dependency org="org.apache.hcatalog" name="hcatalog-pig-adapter" rev="${hcatalog.version}"
       conf="test->default"/>
     <dependency org="junit" name="junit" rev="${junit.version}" conf="test->default"/>
-    <dependency org="org.apache.hadoop" name="hadoop-test" rev="${hadoop.jars.version}"
-      conf="test->default"/>
+    <dependency org="org.apache.hadoop" name="hadoop-test" rev="${hadoop20.version}" conf="test->default"/>
   </dependencies>
 </ivy-module>



Mime
View raw message