incubator-hcatalog-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From tof...@apache.org
Subject svn commit: r1374902 - in /incubator/hcatalog/branches/branch-0.4: ./ ivy/ shims/ shims/src/20/ shims/src/20/java/ shims/src/20/java/org/ shims/src/20/java/org/apache/ shims/src/20/java/org/apache/hcatalog/ shims/src/20/java/org/apache/hcatalog/shims/ ...
Date Mon, 20 Aug 2012 03:08:58 GMT
Author: toffer
Date: Mon Aug 20 03:08:56 2012
New Revision: 1374902

URL: http://svn.apache.org/viewvc?rev=1374902&view=rev
Log:
merged from trunk: HCATALOG-375 Make HCat work for Hadoop 0.23 (cdrome via toffer)

Added:
    incubator/hcatalog/branches/branch-0.4/build.properties
    incubator/hcatalog/branches/branch-0.4/shims/build.xml
    incubator/hcatalog/branches/branch-0.4/shims/ivy.xml
    incubator/hcatalog/branches/branch-0.4/shims/src/20/
    incubator/hcatalog/branches/branch-0.4/shims/src/20/java/
    incubator/hcatalog/branches/branch-0.4/shims/src/20/java/org/
    incubator/hcatalog/branches/branch-0.4/shims/src/20/java/org/apache/
    incubator/hcatalog/branches/branch-0.4/shims/src/20/java/org/apache/hcatalog/
    incubator/hcatalog/branches/branch-0.4/shims/src/20/java/org/apache/hcatalog/shims/
    incubator/hcatalog/branches/branch-0.4/shims/src/20/java/org/apache/hcatalog/shims/HCatHadoopShims20S.java
Removed:
    incubator/hcatalog/branches/branch-0.4/shims/src/20S/
Modified:
    incubator/hcatalog/branches/branch-0.4/CHANGES.txt
    incubator/hcatalog/branches/branch-0.4/build.xml
    incubator/hcatalog/branches/branch-0.4/ivy.xml
    incubator/hcatalog/branches/branch-0.4/ivy/libraries.properties
    incubator/hcatalog/branches/branch-0.4/shims/src/23/java/org/apache/hcatalog/shims/HCatHadoopShims23.java
    incubator/hcatalog/branches/branch-0.4/src/java/org/apache/hadoop/mapred/HCatMapRedUtil.java
    incubator/hcatalog/branches/branch-0.4/src/java/org/apache/hcatalog/data/transfer/impl/HCatInputFormatReader.java
    incubator/hcatalog/branches/branch-0.4/src/java/org/apache/hcatalog/data/transfer/impl/HCatOutputFormatWriter.java
    incubator/hcatalog/branches/branch-0.4/src/java/org/apache/hcatalog/data/transfer/state/StateProvider.java
    incubator/hcatalog/branches/branch-0.4/src/java/org/apache/hcatalog/mapreduce/FileOutputCommitterContainer.java
    incubator/hcatalog/branches/branch-0.4/src/java/org/apache/hcatalog/mapreduce/FileRecordWriterContainer.java
    incubator/hcatalog/branches/branch-0.4/src/java/org/apache/hcatalog/mapreduce/MultiOutputFormat.java
    incubator/hcatalog/branches/branch-0.4/src/java/org/apache/hcatalog/mapreduce/ProgressReporter.java
    incubator/hcatalog/branches/branch-0.4/src/java/org/apache/hcatalog/mapreduce/Security.java
    incubator/hcatalog/branches/branch-0.4/src/java/org/apache/hcatalog/pig/HCatStorer.java
    incubator/hcatalog/branches/branch-0.4/src/java/org/apache/hcatalog/shims/HCatHadoopShims.java
    incubator/hcatalog/branches/branch-0.4/src/test/e2e/hcatalog/build.xml
    incubator/hcatalog/branches/branch-0.4/src/test/e2e/hcatalog/conf/default.conf
    incubator/hcatalog/branches/branch-0.4/src/test/e2e/hcatalog/drivers/Util.pm
    incubator/hcatalog/branches/branch-0.4/src/test/e2e/hcatalog/tests/hcat.conf
    incubator/hcatalog/branches/branch-0.4/src/test/e2e/hcatalog/tools/generate/generate_data.pl
    incubator/hcatalog/branches/branch-0.4/src/test/e2e/hcatalog/udfs/java/org/apache/hcatalog/utils/SimpleRead.java
    incubator/hcatalog/branches/branch-0.4/src/test/org/apache/hcatalog/HcatTestUtils.java
    incubator/hcatalog/branches/branch-0.4/src/test/org/apache/hcatalog/data/TestReaderWriter.java
    incubator/hcatalog/branches/branch-0.4/src/test/org/apache/hcatalog/mapreduce/HCatMapReduceTest.java
    incubator/hcatalog/branches/branch-0.4/src/test/org/apache/hcatalog/mapreduce/TestHCatDynamicPartitioned.java
    incubator/hcatalog/branches/branch-0.4/src/test/org/apache/hcatalog/mapreduce/TestSequenceFileReadWrite.java
    incubator/hcatalog/branches/branch-0.4/storage-handlers/hbase/ivy.xml
    incubator/hcatalog/branches/branch-0.4/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/ImportSequenceFile.java
    incubator/hcatalog/branches/branch-0.4/webhcat/java-client/ivy.xml

Modified: incubator/hcatalog/branches/branch-0.4/CHANGES.txt
URL: http://svn.apache.org/viewvc/incubator/hcatalog/branches/branch-0.4/CHANGES.txt?rev=1374902&r1=1374901&r2=1374902&view=diff
==============================================================================
--- incubator/hcatalog/branches/branch-0.4/CHANGES.txt (original)
+++ incubator/hcatalog/branches/branch-0.4/CHANGES.txt Mon Aug 20 03:08:56 2012
@@ -23,6 +23,8 @@ Trunk (unreleased changes)
   INCOMPATIBLE CHANGES
 
   NEW FEATURES
+  HCAT-375 Make HCat work for Hadoop 0.23 (cdrome via toffer)
+
   HCAT-469 HiveClientCache may return a closed client (amalakar via toffer)
 
   HCAT-370 Create a HiveMetaStoreClient cache in hcatalog (amalakar via toffer) 

Added: incubator/hcatalog/branches/branch-0.4/build.properties
URL: http://svn.apache.org/viewvc/incubator/hcatalog/branches/branch-0.4/build.properties?rev=1374902&view=auto
==============================================================================
--- incubator/hcatalog/branches/branch-0.4/build.properties (added)
+++ incubator/hcatalog/branches/branch-0.4/build.properties Mon Aug 20 03:08:56 2012
@@ -0,0 +1,23 @@
+javac.debug=on
+javac.optimize=on
+javac.deprecation=off
+javac.version=1.6
+javac.args=
+javac.args.warnings=
+
+# Set to 20 to build against hadoop 1.0.2 or 23 to build against hadoop 0.23.1
+hadoopversion=20
+
+build.encoding=UTF-8
+build.dir=${hcatalog.home}/build
+build.classes=${build.dir}/classes
+build.docs=${build.dir}/docs
+build.javadoc=${build.docs}/api
+
+build.ivy.dir=${build.dir}/ivy
+build.ivy.lib.dir=${build.ivy.dir}/lib
+ivy.conf.dir=${hcatalog.home}/ivy
+ivysettings.xml=${ivy.conf.dir}/ivysettings.xml
+ivyresolvelog=download-only
+
+mvnrepo=http://repo2.maven.org/maven2

Modified: incubator/hcatalog/branches/branch-0.4/build.xml
URL: http://svn.apache.org/viewvc/incubator/hcatalog/branches/branch-0.4/build.xml?rev=1374902&r1=1374901&r2=1374902&view=diff
==============================================================================
--- incubator/hcatalog/branches/branch-0.4/build.xml (original)
+++ incubator/hcatalog/branches/branch-0.4/build.xml Mon Aug 20 03:08:56 2012
@@ -19,6 +19,9 @@
 
 
 <project name="hcatalog" default="jar" xmlns:ivy="antlib:org.apache.ivy.ant" >
+  <property name="hcatalog.home" value="${basedir}" />
+  <property file="${hcatalog.home}/build.properties" />
+
   <!--  
   ================================================================================
   Imports
@@ -43,20 +46,10 @@
   <property name="src.dir"  location="${basedir}/src/java"/>
   <property name="package.dir"  location="${basedir}/src/packages"/>
   <property name="docs.src" value="${basedir}/src/docs"/>
-  <property name="build.dir" value="${basedir}/build"/>
-  <property name="build.classes" value="${build.dir}/classes" />
-  <property name="build.docs" value="${build.dir}/docs" />
-  <property name="build.javadoc" value="${build.docs}/api" />
   <property name="dist.dir" value="${build.dir}/${final.name}" />
 
   <!-- javac properties -->
-  <property name="build.encoding" value="UTF8" />
   <property name="excludes" value=""/>
-  <property name="javac.debug" value="on" />
-  <property name="javac.optimize" value="on" />
-  <property name="javac.deprecation" value="off" />
-  <property name="javac.version" value="1.6" />
-  <property name="javac.args" value="" />
 
   <!-- test properties -->
   <property name="test.src.dir" value="${basedir}/src/test" />
@@ -91,24 +84,19 @@
   <!-- e2e test properties -->
   <property name="test.e2e.dir" value="${basedir}/src/test/e2e/hcatalog"/>
 
-  <!-- ivy properteis set here -->
+  <!-- ivy properties set here -->
   <property name="ivy.repo.dir" value="${user.home}/ivyrepo" />
-  <property name="ivy.dir" location="ivy" />
-  <loadproperties srcfile="${ivy.dir}/libraries.properties"/>
+  <loadproperties srcfile="${ivy.conf.dir}/libraries.properties"/>
   <property name="asfrepo" value="https://repository.apache.org"/>
   <property name="asfsnapshotrepo" value="${asfrepo}/content/repositories/snapshots"/>
-  <property name="mvnrepo" value="http://repo2.maven.org/maven2"/>
   <property name="asfstagingrepo" value="${asfrepo}/service/local/staging/deploy/maven2"/>
-  <property name="ivy.jar" location="${ivy.dir}/ivy-${ivy.version}.jar"/>
-  <property name="ant_task.jar" location="${ivy.dir}/maven-ant-tasks-${ant-task.version}.jar"/>
+  <property name="ivy.jar" location="${ivy.conf.dir}/ivy-${ivy.version}.jar"/>
+  <property name="ant_task.jar" location="${ivy.conf.dir}/maven-ant-tasks-${ant-task.version}.jar"/>
   <property name="ant_task_repo_url"
     value="${mvnrepo}/org/apache/maven/maven-ant-tasks/${ant-task.version}/maven-ant-tasks-${ant-task.version}.jar"/>
   <property name="ivy_repo_url" value="${mvnrepo}/org/apache/ivy/ivy/${ivy.version}/ivy-${ivy.version}.jar"/>
   <property name="ivy.xml" location="${basedir}/ivy.xml"/>
-  <property name="ivysettings.xml" location="${ivy.dir}/ivysettings.xml" />
-  <property name="build.ivy.dir" location="${build.dir}/ivy" />
   <property name="pom.file" location="${build.ivy.dir}/${ant.project.name}-${hcatalog.version}.pom"/>
-  <property name="build.ivy.lib.dir" location="${build.ivy.dir}/lib" />
   <property name="ivy.lib.dir" location="${build.ivy.lib.dir}/${ant.project.name}"/>
   <property name="build.ivy.report.dir" location="${build.ivy.dir}/report" />
 
@@ -270,17 +258,6 @@
       <compilerarg line="${javac.args}"/>
       <classpath refid="classpath" />
     </javac>
-    <!-- compile shim for selected hadoop version -->
-    <!--property name="debugclasspath" refid="classpath"/>
-    <echo message="classpath = ${debugclasspath}"/-->
-    <javac encoding="${build.encoding}" srcdir="${basedir}/shims/src/${shims.name}/java" excludes="${excludes}"
-        includes="**/*.java" destdir="${build.classes}" debug="${javac.debug}"
-        optimize="${javac.optimize}" target="${javac.version}"
-        source="${javac.version}" deprecation="${javac.deprecation}"
-        includeantruntime="false">
-      <compilerarg line="${javac.args}"/>
-      <classpath refid="classpath" />
-    </javac>
   </target>
  
   <!-- Build the hcatalog client jar -->
@@ -301,12 +278,16 @@
   </jar>
   </target>
 
+  <target name="shims" depends="compile-src">
+    <ant antfile="shims/build.xml" target="jar" inheritAll="true" useNativeBasedir="true" />
+  </target>
+
   <!--
   ================================================================================
   Build both clientjar and server-extensions
   ================================================================================
   -->
-  <target name="jar" depends="clientjar,server-extensions,jar-storage-handlers,jar-webhcat-java-client"/>
+  <target name="jar" depends="shims,clientjar,server-extensions,jar-storage-handlers,jar-webhcat-java-client"/>
 
   <!--
   ================================================================================

Modified: incubator/hcatalog/branches/branch-0.4/ivy.xml
URL: http://svn.apache.org/viewvc/incubator/hcatalog/branches/branch-0.4/ivy.xml?rev=1374902&r1=1374901&r2=1374902&view=diff
==============================================================================
--- incubator/hcatalog/branches/branch-0.4/ivy.xml (original)
+++ incubator/hcatalog/branches/branch-0.4/ivy.xml Mon Aug 20 03:08:56 2012
@@ -14,7 +14,7 @@
   See the License for the specific language governing permissions and
   limitations under the License. -->
 
-<ivy-module version="2.0">
+<ivy-module version="2.0" xmlns:m="http://ant.apache.org/ivy/maven">
     <info organisation="org.apache.hcatalog" module="${ant.project.name}"
             revision="${hcatalog.version}">
         <license name="Apache 2.0"/>
@@ -28,13 +28,16 @@
     <!--these match the Maven configurations-->
         <conf name="default" extends="master,runtime"/>
         <conf name="master" description="contains the artifact but no dependencies"/>
-        <conf name="runtime" description="runtime but not the artifact" />
+        <conf name="runtime" extends="compile" description="runtime but not the artifact" />
         <conf name="common" visibility="private" 
             extends="runtime"
             description="artifacts needed to compile/test the application"/>
         <conf name="test" visibility="private" extends="runtime"/>
         <conf name="package" visibility="private" extends="master"/>
         <conf name="releaseaudit" visibility="private"/>
+        <conf name="compile" extends="hadoop${hadoopversion}" visibility="private" description="compile artifacts"/>
+        <conf name="hadoop20" visibility="private"/>
+        <conf name="hadoop23" visibility="private"/>
     </configurations>
     <dependencies>
         <!-- needed to compile -->
@@ -48,21 +51,51 @@
           conf="common->master"/>
         <dependency org="commons-lang" name="commons-lang" rev="${commons-lang.version}"
           conf="common->master"/>
-        <dependency org="commons-logging" name="commons-logging"
-          rev="${commons-logging.version}" conf="common->master"/>
-        <dependency org="commons-logging" name="commons-logging-api"
-          rev="${commons-logging.version}" conf="common->master"/>
+
+        <!-- hadoop20 -->
         <dependency org="org.apache.hadoop" name="hadoop-tools"
-          rev="${hadoop-tools.version}" conf="common->master" />
+          rev="${hadoop20.version}" conf="hadoop20->master" />
         <dependency org="org.apache.hadoop" name="hadoop-core"
-          rev="${hadoop-core.version}" conf="common->master" />
+          rev="${hadoop20.version}" conf="hadoop20->master" />
         <dependency org="org.apache.hadoop" name="hadoop-test"
-          rev="${hadoop-test.version}" conf="common->master" />
+          rev="${hadoop20.version}" conf="hadoop20->master" />
+
+        <!-- hadoop23 -->
+        <dependency org="org.apache.hadoop" name="hadoop-common"
+                    rev="${hadoop23.version}"
+                    conf="hadoop23->default">
+          <include type="jar"/>
+          <exclude org="commons-daemon" module="commons-daemon"/><!--bad POM-->
+          <exclude org="org.apache.commons" module="commons-daemon"/><!--bad POM-->
+        </dependency>
+        <dependency org="org.apache.hadoop" name="hadoop-mapreduce-client-core"
+                    rev="${hadoop23.version}"
+                    conf="hadoop23->default">
+          <include type="jar"/>
+          <exclude org="commons-daemon" module="commons-daemon"/><!--bad POM-->
+          <exclude org="org.apache.commons" module="commons-daemon"/><!--bad POM-->
+        </dependency>
+        <dependency org="org.apache.hadoop" name="hadoop-archives"
+                    rev="${hadoop23.version}"
+                    conf="hadoop23->default">
+          <include type="jar"/>
+          <exclude org="commons-daemon" module="commons-daemon"/><!--bad POM-->
+          <exclude org="org.apache.commons" module="commons-daemon"/><!--bad POM-->
+        </dependency>
+        <dependency org="org.apache.hadoop" name="hadoop-hdfs"
+                    rev="${hadoop23.version}"
+                    conf="hadoop23->default">
+          <artifact name="hadoop-hdfs" ext="jar" />
+          <artifact name="hadoop-hdfs" type="tests" ext="jar" m:classifier="tests"/>
+          <exclude org="commons-daemon" module="commons-daemon"/><!--bad POM-->
+          <exclude org="org.apache.commons" module="commons-daemon"/><!--bad POM-->
+        </dependency>
+
         <dependency org="javax.jms" name="jms" rev="${jms.version}"
           conf="common->master" />
         <dependency org="org.apache.activemq" name="activemq-core"
           rev="${activemq.version}" conf="common->master" />
-	<dependency org="org.apache.activemq" name="kahadb"
+        <dependency org="org.apache.activemq" name="kahadb"
           rev="${activemq.version}" conf="common->master" />
         <dependency org="javax.management.j2ee" name="management-api"
           rev="${javax-mgmt.version}" conf="common->master" /> 

Modified: incubator/hcatalog/branches/branch-0.4/ivy/libraries.properties
URL: http://svn.apache.org/viewvc/incubator/hcatalog/branches/branch-0.4/ivy/libraries.properties?rev=1374902&r1=1374901&r2=1374902&view=diff
==============================================================================
--- incubator/hcatalog/branches/branch-0.4/ivy/libraries.properties (original)
+++ incubator/hcatalog/branches/branch-0.4/ivy/libraries.properties Mon Aug 20 03:08:56 2012
@@ -29,9 +29,8 @@ datanucleus-rdbms.version=2.0.3
 derby.version=10.4.2.0
 fb303.version=0.7.0
 guava.version=11.0.2
-hadoop-core.version=1.0.2
-hadoop-test.version=1.0.2
-hadoop-tools.version=1.0.2
+hadoop20.version=1.0.2
+hadoop23.version=0.23.1
 hbase.version=0.92.0
 high-scale-lib.version=1.1.1
 hive.version=0.9.0

Added: incubator/hcatalog/branches/branch-0.4/shims/build.xml
URL: http://svn.apache.org/viewvc/incubator/hcatalog/branches/branch-0.4/shims/build.xml?rev=1374902&view=auto
==============================================================================
--- incubator/hcatalog/branches/branch-0.4/shims/build.xml (added)
+++ incubator/hcatalog/branches/branch-0.4/shims/build.xml Mon Aug 20 03:08:56 2012
@@ -0,0 +1,106 @@
+<?xml version="1.0"?>
+<project xmlns:ivy="antlib:org.apache.ivy.ant" name="shims" default="jar">
+    <property name="hcatalog.home" value="${basedir}/.." />
+    <property file="${hcatalog.home}/build.properties" />
+    <loadproperties srcfile="${ivy.conf.dir}/libraries.properties"/>
+    <!--<property name="mvnrepo" value="http://repo2.maven.org/maven2"/>-->
+
+    <property name="shims.include" value="20,23"/>
+    <!-- sources and hadoop version for each shim -->
+    <property name="shims.0.20.sources" value="${basedir}/src/20/java" />
+    <property name="shims.0.23.sources" value="${basedir}/src/23/java" />
+    <property name="shims.0.20.hadoop.ivy.dir" value="${build.ivy.lib.dir}/hadoop20" />
+    <property name="shims.0.23.hadoop.ivy.dir" value="${build.ivy.lib.dir}/hadoop23" />
+
+    <property name="ivy.artifact.retrieve.pattern" value="[conf]/[artifact]-[revision](-[classifier]).[ext]" />
+
+    <target name="jar" depends="compile">
+        <echo message="Project: ${ant.project.name}"/>
+        <echo message="Building shim jars ${basedir}" />
+        <echo message="Building shim jars ${hcatalog.home}" />
+    </target>
+
+    <target name="compile" depends="init,ivy-retrieve">
+        <echo message="Project: ${ant.project.name}"/>
+        <!--
+        <for param="shimName" list="${shims.include}">
+            <sequential>
+                <echo>Building shims @{shimName}</echo>
+                <echo message="${hadoop.version}" />
+                <antcall target="build_shims" inheritRefs="false" inheritAll="false">
+                    <param name="hadoop.version.ant-internal" value="${shims.@{shimName}.version}" />
+                    <param name="sources" value="${shims.@{shimName}.sources}" />
+                    <param name="hadoop.ivy.dir" value="${shims.@{shimName}.hadoop.ivy.dir}" />
+                    <param name="shim.name" value="@{shimName}" />
+                </antcall>
+            </sequential>
+        </for>
+        -->
+        <antcall target="build-shims" inheritRefs="false" inheritAll="false">
+            <param name="hadoop.version.ant-internal" value="20" />
+            <param name="sources" value="${shims.0.20.sources}" />
+            <param name="hadoop.ivy.dir" value="${shims.0.20.hadoop.ivy.dir}" />
+            <param name="shim.name" value="20" />
+        </antcall>
+        <antcall target="build-shims" inheritRefs="false" inheritAll="false">
+            <param name="hadoop.version.ant-internal" value="23" />
+            <param name="sources" value="${shims.0.23.sources}" />
+            <param name="hadoop.ivy.dir" value="${shims.0.23.hadoop.ivy.dir}" />
+            <param name="shim.name" value="23" />
+        </antcall>
+    </target>
+
+    <target name="build-shims">
+        <echo message="Project: ${ant.project.name}"/>
+
+        <antcall target="ivy-retrieve-hadoop-shim" inheritRefs="false" inheritAll="false">
+            <param name="ivy.hadoop.shim.conf" value="hadoop${shim.name}" />
+        </antcall>
+
+        <path id="shims.classpath">
+            <fileset dir="${hadoop.ivy.dir}" includes="*.jar" />
+        </path>
+
+        <javac
+            encoding="${build.encoding}"
+            includes="**/*.java"
+            destdir="${build.classes}"
+            debug="${javac.debug}"
+            deprecation="${javac.deprecation}"
+            srcdir="${sources}"
+            includeantruntime="false">
+            <compilerarg line="${javac.args} ${javac.args.warnings}" />
+            <classpath refid="shims.classpath"/>
+        </javac>
+    </target>
+
+    <target name="init">
+        <echo message="Project: ${ant.project.name}"/>
+    </target>
+
+    <target name="ivy-retrieve">
+        <echo message="Project: ${ant.project.name}"/>
+    </target>
+
+    <target name="ivy-init-settings">
+        <!--Configure Ivy by reading in the settings file
+            If anyone has already read in a settings file into this settings ID, it gets priority
+        -->
+        <echo message="Project: ${ant.project.name}"/>
+        <ivy:settings id="${ant.project.name}.ivy.settings" file="${ivysettings.xml}"/>
+    </target>
+
+    <target name="ivy-resolve-hadoop-shim" depends="ivy-init-settings" unless="offline">
+        <echo message="Project: ${ant.project.name}"/>
+        <ivy:resolve settingsRef="${ant.project.name}.ivy.settings"
+            conf="${ivy.hadoop.shim.conf}" log="${ivyresolvelog}"/>
+    </target>
+
+    <target name="ivy-retrieve-hadoop-shim" depends="ivy-resolve-hadoop-shim"
+        description="Retrieve Ivy-managed artifacts">
+        <echo message="Project: ${ant.project.name}"/>
+        <ivy:retrieve settingsRef="${ant.project.name}.ivy.settings"
+            pattern="${build.ivy.lib.dir}/${ivy.artifact.retrieve.pattern}"
+            log="${ivyresolvelog}" conf="${ivy.hadoop.shim.conf}"/>
+    </target>
+</project>

Added: incubator/hcatalog/branches/branch-0.4/shims/ivy.xml
URL: http://svn.apache.org/viewvc/incubator/hcatalog/branches/branch-0.4/shims/ivy.xml?rev=1374902&view=auto
==============================================================================
--- incubator/hcatalog/branches/branch-0.4/shims/ivy.xml (added)
+++ incubator/hcatalog/branches/branch-0.4/shims/ivy.xml Mon Aug 20 03:08:56 2012
@@ -0,0 +1,84 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<ivy-module version="2.0" xmlns:m="http://ant.apache.org/ivy/maven">
+  <info organisation="org.apache.hcatalog" module="${ant.project.name}" revision="${hcatalog.version}">
+    <license name="The Apache Software License, Version 2.0" url="http://www.apache.org/licenses/LICENSE-2.0.txt" />
+    <description homepage="http://hive.apache.org">
+        Apache HCatalog
+    </description>
+  </info>
+  <configurations>
+    <conf name="default" extends="master,runtime"/>
+    <conf name="master" description="contains the artifact but no dependencies"/>
+    <conf name="runtime" extends="compile" description="runtime but not the artifact" />
+    <conf name="common" visibility="private" 
+        extends="runtime"
+        description="artifacts needed to compile/test the application"/>
+    <conf name="test" visibility="private" extends="runtime"/>
+    <conf name="package" visibility="private" extends="master"/>
+    <conf name="releaseaudit" visibility="private"/>
+    <conf name="compile" extends="hadoop${hadoopversion}" visibility="private" description="compile artifacts"/>
+    <conf name="hadoop20" visibility="private"/>
+    <conf name="hadoop20S" visibility="private"/>
+    <conf name="hadoop23" visibility="private"/>
+  </configurations>
+  <dependencies>
+    <dependency org="org.apache.pig" name="pig" rev="${pig.version}"
+        conf="hadoop20,hadoop23->default" />
+
+    <!-- Hadoop 0.23 dependencies. Used both for shims and for building against Hadoop 0.23. -->
+    <dependency org="org.apache.hadoop" name="hadoop-common"
+                rev="${hadoop23.version}"
+                conf="hadoop23->default">
+      <include type="jar"/>
+      <exclude org="commons-daemon" module="commons-daemon"/><!--bad POM-->
+      <exclude org="org.apache.commons" module="commons-daemon"/><!--bad POM-->
+    </dependency>
+    <dependency org="org.apache.hadoop" name="hadoop-mapreduce-client-core"
+                rev="${hadoop23.version}"
+                conf="hadoop23->default">
+      <include type="jar"/>
+      <exclude org="commons-daemon" module="commons-daemon"/><!--bad POM-->
+      <exclude org="org.apache.commons" module="commons-daemon"/><!--bad POM-->
+    </dependency>
+    <dependency org="org.apache.hadoop" name="hadoop-archives"
+                rev="${hadoop23.version}"
+                conf="hadoop23->default">
+      <include type="jar"/>
+      <exclude org="commons-daemon" module="commons-daemon"/><!--bad POM-->
+      <exclude org="org.apache.commons" module="commons-daemon"/><!--bad POM-->
+    </dependency>
+    <dependency org="org.apache.hadoop" name="hadoop-hdfs"
+                rev="${hadoop23.version}"
+                conf="hadoop23->default">
+      <artifact name="hadoop-hdfs" ext="jar" />
+      <artifact name="hadoop-hdfs" type="tests" ext="jar" m:classifier="tests"/>
+      <exclude org="commons-daemon" module="commons-daemon"/><!--bad POM-->
+      <exclude org="org.apache.commons" module="commons-daemon"/><!--bad POM-->
+    </dependency>
+
+    <!-- Hadoop 0.20 shim dependencies. Used for building 0.20 shims. -->
+    <dependency org="org.apache.hadoop" name="hadoop-core"
+                rev="${hadoop20.version}"
+                conf="hadoop20->default">
+      <include type="jar"/>
+      <exclude org="commons-daemon" module="commons-daemon"/><!--bad POM-->
+      <exclude org="org.apache.commons" module="commons-daemon"/><!--bad POM-->
+    </dependency>
+    <dependency org="org.apache.hadoop" name="hadoop-tools"
+                rev="${hadoop20.version}"
+                conf="hadoop20->default">
+      <include type="jar"/>
+      <exclude org="commons-daemon" module="commons-daemon"/><!--bad POM-->
+      <exclude org="org.apache.commons" module="commons-daemon"/><!--bad POM-->
+    </dependency>
+    <dependency org="org.apache.hadoop" name="hadoop-test"
+                rev="${hadoop20.version}"
+                conf="hadoop20->default">
+      <include type="jar"/>
+      <exclude org="commons-daemon" module="commons-daemon"/><!--bad POM-->
+      <exclude org="org.apache.commons" module="commons-daemon"/><!--bad POM-->
+    </dependency>
+
+    <conflict manager="all" />
+  </dependencies>
+</ivy-module>

Added: incubator/hcatalog/branches/branch-0.4/shims/src/20/java/org/apache/hcatalog/shims/HCatHadoopShims20S.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/branches/branch-0.4/shims/src/20/java/org/apache/hcatalog/shims/HCatHadoopShims20S.java?rev=1374902&view=auto
==============================================================================
--- incubator/hcatalog/branches/branch-0.4/shims/src/20/java/org/apache/hcatalog/shims/HCatHadoopShims20S.java (added)
+++ incubator/hcatalog/branches/branch-0.4/shims/src/20/java/org/apache/hcatalog/shims/HCatHadoopShims20S.java Mon Aug 20 03:08:56 2012
@@ -0,0 +1,128 @@
+package org.apache.hcatalog.shims;
+
+import java.io.IOException;
+import java.net.InetSocketAddress;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.filecache.DistributedCache;
+import org.apache.hadoop.mapred.JobTracker;
+import org.apache.hadoop.mapreduce.Job;
+import org.apache.hadoop.mapreduce.JobContext;
+import org.apache.hadoop.mapreduce.JobID;
+import org.apache.hadoop.mapreduce.JobStatus.State;
+import org.apache.hadoop.mapreduce.OutputFormat;
+import org.apache.hadoop.mapreduce.TaskAttemptContext;
+import org.apache.hadoop.mapreduce.TaskAttemptID;
+import org.apache.hadoop.mapreduce.TaskID;
+import org.apache.hadoop.util.Progressable;
+import org.apache.pig.ResourceSchema;
+
+public class HCatHadoopShims20S implements HCatHadoopShims {
+    @Override
+    public TaskID createTaskID() {
+        return new TaskID();
+    }
+
+    @Override
+    public TaskAttemptID createTaskAttemptID() {
+        return new TaskAttemptID();
+    }
+
+	@Override
+	public TaskAttemptContext createTaskAttemptContext(Configuration conf,
+			TaskAttemptID taskId) {
+        return new TaskAttemptContext(conf, taskId);
+    }
+
+    @Override
+    public org.apache.hadoop.mapred.TaskAttemptContext createTaskAttemptContext(org.apache.hadoop.mapred.JobConf conf,
+            org.apache.hadoop.mapred.TaskAttemptID taskId, Progressable progressable) {
+        org.apache.hadoop.mapred.TaskAttemptContext newContext = null;
+        try {
+            java.lang.reflect.Constructor construct = org.apache.hadoop.mapred.TaskAttemptContext.class.getDeclaredConstructor(
+                    org.apache.hadoop.mapred.JobConf.class, org.apache.hadoop.mapred.TaskAttemptID.class,
+                    Progressable.class);
+            construct.setAccessible(true);
+            newContext = (org.apache.hadoop.mapred.TaskAttemptContext)construct.newInstance(conf, taskId, progressable);
+        } catch (Exception e) {
+            throw new RuntimeException(e);
+        }
+        return newContext;
+    }
+
+    @Override
+    public JobContext createJobContext(Configuration conf,
+            JobID jobId) {
+        return new JobContext(conf, jobId);
+    }
+
+    @Override
+    public org.apache.hadoop.mapred.JobContext createJobContext(org.apache.hadoop.mapred.JobConf conf,
+            org.apache.hadoop.mapreduce.JobID jobId, Progressable progressable) {
+        org.apache.hadoop.mapred.JobContext newContext = null;
+        try {
+            java.lang.reflect.Constructor construct = org.apache.hadoop.mapred.JobContext.class.getDeclaredConstructor(
+                    org.apache.hadoop.mapred.JobConf.class, org.apache.hadoop.mapreduce.JobID.class,
+                    Progressable.class);
+            construct.setAccessible(true);
+            newContext = (org.apache.hadoop.mapred.JobContext)construct.newInstance(conf, jobId, progressable);
+        } catch (Exception e) {
+            throw new RuntimeException(e);
+        }
+        return newContext;
+    }
+
+    @Override
+    public void commitJob(OutputFormat outputFormat, ResourceSchema schema,
+            String arg1, Job job) throws IOException {
+        if( job.getConfiguration().get("mapred.job.tracker", "").equalsIgnoreCase("local") ) {
+            try {
+                //In local mode, mapreduce will not call OutputCommitter.cleanupJob.
+                //Calling it from here so that the partition publish happens.
+                //This call needs to be removed after MAPREDUCE-1447 is fixed.
+                outputFormat.getOutputCommitter(HCatHadoopShims.Instance.get().createTaskAttemptContext(
+                            job.getConfiguration(), HCatHadoopShims.Instance.get().createTaskAttemptID())).commitJob(job);
+            } catch (IOException e) {
+                throw new IOException("Failed to cleanup job",e);
+            } catch (InterruptedException e) {
+                throw new IOException("Failed to cleanup job",e);
+            }
+        }
+    }
+
+    @Override
+    public void abortJob(OutputFormat outputFormat, Job job) throws IOException {
+        if (job.getConfiguration().get("mapred.job.tracker", "")
+                .equalsIgnoreCase("local")) {
+            try {
+                // This call needs to be removed after MAPREDUCE-1447 is fixed.
+                outputFormat.getOutputCommitter(HCatHadoopShims.Instance.get().createTaskAttemptContext(
+                            job.getConfiguration(), new TaskAttemptID())).abortJob(job, State.FAILED);
+            } catch (IOException e) {
+                throw new IOException("Failed to abort job", e);
+            } catch (InterruptedException e) {
+                throw new IOException("Failed to abort job", e);
+            }
+        }
+    }
+
+    @Override
+    public InetSocketAddress getResourceManagerAddress(Configuration conf)
+    {
+        return JobTracker.getAddress(conf);
+    }
+
+    @Override
+    public String getPropertyName(PropertyName name) {
+        switch (name) {
+            case CACHE_ARCHIVES:
+                return DistributedCache.CACHE_ARCHIVES;
+            case CACHE_FILES:
+                return DistributedCache.CACHE_FILES;
+            case CACHE_SYMLINK:
+                return DistributedCache.CACHE_SYMLINK;
+        }
+
+        return "";
+    }
+}

Modified: incubator/hcatalog/branches/branch-0.4/shims/src/23/java/org/apache/hcatalog/shims/HCatHadoopShims23.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/branches/branch-0.4/shims/src/23/java/org/apache/hcatalog/shims/HCatHadoopShims23.java?rev=1374902&r1=1374901&r2=1374902&view=diff
==============================================================================
--- incubator/hcatalog/branches/branch-0.4/shims/src/23/java/org/apache/hcatalog/shims/HCatHadoopShims23.java (original)
+++ incubator/hcatalog/branches/branch-0.4/shims/src/23/java/org/apache/hcatalog/shims/HCatHadoopShims23.java Mon Aug 20 03:08:56 2012
@@ -17,27 +17,105 @@
  */
 package org.apache.hcatalog.shims;
 
+import java.io.IOException;
+import java.net.InetSocketAddress;
+
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.mapred.Reporter;
+import org.apache.hadoop.mapreduce.Job;
 import org.apache.hadoop.mapreduce.JobContext;
 import org.apache.hadoop.mapreduce.JobID;
+import org.apache.hadoop.mapreduce.OutputFormat;
 import org.apache.hadoop.mapreduce.TaskAttemptContext;
 import org.apache.hadoop.mapreduce.TaskAttemptID;
+import org.apache.hadoop.mapreduce.TaskID;
+import org.apache.hadoop.mapreduce.TaskType;
 import org.apache.hadoop.mapreduce.task.JobContextImpl;
 import org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl;
+import org.apache.hadoop.util.Progressable;
+import org.apache.pig.ResourceSchema;
+
+import org.apache.hadoop.mapreduce.MRJobConfig;
+import org.apache.hadoop.net.NetUtils;
 
 public class HCatHadoopShims23 implements HCatHadoopShims {
+    @Override
+    public TaskID createTaskID() {
+        return new TaskID("", 0, TaskType.MAP, 0);
+    }
 
-	@Override
-	public TaskAttemptContext createTaskAttemptContext(Configuration conf,
-			TaskAttemptID taskId) {
-        return new TaskAttemptContextImpl(conf, taskId);
-	}
+    @Override
+    public TaskAttemptID createTaskAttemptID() {
+        return new TaskAttemptID("", 0, TaskType.MAP, 0, 0);
+    }
+
+    @Override
+    public org.apache.hadoop.mapreduce.TaskAttemptContext createTaskAttemptContext(Configuration conf,
+            org.apache.hadoop.mapreduce.TaskAttemptID taskId) {
+        return new org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl(conf, taskId);
+    }
 
-	@Override
+    @Override
+    public org.apache.hadoop.mapred.TaskAttemptContext createTaskAttemptContext(org.apache.hadoop.mapred.JobConf conf,
+            org.apache.hadoop.mapred.TaskAttemptID taskId, Progressable progressable) {
+        org.apache.hadoop.mapred.TaskAttemptContext newContext = null;
+        try {
+            java.lang.reflect.Constructor construct = org.apache.hadoop.mapred.TaskAttemptContextImpl.class.getDeclaredConstructor(
+                    org.apache.hadoop.mapred.JobConf.class, org.apache.hadoop.mapred.TaskAttemptID.class,
+                    Reporter.class);
+            construct.setAccessible(true);
+            newContext = (org.apache.hadoop.mapred.TaskAttemptContext)construct.newInstance(conf, taskId, (Reporter)progressable);
+        } catch (Exception e) {
+            throw new RuntimeException(e);
+        }
+        return newContext;
+    }
+
+    @Override
     public JobContext createJobContext(Configuration conf,
             JobID jobId) {
-        JobContext newContext = new JobContextImpl(conf, jobId);
+        JobContext ctxt = new JobContextImpl(conf, jobId);
+
+        return ctxt;
+    }
+
+    @Override
+    public org.apache.hadoop.mapred.JobContext createJobContext(org.apache.hadoop.mapred.JobConf conf,
+            org.apache.hadoop.mapreduce.JobID jobId, Progressable progressable) {
+        org.apache.hadoop.mapred.JobContext newContext = 
+            new org.apache.hadoop.mapred.JobContextImpl(conf, jobId, (org.apache.hadoop.mapred.Reporter)progressable);
         return newContext;
     }
 
+    @Override
+    public void commitJob(OutputFormat outputFormat, ResourceSchema schema,
+            String arg1, Job job) throws IOException {
+        // Do nothing as this was fixed by MAPREDUCE-1447.
+    }
+
+    @Override
+    public void abortJob(OutputFormat outputFormat, Job job) throws IOException {
+        // Do nothing as this was fixed by MAPREDUCE-1447.
+    }
+
+    @Override
+    public InetSocketAddress getResourceManagerAddress(Configuration conf) {
+        String addr = conf.get("yarn.resourcemanager.address", "localhost:8032");
+
+        return NetUtils.createSocketAddr(addr);
+    }
+
+    @Override
+    public String getPropertyName(PropertyName name) {
+        switch (name) {
+            case CACHE_ARCHIVES:
+                return MRJobConfig.CACHE_ARCHIVES;
+            case CACHE_FILES:
+                return MRJobConfig.CACHE_FILES;
+            case CACHE_SYMLINK:
+                return MRJobConfig.CACHE_SYMLINK;
+        }
+
+        return "";
+    }
 }

Modified: incubator/hcatalog/branches/branch-0.4/src/java/org/apache/hadoop/mapred/HCatMapRedUtil.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/branches/branch-0.4/src/java/org/apache/hadoop/mapred/HCatMapRedUtil.java?rev=1374902&r1=1374901&r2=1374902&view=diff
==============================================================================
--- incubator/hcatalog/branches/branch-0.4/src/java/org/apache/hadoop/mapred/HCatMapRedUtil.java (original)
+++ incubator/hcatalog/branches/branch-0.4/src/java/org/apache/hadoop/mapred/HCatMapRedUtil.java Mon Aug 20 03:08:56 2012
@@ -19,6 +19,8 @@
 package org.apache.hadoop.mapred;
 
 import org.apache.hadoop.util.Progressable;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hcatalog.shims.HCatHadoopShims;
 
 public class HCatMapRedUtil {
 
@@ -28,8 +30,12 @@ public class HCatMapRedUtil {
                                                              Reporter.NULL);
     }
 
+    public static org.apache.hadoop.mapreduce.TaskAttemptContext createTaskAttemptContext(Configuration conf, org.apache.hadoop.mapreduce.TaskAttemptID id) {
+        return  HCatHadoopShims.Instance.get().createTaskAttemptContext(conf,id);
+    }
+
     public static TaskAttemptContext createTaskAttemptContext(JobConf conf, TaskAttemptID id, Progressable progressable) {
-        return  new TaskAttemptContext(conf,id,progressable);
+        return HCatHadoopShims.Instance.get ().createTaskAttemptContext(conf, id, (Reporter) progressable);
     }
 
     public static org.apache.hadoop.mapred.JobContext createJobContext(org.apache.hadoop.mapreduce.JobContext context) {
@@ -39,6 +45,6 @@ public class HCatMapRedUtil {
     }
 
     public static JobContext createJobContext(JobConf conf, org.apache.hadoop.mapreduce.JobID id, Progressable progressable) {
-        return  new JobContext(conf,id,progressable);
+        return HCatHadoopShims.Instance.get ().createJobContext(conf, id, (Reporter) progressable);
     }
 }

Modified: incubator/hcatalog/branches/branch-0.4/src/java/org/apache/hcatalog/data/transfer/impl/HCatInputFormatReader.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/branches/branch-0.4/src/java/org/apache/hcatalog/data/transfer/impl/HCatInputFormatReader.java?rev=1374902&r1=1374901&r2=1374902&view=diff
==============================================================================
--- incubator/hcatalog/branches/branch-0.4/src/java/org/apache/hcatalog/data/transfer/impl/HCatInputFormatReader.java (original)
+++ incubator/hcatalog/branches/branch-0.4/src/java/org/apache/hcatalog/data/transfer/impl/HCatInputFormatReader.java Mon Aug 20 03:08:56 2012
@@ -39,6 +39,7 @@ import org.apache.hcatalog.data.transfer
 import org.apache.hcatalog.data.transfer.state.StateProvider;
 import org.apache.hcatalog.mapreduce.HCatInputFormat;
 import org.apache.hcatalog.mapreduce.InputJobInfo;
+import org.apache.hcatalog.shims.HCatHadoopShims;
 
 /** This reader reads via {@link HCatInputFormat}
  * 
@@ -65,7 +66,8 @@ public class HCatInputFormatReader exten
 			HCatInputFormat.setInput(job, jobInfo);
 			HCatInputFormat hcif = new HCatInputFormat();
 			ReaderContext cntxt = new ReaderContext();
-			cntxt.setInputSplits(hcif.getSplits(new JobContext(job.getConfiguration(), null)));
+            cntxt.setInputSplits(hcif.getSplits(
+                        HCatHadoopShims.Instance.get().createJobContext(job.getConfiguration(), null)));
 			cntxt.setConf(job.getConfiguration());
 			return cntxt;
 		} catch (IOException e) {
@@ -81,7 +83,7 @@ public class HCatInputFormatReader exten
 		HCatInputFormat inpFmt = new HCatInputFormat();
 		RecordReader<WritableComparable, HCatRecord> rr;
 		try {
-			TaskAttemptContext cntxt = new TaskAttemptContext(conf, new TaskAttemptID());
+            TaskAttemptContext cntxt = HCatHadoopShims.Instance.get().createTaskAttemptContext(conf, new TaskAttemptID());
 			rr = inpFmt.createRecordReader(split, cntxt);
 			rr.initialize(split, cntxt);
 		} catch (IOException e) {

Modified: incubator/hcatalog/branches/branch-0.4/src/java/org/apache/hcatalog/data/transfer/impl/HCatOutputFormatWriter.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/branches/branch-0.4/src/java/org/apache/hcatalog/data/transfer/impl/HCatOutputFormatWriter.java?rev=1374902&r1=1374901&r2=1374902&view=diff
==============================================================================
--- incubator/hcatalog/branches/branch-0.4/src/java/org/apache/hcatalog/data/transfer/impl/HCatOutputFormatWriter.java (original)
+++ incubator/hcatalog/branches/branch-0.4/src/java/org/apache/hcatalog/data/transfer/impl/HCatOutputFormatWriter.java Mon Aug 20 03:08:56 2012
@@ -32,6 +32,7 @@ import org.apache.hadoop.mapreduce.Recor
 import org.apache.hadoop.mapreduce.TaskAttemptContext;
 import org.apache.hadoop.mapreduce.TaskAttemptID;
 import org.apache.hadoop.mapreduce.TaskID;
+import org.apache.hadoop.mapreduce.TaskType;
 import org.apache.hcatalog.common.ErrorType;
 import org.apache.hcatalog.common.HCatException;
 import org.apache.hcatalog.data.HCatRecord;
@@ -41,6 +42,7 @@ import org.apache.hcatalog.data.transfer
 import org.apache.hcatalog.data.transfer.state.StateProvider;
 import org.apache.hcatalog.mapreduce.HCatOutputFormat;
 import org.apache.hcatalog.mapreduce.OutputJobInfo;
+import org.apache.hcatalog.shims.HCatHadoopShims;
 
 /** This writer writes via {@link HCatOutputFormat}
  * 
@@ -65,7 +67,8 @@ public class HCatOutputFormatWriter exte
 			HCatOutputFormat.setSchema(job, HCatOutputFormat.getTableSchema(job));
 			HCatOutputFormat outFormat = new HCatOutputFormat();
 			outFormat.checkOutputSpecs(job);
-			outFormat.getOutputCommitter(new TaskAttemptContext(job.getConfiguration(), new TaskAttemptID())).setupJob(job);
+            outFormat.getOutputCommitter(HCatHadoopShims.Instance.get().createTaskAttemptContext
+                    (job.getConfiguration(), HCatHadoopShims.Instance.get().createTaskAttemptID())).setupJob(job);
 		} catch (IOException e) {
 			throw new HCatException(ErrorType.ERROR_NOT_INITIALIZED, e);
 		} catch (InterruptedException e) {
@@ -82,7 +85,8 @@ public class HCatOutputFormatWriter exte
 		int id = sp.getId();
 		setVarsInConf(id);
 		HCatOutputFormat outFormat = new HCatOutputFormat();
-		TaskAttemptContext cntxt = new TaskAttemptContext(conf, new TaskAttemptID(new TaskID(), id));
+        TaskAttemptContext cntxt = HCatHadoopShims.Instance.get().createTaskAttemptContext
+            (conf, new TaskAttemptID(HCatHadoopShims.Instance.get().createTaskID(), id));
 		OutputCommitter committer = null;
 		RecordWriter<WritableComparable<?>, HCatRecord> writer;
 		try {
@@ -121,8 +125,9 @@ public class HCatOutputFormatWriter exte
 	@Override
 	public void commit(WriterContext context) throws HCatException {
 		try {
-			new HCatOutputFormat().getOutputCommitter(new TaskAttemptContext(context.getConf(), new TaskAttemptID()))
-			.commitJob(new JobContext(context.getConf(), null));
+            new HCatOutputFormat().getOutputCommitter(HCatHadoopShims.Instance.get().createTaskAttemptContext
+                (context.getConf(), HCatHadoopShims.Instance.get().createTaskAttemptID()))
+                .commitJob(HCatHadoopShims.Instance.get().createJobContext(context.getConf(), null));
 		} catch (IOException e) {
 			throw new HCatException(ErrorType.ERROR_NOT_INITIALIZED, e);
 		} catch (InterruptedException e) {
@@ -133,8 +138,9 @@ public class HCatOutputFormatWriter exte
 	@Override
 	public void abort(WriterContext context) throws HCatException {
 		try {
-			new HCatOutputFormat().getOutputCommitter(new TaskAttemptContext(context.getConf(), new TaskAttemptID()))
-			.abortJob(new JobContext(context.getConf(), null),State.FAILED);
+            new HCatOutputFormat().getOutputCommitter(HCatHadoopShims.Instance.get().createTaskAttemptContext
+                (context.getConf(), HCatHadoopShims.Instance.get().createTaskAttemptID()))
+                .abortJob(HCatHadoopShims.Instance.get().createJobContext(context.getConf(), null),State.FAILED);
 		} catch (IOException e) {
 			throw new HCatException(ErrorType.ERROR_NOT_INITIALIZED, e);
 		} catch (InterruptedException e) {

Modified: incubator/hcatalog/branches/branch-0.4/src/java/org/apache/hcatalog/data/transfer/state/StateProvider.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/branches/branch-0.4/src/java/org/apache/hcatalog/data/transfer/state/StateProvider.java?rev=1374902&r1=1374901&r2=1374902&view=diff
==============================================================================
--- incubator/hcatalog/branches/branch-0.4/src/java/org/apache/hcatalog/data/transfer/state/StateProvider.java (original)
+++ incubator/hcatalog/branches/branch-0.4/src/java/org/apache/hcatalog/data/transfer/state/StateProvider.java Mon Aug 20 03:08:56 2012
@@ -18,9 +18,6 @@
 
 package org.apache.hcatalog.data.transfer.state;
 
-import org.apache.hadoop.mapred.JobTracker;
-import org.apache.hadoop.mapred.TaskTracker;
-
 /** If external system wants to communicate any state to slaves, they can do so via this interface.
  * One example of this in case of Map-Reduce is ids assigned by {@link JobTracker} to 
  * {@link TaskTracker}

Modified: incubator/hcatalog/branches/branch-0.4/src/java/org/apache/hcatalog/mapreduce/FileOutputCommitterContainer.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/branches/branch-0.4/src/java/org/apache/hcatalog/mapreduce/FileOutputCommitterContainer.java?rev=1374902&r1=1374901&r2=1374902&view=diff
==============================================================================
--- incubator/hcatalog/branches/branch-0.4/src/java/org/apache/hcatalog/mapreduce/FileOutputCommitterContainer.java (original)
+++ incubator/hcatalog/branches/branch-0.4/src/java/org/apache/hcatalog/mapreduce/FileOutputCommitterContainer.java Mon Aug 20 03:08:56 2012
@@ -47,11 +47,16 @@ import org.apache.hcatalog.data.schema.H
 import org.apache.hcatalog.data.schema.HCatSchema;
 import org.apache.hcatalog.data.schema.HCatSchemaUtils;
 import org.apache.hcatalog.har.HarOutputCommitterPostProcessor;
+import org.apache.hcatalog.shims.HCatHadoopShims;
 import org.apache.thrift.TException;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import java.io.BufferedWriter;
+import java.io.FileWriter;
 import java.io.IOException;
+import java.io.PrintWriter;
+import java.io.Writer;
 import java.net.URI;
 import java.util.ArrayList;
 import java.util.HashMap;
@@ -468,7 +473,7 @@ class FileOutputCommitterContainer exten
                     LinkedHashMap<String, String> fullPartSpec = new LinkedHashMap<String, String>();
                     Warehouse.makeSpecFromName(fullPartSpec, st.getPath());
                     partitionsDiscoveredByPath.put(st.getPath().toString(),fullPartSpec);
-                    JobContext currContext = new JobContext(context.getConfiguration(),context.getJobID());
+                    JobContext currContext = HCatHadoopShims.Instance.get().createJobContext(context.getConfiguration(),context.getJobID());
                     HCatOutputFormat.configureOutputStorageHandler(context, jobInfo, fullPartSpec);
                     contextDiscoveredByPath.put(st.getPath().toString(),currContext);
                 }

Modified: incubator/hcatalog/branches/branch-0.4/src/java/org/apache/hcatalog/mapreduce/FileRecordWriterContainer.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/branches/branch-0.4/src/java/org/apache/hcatalog/mapreduce/FileRecordWriterContainer.java?rev=1374902&r1=1374901&r2=1374902&view=diff
==============================================================================
--- incubator/hcatalog/branches/branch-0.4/src/java/org/apache/hcatalog/mapreduce/FileRecordWriterContainer.java (original)
+++ incubator/hcatalog/branches/branch-0.4/src/java/org/apache/hcatalog/mapreduce/FileRecordWriterContainer.java Mon Aug 20 03:08:56 2012
@@ -147,6 +147,8 @@ class FileRecordWriterContainer extends 
                 if (baseOutputCommitter.needsTaskCommit(currContext)){
                     baseOutputCommitter.commitTask(currContext);
                 }
+                org.apache.hadoop.mapred.JobContext currJobContext = HCatMapRedUtil.createJobContext(currContext);
+                baseOutputCommitter.commitJob(currJobContext);
             }
         } else {
             getBaseRecordWriter().close(reporter);
@@ -156,7 +158,6 @@ class FileRecordWriterContainer extends 
     @Override
     public void write(WritableComparable<?> key, HCatRecord value) throws IOException,
             InterruptedException {
-
         org.apache.hadoop.mapred.RecordWriter localWriter;
         ObjectInspector localObjectInspector;
         SerDe localSerDe;

Modified: incubator/hcatalog/branches/branch-0.4/src/java/org/apache/hcatalog/mapreduce/MultiOutputFormat.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/branches/branch-0.4/src/java/org/apache/hcatalog/mapreduce/MultiOutputFormat.java?rev=1374902&r1=1374901&r2=1374902&view=diff
==============================================================================
--- incubator/hcatalog/branches/branch-0.4/src/java/org/apache/hcatalog/mapreduce/MultiOutputFormat.java (original)
+++ incubator/hcatalog/branches/branch-0.4/src/java/org/apache/hcatalog/mapreduce/MultiOutputFormat.java Mon Aug 20 03:08:56 2012
@@ -47,6 +47,7 @@ import org.apache.hadoop.mapreduce.Recor
 import org.apache.hadoop.mapreduce.TaskAttemptContext;
 import org.apache.hadoop.mapreduce.TaskInputOutputContext;
 import org.apache.hadoop.util.ReflectionUtils;
+import org.apache.hcatalog.shims.HCatHadoopShims;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -144,13 +145,13 @@ public class MultiOutputFormat extends O
 
     static {
         configsToOverride.add("mapred.output.dir");
-        configsToOverride.add(DistributedCache.CACHE_SYMLINK);
+        configsToOverride.add(HCatHadoopShims.Instance.get().getPropertyName(HCatHadoopShims.PropertyName.CACHE_SYMLINK));
         configsToMerge.put(JobContext.JOB_NAMENODES, COMMA_DELIM);
         configsToMerge.put("tmpfiles", COMMA_DELIM);
         configsToMerge.put("tmpjars", COMMA_DELIM);
         configsToMerge.put("tmparchives", COMMA_DELIM);
-        configsToMerge.put(DistributedCache.CACHE_ARCHIVES, COMMA_DELIM);
-        configsToMerge.put(DistributedCache.CACHE_FILES, COMMA_DELIM);
+        configsToMerge.put(HCatHadoopShims.Instance.get().getPropertyName(HCatHadoopShims.PropertyName.CACHE_ARCHIVES), COMMA_DELIM);
+        configsToMerge.put(HCatHadoopShims.Instance.get().getPropertyName(HCatHadoopShims.PropertyName.CACHE_FILES), COMMA_DELIM);
         configsToMerge.put("mapred.job.classpath.archives", System.getProperty("path.separator"));
         configsToMerge.put("mapred.job.classpath.files", System.getProperty("path.separator"));
     }
@@ -175,7 +176,7 @@ public class MultiOutputFormat extends O
      */
     public static JobContext getJobContext(String alias, JobContext context) {
         String aliasConf = context.getConfiguration().get(getAliasConfName(alias));
-        JobContext aliasContext = new JobContext(context.getConfiguration(), context.getJobID());
+        JobContext aliasContext = HCatHadoopShims.Instance.get().createJobContext(context.getConfiguration(), context.getJobID());
         addToConfig(aliasConf, aliasContext.getConfiguration());
         return aliasContext;
     }
@@ -189,8 +190,7 @@ public class MultiOutputFormat extends O
      */
     public static TaskAttemptContext getTaskAttemptContext(String alias, TaskAttemptContext context) {
         String aliasConf = context.getConfiguration().get(getAliasConfName(alias));
-        TaskAttemptContext aliasContext = new TaskAttemptContext(context.getConfiguration(),
-                context.getTaskAttemptID());
+        TaskAttemptContext aliasContext = HCatHadoopShims.Instance.get().createTaskAttemptContext(context.getConfiguration(), context.getTaskAttemptID());
         addToConfig(aliasConf, aliasContext.getConfiguration());
         return aliasContext;
     }

Modified: incubator/hcatalog/branches/branch-0.4/src/java/org/apache/hcatalog/mapreduce/ProgressReporter.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/branches/branch-0.4/src/java/org/apache/hcatalog/mapreduce/ProgressReporter.java?rev=1374902&r1=1374901&r2=1374902&view=diff
==============================================================================
--- incubator/hcatalog/branches/branch-0.4/src/java/org/apache/hcatalog/mapreduce/ProgressReporter.java (original)
+++ incubator/hcatalog/branches/branch-0.4/src/java/org/apache/hcatalog/mapreduce/ProgressReporter.java Mon Aug 20 03:08:56 2012
@@ -74,6 +74,12 @@ class ProgressReporter extends StatusRep
     return null;
   }
 
+  public float getProgress() {
+      /* Required to build against 0.23 Reporter and StatusReporter. */
+      /* TODO: determine the progress. */
+      return 0.0f;
+  }
+
   @Override
   public void progress() {
     if (context != null) {

Modified: incubator/hcatalog/branches/branch-0.4/src/java/org/apache/hcatalog/mapreduce/Security.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/branches/branch-0.4/src/java/org/apache/hcatalog/mapreduce/Security.java?rev=1374902&r1=1374901&r2=1374902&view=diff
==============================================================================
--- incubator/hcatalog/branches/branch-0.4/src/java/org/apache/hcatalog/mapreduce/Security.java (original)
+++ incubator/hcatalog/branches/branch-0.4/src/java/org/apache/hcatalog/mapreduce/Security.java Mon Aug 20 03:08:56 2012
@@ -32,7 +32,6 @@ import org.apache.hadoop.hive.metastore.
 import org.apache.hadoop.hive.metastore.api.MetaException;
 import org.apache.hadoop.hive.thrift.DelegationTokenSelector;
 import org.apache.hadoop.io.Text;
-import org.apache.hadoop.mapred.JobTracker;
 import org.apache.hadoop.mapreduce.Job;
 import org.apache.hadoop.mapreduce.JobContext;
 import org.apache.hadoop.security.UserGroupInformation;
@@ -42,6 +41,7 @@ import org.apache.hadoop.security.token.
 import org.apache.hadoop.security.token.delegation.AbstractDelegationTokenIdentifier;
 import org.apache.hcatalog.common.HCatConstants;
 import org.apache.hcatalog.common.HCatUtil;
+import org.apache.hcatalog.shims.HCatHadoopShims;
 import org.apache.thrift.TException;
 
 final class Security {
@@ -141,9 +141,8 @@ final class Security {
         if (harRequested){
           TokenSelector<? extends TokenIdentifier> jtTokenSelector =
             new org.apache.hadoop.mapreduce.security.token.delegation.DelegationTokenSelector();
-          Token jtToken =
-            jtTokenSelector.selectToken(org.apache.hadoop.security.SecurityUtil.buildTokenService(JobTracker.getAddress(conf)),
-                                            ugi.getTokens());
+          Token jtToken = jtTokenSelector.selectToken(org.apache.hadoop.security.SecurityUtil.buildTokenService(HCatHadoopShims.Instance.get().getResourceManagerAddress(conf)), ugi.getTokens());
+
           if(jtToken == null) {
             //we don't need to cancel this token as the TokenRenewer for JT tokens
             //takes care of cancelling them

Modified: incubator/hcatalog/branches/branch-0.4/src/java/org/apache/hcatalog/pig/HCatStorer.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/branches/branch-0.4/src/java/org/apache/hcatalog/pig/HCatStorer.java?rev=1374902&r1=1374901&r2=1374902&view=diff
==============================================================================
--- incubator/hcatalog/branches/branch-0.4/src/java/org/apache/hcatalog/pig/HCatStorer.java (original)
+++ incubator/hcatalog/branches/branch-0.4/src/java/org/apache/hcatalog/pig/HCatStorer.java Mon Aug 20 03:08:56 2012
@@ -30,6 +30,8 @@ import org.apache.hadoop.mapreduce.Job;
 import org.apache.hadoop.mapreduce.JobStatus.State;
 import org.apache.hadoop.mapreduce.OutputFormat;
 import org.apache.hadoop.mapreduce.TaskAttemptID;
+import org.apache.hadoop.mapreduce.TaskType;
+import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
 import org.apache.hadoop.security.Credentials;
 import org.apache.hcatalog.common.HCatConstants;
 import org.apache.hcatalog.common.HCatException;
@@ -151,37 +153,11 @@ public class HCatStorer extends HCatBase
 
   @Override
   public void storeSchema(ResourceSchema schema, String arg1, Job job) throws IOException {
-    if( job.getConfiguration().get("mapred.job.tracker", "").equalsIgnoreCase("local") ) {
-      try {
-      //In local mode, mapreduce will not call OutputCommitter.cleanupJob.
-      //Calling it from here so that the partition publish happens.
-      //This call needs to be removed after MAPREDUCE-1447 is fixed.
-        getOutputFormat().getOutputCommitter(HCatHadoopShims.Instance.get().createTaskAttemptContext(
-        		job.getConfiguration(), new TaskAttemptID())).commitJob(job);
-      } catch (IOException e) {
-        throw new IOException("Failed to cleanup job",e);
-      } catch (InterruptedException e) {
-        throw new IOException("Failed to cleanup job",e);
-      }
-    }
+    HCatHadoopShims.Instance.get().commitJob(getOutputFormat(), schema, arg1, job);
   }
 
-   @Override
-    public void cleanupOnFailure(String location, Job job) throws IOException {
-        if (job.getConfiguration().get("mapred.job.tracker", "")
-                .equalsIgnoreCase("local")) {
-            try {
-                // This call needs to be removed after MAPREDUCE-1447 is fixed.
-                getOutputFormat().getOutputCommitter(
-                        HCatHadoopShims.Instance.get()
-                                .createTaskAttemptContext(
-                                        job.getConfiguration(),
-                                        new TaskAttemptID())).abortJob(job, State.FAILED);
-            } catch (IOException e) {
-                throw new IOException("Failed to abort job", e);
-            } catch (InterruptedException e) {
-                throw new IOException("Failed to abort job", e);
-            }
-        }
-    }
+  @Override
+  public void cleanupOnFailure(String location, Job job) throws IOException {
+      HCatHadoopShims.Instance.get().abortJob(getOutputFormat(), job);
+  }
 }

Modified: incubator/hcatalog/branches/branch-0.4/src/java/org/apache/hcatalog/shims/HCatHadoopShims.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/branches/branch-0.4/src/java/org/apache/hcatalog/shims/HCatHadoopShims.java?rev=1374902&r1=1374901&r2=1374902&view=diff
==============================================================================
--- incubator/hcatalog/branches/branch-0.4/src/java/org/apache/hcatalog/shims/HCatHadoopShims.java (original)
+++ incubator/hcatalog/branches/branch-0.4/src/java/org/apache/hcatalog/shims/HCatHadoopShims.java Mon Aug 20 03:08:56 2012
@@ -17,46 +17,76 @@
  */
 package org.apache.hcatalog.shims;
 
+import java.io.IOException;
+import java.net.InetSocketAddress;
+
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hive.shims.ShimLoader;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.mapreduce.Job;
 import org.apache.hadoop.mapreduce.JobContext;
 import org.apache.hadoop.mapreduce.JobID;
+import org.apache.hadoop.mapreduce.OutputFormat;
 import org.apache.hadoop.mapreduce.TaskAttemptContext;
 import org.apache.hadoop.mapreduce.TaskAttemptID;
+import org.apache.hadoop.mapreduce.TaskID;
+import org.apache.hadoop.util.Progressable;
+import org.apache.pig.ResourceSchema;
 
 /**
- * Shim layer to abstract differences between Hadoop 0.20 and 0.23 (HCATALOG-179).
- * This mirrors Hive shims, but is kept separate for HCatalog dependencies.
+ * Shim layer to abstract differences between Hadoop 0.20 and 0.23
+ * (HCATALOG-179). This mirrors Hive shims, but is kept separate for HCatalog
+ * dependencies.
  **/
 public interface HCatHadoopShims {
 
-	public static abstract class Instance {
-		static HCatHadoopShims instance = selectShim();
-		public static HCatHadoopShims get() {
-			return instance;
-		}
-
-		private static HCatHadoopShims selectShim() {
-			// piggyback on Hive's detection logic
-			String major = ShimLoader.getMajorVersion();
-			String shimFQN = "org.apache.hcatalog.shims.HCatHadoopShims20S";
-			if (major.startsWith("0.23")) {
-				shimFQN = "org.apache.hcatalog.shims.HCatHadoopShims23";
-			}
-			try {
-				Class<? extends HCatHadoopShims> clasz =
-						Class.forName(shimFQN).asSubclass(HCatHadoopShims.class);
-				return clasz.newInstance();
-			} catch (Exception e) {
-				throw new RuntimeException("Failed to instantiate: " + shimFQN, e);
-			}
-		}
-	}
+  enum PropertyName { CACHE_ARCHIVES, CACHE_FILES, CACHE_SYMLINK };
+
+  public static abstract class Instance {
+    static HCatHadoopShims instance = selectShim();
+
+    public static HCatHadoopShims get() {
+      return instance;
+    }
+
+    private static HCatHadoopShims selectShim() {
+      // piggyback on Hive's detection logic
+      String major = ShimLoader.getMajorVersion();
+      String shimFQN = "org.apache.hcatalog.shims.HCatHadoopShims20S";
+      if (major.startsWith("0.23")) {
+        shimFQN = "org.apache.hcatalog.shims.HCatHadoopShims23";
+      }
+      try {
+        Class<? extends HCatHadoopShims> clasz = Class.forName(shimFQN)
+            .asSubclass(HCatHadoopShims.class);
+        return clasz.newInstance();
+      } catch (Exception e) {
+        throw new RuntimeException("Failed to instantiate: " + shimFQN, e);
+      }
+    }
+  }
+
+  public TaskID createTaskID();
+
+  public TaskAttemptID createTaskAttemptID();
+
+  public org.apache.hadoop.mapreduce.TaskAttemptContext createTaskAttemptContext(Configuration conf,
+          TaskAttemptID taskId);
+
+  public org.apache.hadoop.mapred.TaskAttemptContext createTaskAttemptContext(JobConf conf,
+          org.apache.hadoop.mapred.TaskAttemptID taskId, Progressable progressable);
+
+  public JobContext createJobContext(Configuration conf, JobID jobId);
+
+  public org.apache.hadoop.mapred.JobContext createJobContext(JobConf conf, JobID jobId, Progressable progressable);
+
+  public void commitJob(OutputFormat outputFormat, ResourceSchema schema,
+          String arg1, Job job) throws IOException;
 
-    public TaskAttemptContext createTaskAttemptContext(Configuration conf,
-                                TaskAttemptID taskId);
+  public void abortJob(OutputFormat outputFormat, Job job) throws IOException;
 
-    public JobContext createJobContext(Configuration conf,
-            JobID jobId);
+  /* Referring to job tracker in 0.20 and resource manager in 0.23 */
+  public InetSocketAddress getResourceManagerAddress(Configuration conf);
 
+  public String getPropertyName(PropertyName name);
 }

Modified: incubator/hcatalog/branches/branch-0.4/src/test/e2e/hcatalog/build.xml
URL: http://svn.apache.org/viewvc/incubator/hcatalog/branches/branch-0.4/src/test/e2e/hcatalog/build.xml?rev=1374902&r1=1374901&r2=1374902&view=diff
==============================================================================
--- incubator/hcatalog/branches/branch-0.4/src/test/e2e/hcatalog/build.xml (original)
+++ incubator/hcatalog/branches/branch-0.4/src/test/e2e/hcatalog/build.xml Mon Aug 20 03:08:56 2012
@@ -49,6 +49,12 @@
   <!-- Default value for output directory -->
   <property name="harness.PH_LOCAL" value="out"/>
 
+  <property name="hadoopversion" value="20" />
+
+  <condition property="isHadoop23">
+    <equals arg1="${hadoopversion}" arg2="23"/>
+  </condition>
+
   <!-- Build the UDFs -->
   <target name="udfs" >
     <ant dir="${udf.java.dir}"/>
@@ -56,7 +62,14 @@
 
   <path id="hadoop.core.jar.location">
     <fileset dir="${hadoop.core.path}">
-      <include name="hadoop-core-*.jar"/>
+      <include name="hadoop-core-*.jar" unless="isHadoop23"/>
+      <include name="**/hadoop-common-*.jar" if="isHadoop23"/>
+      <include name="**/hadoop-auth-*.jar" if="isHadoop23"/>
+      <include name="**/hadoop-hdfs-*.jar" if="isHadoop23"/>
+      <include name="**/hadoop-mapreduce-client-core-*.jar" if="isHadoop23"/>
+      <include name="**/hadoop-yarn-api-*.jar" if="isHadoop23"/>
+      <include name="**/hadoop-yarn-common-*.jar" if="isHadoop23"/>
+      <include name="**/hadoop-annotations-*.jar" if="isHadoop23"/>
     </fileset>
   </path>
 
@@ -161,6 +174,18 @@
         </and>
       </condition>
     </fail>
+    <fail message="Please set the property harness.cluster.conf to the location Hadoop conf is installed ">
+      <condition>
+        <and>
+          <not>
+            <isset property="harness.cluster.conf"/>
+          </not>
+          <not>
+            <contains string="${harness.conf}" substring="rpm.conf"/>
+          </not>
+        </and>
+      </condition>
+    </fail>
     <fail message="Please set the property hive.home to the location Hive is installed ">
       <condition>
         <and>
@@ -236,6 +261,7 @@
       <env key="HARNESS_ROOT" value="."/>
       <env key="PH_LOCAL" value="${harness.PH_LOCAL}"/>
       <env key="HADOOP_HOME" value="${hadoop.home}"/>
+      <env key="HADOOP_CONF_DIR" value="${harness.cluster.conf}"/>
       <env key="HIVE_HOME" value="${hive.home}"/>
       <env key="HCAT_HOME" value="${hcat.home}"/>
       <env key="PIG_HOME" value="${pig.home}"/>

Modified: incubator/hcatalog/branches/branch-0.4/src/test/e2e/hcatalog/conf/default.conf
URL: http://svn.apache.org/viewvc/incubator/hcatalog/branches/branch-0.4/src/test/e2e/hcatalog/conf/default.conf?rev=1374902&r1=1374901&r2=1374902&view=diff
==============================================================================
--- incubator/hcatalog/branches/branch-0.4/src/test/e2e/hcatalog/conf/default.conf (original)
+++ incubator/hcatalog/branches/branch-0.4/src/test/e2e/hcatalog/conf/default.conf Mon Aug 20 03:08:56 2012
@@ -66,7 +66,7 @@ $cfg = {
     , 'pigbin'           => "$ENV{'PIG_HOME'}/bin/pig"
 
     #HADOOP
-    , 'hadoopconfdir'    => "$ENV{'HADOOP_HOME'}/conf"
+	, 'hadoopconfdir'    => "$ENV{'HADOOP_CONF_DIR'}"
     , 'hadoopbin'        => "$ENV{'HADOOP_HOME'}/bin/hadoop"
 
     #HIVE

Modified: incubator/hcatalog/branches/branch-0.4/src/test/e2e/hcatalog/drivers/Util.pm
URL: http://svn.apache.org/viewvc/incubator/hcatalog/branches/branch-0.4/src/test/e2e/hcatalog/drivers/Util.pm?rev=1374902&r1=1374901&r2=1374902&view=diff
==============================================================================
--- incubator/hcatalog/branches/branch-0.4/src/test/e2e/hcatalog/drivers/Util.pm (original)
+++ incubator/hcatalog/branches/branch-0.4/src/test/e2e/hcatalog/drivers/Util.pm Mon Aug 20 03:08:56 2012
@@ -357,6 +357,7 @@ sub replaceParameters
     # $testCmd
     $cmd =~ s/:INPATH:/$testCmd->{'inpathbase'}/g;
     $cmd =~ s/:OUTPATH:/$outfile/g;
+    $cmd =~ s/:OUTPATHPARENT:/$testCmd->{'outpath'}/g;
     $cmd =~ s/:FUNCPATH:/$testCmd->{'funcjarPath'}/g;
     $cmd =~ s/:PIGPATH:/$testCmd->{'pighome'}/g;
     $cmd =~ s/:RUNID:/$testCmd->{'UID'}/g;

Modified: incubator/hcatalog/branches/branch-0.4/src/test/e2e/hcatalog/tests/hcat.conf
URL: http://svn.apache.org/viewvc/incubator/hcatalog/branches/branch-0.4/src/test/e2e/hcatalog/tests/hcat.conf?rev=1374902&r1=1374901&r2=1374902&view=diff
==============================================================================
--- incubator/hcatalog/branches/branch-0.4/src/test/e2e/hcatalog/tests/hcat.conf (original)
+++ incubator/hcatalog/branches/branch-0.4/src/test/e2e/hcatalog/tests/hcat.conf Mon Aug 20 03:08:56 2012
@@ -154,7 +154,8 @@ describe hcat_droptable_2;\,
 				 'num' => 3
                                 ,'hcat' => q\
 drop table if exists hcat_drop_table_4;
-dfs -cp :INPATH:/studentnull10k/ :OUTPATH:/../drop_table_ext;
+dfs -mkdir :OUTPATHPARENT:/drop_table_ext;
+dfs -cp :INPATH:/studentnull10k/ :OUTPATHPARENT:drop_table_ext;
 \,
 				,'rc'   => 0
 				}, 
@@ -162,7 +163,7 @@ dfs -cp :INPATH:/studentnull10k/ :OUTPAT
 				 'num' => 4
                                 ,'depends_on' => 'HCat_DropTable_3'
                                 ,'hcat' => q\
-create external table hcat_drop_table_4(name string, age int, gpa double) stored as textfile location 'hdfs://:OUTPATH:/../drop_table_ext';
+create external table hcat_drop_table_4(name string, age int, gpa double) stored as textfile location 'hdfs://:OUTPATHPARENT:drop_table_ext';
 describe extended hcat_drop_table_4;
 \,
 				,'rc'   => 0
@@ -180,7 +181,7 @@ drop table hcat_drop_table_4;
 				 'num' => 6
                                 ,'depends_on' => 'HCat_DropTable_5'
                                 ,'hcat' => q\
-dfs -ls :OUTPATH:/../drop_table_ext
+dfs -ls :OUTPATHPARENT:drop_table_ext
 \,
 				,'rc'   => 0
                                 ,'expected_out_regex' => '(.*(\s))*.*drop_table_ext/studentnull10k'

Modified: incubator/hcatalog/branches/branch-0.4/src/test/e2e/hcatalog/tools/generate/generate_data.pl
URL: http://svn.apache.org/viewvc/incubator/hcatalog/branches/branch-0.4/src/test/e2e/hcatalog/tools/generate/generate_data.pl?rev=1374902&r1=1374901&r2=1374902&view=diff
==============================================================================
--- incubator/hcatalog/branches/branch-0.4/src/test/e2e/hcatalog/tools/generate/generate_data.pl (original)
+++ incubator/hcatalog/branches/branch-0.4/src/test/e2e/hcatalog/tools/generate/generate_data.pl Mon Aug 20 03:08:56 2012
@@ -326,46 +326,16 @@ location '$location'
     }
 }
 
-our $hadoopCoreJar = undef;
-
-sub findHadoopJars()
-{
-    my $hadoopClassRoot=$ENV{'HADOOP_HOME'};
-    my $coreJar = `ls $hadoopClassRoot/hadoop-core-*.jar`;
-    #if you do not find hadoop core jar under hadoop home change the path for rpm's
-    if (! $coreJar) {
-      $hadoopClassRoot="$hadoopClassRoot/share/hadoop";
-      $coreJar = `ls $hadoopClassRoot/hadoop-core-*.jar`;
-    }
-
-    my $loggingJar = `ls $hadoopClassRoot/lib/commons-logging-*.jar | grep -v api`;
-    my $cfgJar = `ls $hadoopClassRoot/lib/commons-configuration-*.jar`;
-    my $langJar = `ls $hadoopClassRoot/lib/commons-lang-*.jar`;
-    my $cliJar = `ls $hadoopClassRoot/lib/commons-cli-*.jar`;
-
-    if (! $coreJar) {
-        die 'Please set $HADOOP_HOME\n';
-    }
-
-    chomp $coreJar;
-    chomp $loggingJar;
-    chomp $cfgJar;
-    chomp $langJar;
-    chomp $cliJar;
-    return ($coreJar, $loggingJar, $cfgJar, $langJar, $cliJar);
-}
-
-sub findHiveJars()
+sub findAllJars()
 {
-    if (not defined $ENV{'HIVE_HOME'}) {
-        die 'Please set $HIVE_HOME\n';
+    my @files = <../../../../../build/ivy/lib/hcatalog/*.jar>;
+    my $classpath = "";
+    my $file = undef;
+    foreach $file (@files) {
+        $classpath = $classpath . ":" . $file;
     }
 
-    my $execJar = `ls $ENV{HIVE_HOME}/lib/hive-exec-*.jar`;
-    my $cliJar = `ls $ENV{HIVE_HOME}/lib/hive-cli-*.jar`;
-    chomp $execJar;
-    chomp $cliJar;
-    return ($execJar, $cliJar);
+    return $classpath;
 }
 
 sub getJavaCmd() 
@@ -430,13 +400,9 @@ sub getJavaCmd() 
             }
         } elsif ($format eq "rc") {
             print MYSQL &getBulkCopyCmd($tableName, "\t", "$tableName.plain");
-            my ($hadoopCoreJar, $commonsLoggingJar, $commonsConfigJar,
-                $commonsLangJar, $commonsCliJar) = findHadoopJars();
-            my ($hiveExecJar, $hiveCliJar) = findHiveJars();
+            my $allJars = findAllJars();
             my @cmd = (getJavaCmd(), '-cp',
-                "../tools/generate/java/hive-gen.jar:$hadoopCoreJar:" .
-                "$commonsLoggingJar:$commonsConfigJar:$commonsLangJar:" .
-                "$hiveExecJar",
+                "../tools/generate/java/hive-gen.jar:$allJars",
                 'org.apache.hadoop.hive.tools.generate.RCFileGenerator',
                 'student', $numRows, "$tableName", "$tableName.plain");
             run(\@cmd) or die "Unable to run command [" . join(" ", @cmd) 

Modified: incubator/hcatalog/branches/branch-0.4/src/test/e2e/hcatalog/udfs/java/org/apache/hcatalog/utils/SimpleRead.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/branches/branch-0.4/src/test/e2e/hcatalog/udfs/java/org/apache/hcatalog/utils/SimpleRead.java?rev=1374902&r1=1374901&r2=1374902&view=diff
==============================================================================
--- incubator/hcatalog/branches/branch-0.4/src/test/e2e/hcatalog/udfs/java/org/apache/hcatalog/utils/SimpleRead.java (original)
+++ incubator/hcatalog/branches/branch-0.4/src/test/e2e/hcatalog/udfs/java/org/apache/hcatalog/utils/SimpleRead.java Mon Aug 20 03:08:56 2012
@@ -66,7 +66,6 @@ public class SimpleRead extends Configur
           Text,IntWritable>.Context context) 
     throws IOException ,InterruptedException {
         name = (String) value.get(0);
-System.out.println(name);
         age = (Integer) value.get(1);
         gpa = (Double) value.get(2);
         context.write(new Text(name), new IntWritable(age));

Modified: incubator/hcatalog/branches/branch-0.4/src/test/org/apache/hcatalog/HcatTestUtils.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/branches/branch-0.4/src/test/org/apache/hcatalog/HcatTestUtils.java?rev=1374902&r1=1374901&r2=1374902&view=diff
==============================================================================
--- incubator/hcatalog/branches/branch-0.4/src/test/org/apache/hcatalog/HcatTestUtils.java (original)
+++ incubator/hcatalog/branches/branch-0.4/src/test/org/apache/hcatalog/HcatTestUtils.java Mon Aug 20 03:08:56 2012
@@ -98,4 +98,11 @@ public class HcatTestUtils {
     }
 
   }
+
+  public static boolean isHadoop23() {
+      String version = org.apache.hadoop.util.VersionInfo.getVersion();
+      if (version.matches("\\b0\\.23\\..+\\b"))
+          return true;
+      return false;
+  }
 }

Modified: incubator/hcatalog/branches/branch-0.4/src/test/org/apache/hcatalog/data/TestReaderWriter.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/branches/branch-0.4/src/test/org/apache/hcatalog/data/TestReaderWriter.java?rev=1374902&r1=1374901&r2=1374902&view=diff
==============================================================================
--- incubator/hcatalog/branches/branch-0.4/src/test/org/apache/hcatalog/data/TestReaderWriter.java (original)
+++ incubator/hcatalog/branches/branch-0.4/src/test/org/apache/hcatalog/data/TestReaderWriter.java Mon Aug 20 03:08:56 2012
@@ -136,7 +136,7 @@ public class TestReaderWriter {
 			Assert.assertTrue("Read: " + read.get(1) + "Written: " + written.get(1),  written.get(1).equals(read.get(1)));
 			Assert.assertEquals(2, read.size());
 		}
-		Assert.assertFalse(itr.hasNext());
+		//Assert.assertFalse(itr.hasNext());
 	}
 	
 	private void runsInSlave(WriterContext context) throws HCatException {

Modified: incubator/hcatalog/branches/branch-0.4/src/test/org/apache/hcatalog/mapreduce/HCatMapReduceTest.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/branches/branch-0.4/src/test/org/apache/hcatalog/mapreduce/HCatMapReduceTest.java?rev=1374902&r1=1374901&r2=1374902&view=diff
==============================================================================
--- incubator/hcatalog/branches/branch-0.4/src/test/org/apache/hcatalog/mapreduce/HCatMapReduceTest.java (original)
+++ incubator/hcatalog/branches/branch-0.4/src/test/org/apache/hcatalog/mapreduce/HCatMapReduceTest.java Mon Aug 20 03:08:56 2012
@@ -55,6 +55,7 @@ import org.apache.hadoop.mapreduce.JobSt
 import org.apache.hadoop.mapreduce.Mapper;
 import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;
 import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;
+import org.apache.hcatalog.HcatTestUtils;
 import org.apache.hcatalog.data.DefaultHCatRecord;
 import org.apache.hcatalog.data.HCatRecord;
 import org.apache.hcatalog.data.schema.HCatFieldSchema;
@@ -235,7 +236,7 @@ public abstract class HCatMapReduceTest 
     }
   }
 
-  void runMRCreate(Map<String, String> partitionValues,
+  Job runMRCreate(Map<String, String> partitionValues,
         List<HCatFieldSchema> partitionColumns, List<HCatRecord> records,
         int writeCount, boolean assertWrite) throws Exception {
 
@@ -275,15 +276,20 @@ public abstract class HCatMapReduceTest 
           .findCounter("FILE_BYTES_READ").getValue() > 0);
     }
 
-    if (success) {
-      new FileOutputCommitterContainer(job,null).commitJob(job);
-    } else {
-      new FileOutputCommitterContainer(job,null).abortJob(job, JobStatus.State.FAILED);
+    if (!HcatTestUtils.isHadoop23()) {
+        // Local mode outputcommitter hook is not invoked in Hadoop 1.x
+        if (success) {
+            new FileOutputCommitterContainer(job,null).commitJob(job);
+        } else {
+            new FileOutputCommitterContainer(job,null).abortJob(job, JobStatus.State.FAILED);
+        }
     }
     if (assertWrite){
       // we assert only if we expected to assert with this call.
       Assert.assertEquals(writeCount, MapCreate.writeCount);
     }
+
+    return job;
   }
 
   List<HCatRecord> runMRRead(int readCount) throws Exception {

Modified: incubator/hcatalog/branches/branch-0.4/src/test/org/apache/hcatalog/mapreduce/TestHCatDynamicPartitioned.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/branches/branch-0.4/src/test/org/apache/hcatalog/mapreduce/TestHCatDynamicPartitioned.java?rev=1374902&r1=1374901&r2=1374902&view=diff
==============================================================================
--- incubator/hcatalog/branches/branch-0.4/src/test/org/apache/hcatalog/mapreduce/TestHCatDynamicPartitioned.java (original)
+++ incubator/hcatalog/branches/branch-0.4/src/test/org/apache/hcatalog/mapreduce/TestHCatDynamicPartitioned.java Mon Aug 20 03:08:56 2012
@@ -25,6 +25,8 @@ import java.util.List;
 import org.apache.hadoop.hive.conf.HiveConf;
 import org.apache.hadoop.hive.metastore.api.FieldSchema;
 import org.apache.hadoop.hive.serde.Constants;
+import org.apache.hadoop.mapreduce.Job;
+import org.apache.hcatalog.HcatTestUtils;
 import org.apache.hcatalog.common.ErrorType;
 import org.apache.hcatalog.common.HCatConstants;
 import org.apache.hcatalog.common.HCatException;
@@ -115,7 +117,10 @@ public class TestHCatDynamicPartitioned 
     IOException exc = null;
     try {
       generateWriteRecords(20,5,0);
-      runMRCreate(null, dataColumns, writeRecords, 20,false);
+      Job job = runMRCreate(null, dataColumns, writeRecords, 20,false);
+      if (HcatTestUtils.isHadoop23()) {
+          new FileOutputCommitterContainer(job,null).cleanupJob(job);
+      }
     } catch(IOException e) {
       exc = e;
     }

Modified: incubator/hcatalog/branches/branch-0.4/src/test/org/apache/hcatalog/mapreduce/TestSequenceFileReadWrite.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/branches/branch-0.4/src/test/org/apache/hcatalog/mapreduce/TestSequenceFileReadWrite.java?rev=1374902&r1=1374901&r2=1374902&view=diff
==============================================================================
--- incubator/hcatalog/branches/branch-0.4/src/test/org/apache/hcatalog/mapreduce/TestSequenceFileReadWrite.java (original)
+++ incubator/hcatalog/branches/branch-0.4/src/test/org/apache/hcatalog/mapreduce/TestSequenceFileReadWrite.java Mon Aug 20 03:08:56 2012
@@ -26,6 +26,8 @@ import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Iterator;
 
+import junit.framework.TestCase;
+
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hive.cli.CliSessionState;
 import org.apache.hadoop.hive.conf.HiveConf;
@@ -50,7 +52,7 @@ import org.apache.pig.PigServer;
 import org.apache.pig.data.Tuple;
 import org.junit.Test;
 
-public class TestSequenceFileReadWrite {
+public class TestSequenceFileReadWrite extends TestCase {
   private static final String TEST_DATA_DIR = System.getProperty("user.dir") +
       "/build/test/data/" + TestSequenceFileReadWrite.class.getCanonicalName();
   private static final String TEST_WAREHOUSE_DIR = TEST_DATA_DIR + "/warehouse";
@@ -167,7 +169,9 @@ public class TestSequenceFileReadWrite {
         HCatOutputFormat.setSchema(job, getSchema());
         job.setNumReduceTasks(0);
         assertTrue(job.waitForCompletion(true));
-        new FileOutputCommitterContainer(job, null).commitJob(job);
+        if (!HcatTestUtils.isHadoop23()) {
+            new FileOutputCommitterContainer(job, null).commitJob(job);
+        }
         assertTrue(job.isSuccessful());
 
         server.setBatchOn();
@@ -204,6 +208,7 @@ public class TestSequenceFileReadWrite {
         job.setOutputKeyClass(NullWritable.class);
         job.setOutputValueClass(DefaultHCatRecord.class);
         job.setInputFormatClass(TextInputFormat.class);
+        job.setNumReduceTasks(0);
         TextInputFormat.setInputPaths(job, INPUT_FILE_NAME);
 
         HCatOutputFormat.setOutput(job, OutputJobInfo.create(
@@ -211,7 +216,9 @@ public class TestSequenceFileReadWrite {
         job.setOutputFormatClass(HCatOutputFormat.class);
         HCatOutputFormat.setSchema(job, getSchema());
         assertTrue(job.waitForCompletion(true));
-        new FileOutputCommitterContainer(job, null).commitJob(job);
+        if (!HcatTestUtils.isHadoop23()) {
+            new FileOutputCommitterContainer(job, null).commitJob(job);
+        }
         assertTrue(job.isSuccessful());
 
         server.setBatchOn();

Modified: incubator/hcatalog/branches/branch-0.4/storage-handlers/hbase/ivy.xml
URL: http://svn.apache.org/viewvc/incubator/hcatalog/branches/branch-0.4/storage-handlers/hbase/ivy.xml?rev=1374902&r1=1374901&r2=1374902&view=diff
==============================================================================
--- incubator/hcatalog/branches/branch-0.4/storage-handlers/hbase/ivy.xml (original)
+++ incubator/hcatalog/branches/branch-0.4/storage-handlers/hbase/ivy.xml Mon Aug 20 03:08:56 2012
@@ -48,9 +48,9 @@
         </dependency>
 
         <dependency org="org.apache.hadoop" name="hadoop-core"
-          rev="${hadoop-core.version}" conf="common->master" />
+          rev="${hadoop20.version}" conf="common->master" />
         <dependency org="org.apache.hadoop" name="hadoop-test"
-          rev="${hadoop-test.version}" conf="common->master" />
+          rev="${hadoop20.version}" conf="common->master" />
         <dependency org="org.apache.hive" name="hive-metastore"
           rev="${hive.version}" conf="common->master"/>
         <dependency org="org.apache.hive" name="hive-common"

Modified: incubator/hcatalog/branches/branch-0.4/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/ImportSequenceFile.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/branches/branch-0.4/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/ImportSequenceFile.java?rev=1374902&r1=1374901&r2=1374902&view=diff
==============================================================================
--- incubator/hcatalog/branches/branch-0.4/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/ImportSequenceFile.java (original)
+++ incubator/hcatalog/branches/branch-0.4/storage-handlers/hbase/src/java/org/apache/hcatalog/hbase/ImportSequenceFile.java Mon Aug 20 03:08:56 2012
@@ -237,7 +237,7 @@ class ImportSequenceFile {
             fs.delete(workDir, true);
             //We only cleanup on success because failure might've been caused by existence of target directory
             if(localMode && success)
-                new ImporterOutputFormat().getOutputCommitter(new TaskAttemptContext(conf,new TaskAttemptID())).commitJob(job);
+                new ImporterOutputFormat().getOutputCommitter(org.apache.hadoop.mapred.HCatMapRedUtil.createTaskAttemptContext(conf,new TaskAttemptID())).commitJob(job);
         } catch (InterruptedException e) {
             LOG.error("ImportSequenceFile Failed", e);
         } catch (ClassNotFoundException e) {

Modified: incubator/hcatalog/branches/branch-0.4/webhcat/java-client/ivy.xml
URL: http://svn.apache.org/viewvc/incubator/hcatalog/branches/branch-0.4/webhcat/java-client/ivy.xml?rev=1374902&r1=1374901&r2=1374902&view=diff
==============================================================================
--- incubator/hcatalog/branches/branch-0.4/webhcat/java-client/ivy.xml (original)
+++ incubator/hcatalog/branches/branch-0.4/webhcat/java-client/ivy.xml Mon Aug 20 03:08:56 2012
@@ -37,9 +37,9 @@
     </configurations>
     <dependencies>
         <dependency org="org.apache.hadoop" name="hadoop-core"
-          rev="${hadoop-core.version}" conf="common->master" />
+          rev="${hadoop20.version}" conf="common->master" />
         <dependency org="org.apache.hadoop" name="hadoop-test"
-          rev="${hadoop-test.version}" conf="common->master" />
+          rev="${hadoop20.version}" conf="common->master" />
         <dependency org="org.apache.hive" name="hive-metastore"
           rev="${hive.version}" conf="common->master"/>
         <dependency org="org.apache.hive" name="hive-common"



Mime
View raw message