pig-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From da...@apache.org
Subject svn commit: r1196767 [1/2] - in /pig/branches/branch-0.9: ./ ivy/ shims/src/hadoop20/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/ shims/src/hadoop20/org/apache/pig/backend/hadoop/executionengine/shims/ shims/src/hadoop23/org/apache/pig...
Date Wed, 02 Nov 2011 19:20:19 GMT
Author: daijy
Date: Wed Nov  2 19:20:18 2011
New Revision: 1196767

URL: http://svn.apache.org/viewvc?rev=1196767&view=rev
Log:
PIG-2125: Make Pig work with hadoop .NEXT (PIG-2125-10.patch)

Added:
    pig/branches/branch-0.9/src/META-INF/
    pig/branches/branch-0.9/src/META-INF/services/
    pig/branches/branch-0.9/src/META-INF/services/org.apache.hadoop.mapreduce.protocol.ClientProtocolProvider
Modified:
    pig/branches/branch-0.9/build.xml
    pig/branches/branch-0.9/ivy.xml
    pig/branches/branch-0.9/ivy/ivysettings.xml
    pig/branches/branch-0.9/ivy/libraries.properties
    pig/branches/branch-0.9/shims/src/hadoop20/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigMapBase.java
    pig/branches/branch-0.9/shims/src/hadoop20/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigMapReduce.java
    pig/branches/branch-0.9/shims/src/hadoop20/org/apache/pig/backend/hadoop/executionengine/shims/HadoopShims.java
    pig/branches/branch-0.9/shims/src/hadoop23/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigMapBase.java
    pig/branches/branch-0.9/shims/src/hadoop23/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigMapReduce.java
    pig/branches/branch-0.9/shims/src/hadoop23/org/apache/pig/backend/hadoop/executionengine/shims/HadoopShims.java
    pig/branches/branch-0.9/shims/test/hadoop23/org/apache/pig/test/MiniCluster.java
    pig/branches/branch-0.9/src/org/apache/pig/backend/hadoop/datastorage/ConfigurationUtil.java
    pig/branches/branch-0.9/src/org/apache/pig/backend/hadoop/executionengine/HExecutionEngine.java
    pig/branches/branch-0.9/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/JobControlCompiler.java
    pig/branches/branch-0.9/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigGenericMapBase.java
    pig/branches/branch-0.9/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigGenericMapReduce.java
    pig/branches/branch-0.9/src/org/apache/pig/impl/io/PigFile.java
    pig/branches/branch-0.9/src/org/apache/pig/tools/pigstats/PigStatusReporter.java
    pig/branches/branch-0.9/test/findbugsExcludeFile.xml
    pig/branches/branch-0.9/test/org/apache/pig/test/TestAlgebraicEvalLocal.java
    pig/branches/branch-0.9/test/org/apache/pig/test/TestBZip.java
    pig/branches/branch-0.9/test/org/apache/pig/test/TestCombiner.java
    pig/branches/branch-0.9/test/org/apache/pig/test/TestCommit.java
    pig/branches/branch-0.9/test/org/apache/pig/test/TestNewPlanOperatorPlan.java
    pig/branches/branch-0.9/test/org/apache/pig/test/TestPigRunner.java
    pig/branches/branch-0.9/test/org/apache/pig/test/TestProject.java
    pig/branches/branch-0.9/test/org/apache/pig/test/TestPruneColumn.java
    pig/branches/branch-0.9/test/org/apache/pig/test/TestScalarAliases.java
    pig/branches/branch-0.9/test/org/apache/pig/test/TestScriptUDF.java
    pig/branches/branch-0.9/test/org/apache/pig/test/TestSecondarySort.java
    pig/branches/branch-0.9/test/org/apache/pig/test/TestStreaming.java
    pig/branches/branch-0.9/test/org/apache/pig/test/TestStreamingLocal.java
    pig/branches/branch-0.9/test/org/apache/pig/test/Util.java

Modified: pig/branches/branch-0.9/build.xml
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.9/build.xml?rev=1196767&r1=1196766&r2=1196767&view=diff
==============================================================================
--- pig/branches/branch-0.9/build.xml (original)
+++ pig/branches/branch-0.9/build.xml Wed Nov  2 19:20:18 2011
@@ -67,7 +67,9 @@
 
     <!-- jar names. TODO we might want to use the svn reversion name in the name in case it is a dev version -->
     <property name="output.jarfile" value="${build.dir}/${final.name}.jar" />
+    <property name="output.stage.jarfile" value="${build.dir}/${final.name}.stage.jar" />
     <property name="output.jarfile.withouthadoop" value="${build.dir}/${final.name}-withouthadoop.jar" />
+    <property name="output.stage.jarfile.withouthadoop" value="${build.dir}/${final.name}-withouthadoop.stage.jar" />
     <property name="output.jarfile.core" value="${build.dir}/${final.name}-core.jar" />
     <property name="output.jarfile.sources" value="${build.dir}/${final.name}-sources.jar" />
     <property name="output.jarfile.javadoc" value="${build.dir}/${final.name}-javadoc.jar" />
@@ -157,6 +159,10 @@
            select="\1" /-->
     <property name="hadoopversion" value="20" />
 
+    <condition property="isHadoop23">
+        <equals arg1="hadoopversion" arg2="23"/>
+    </condition>
+
     <property name="src.shims.dir" value="${basedir}/shims/src/hadoop${hadoopversion}" />
     <property name="src.shims.test.dir" value="${basedir}/shims/test/hadoop${hadoopversion}" />
 
@@ -170,7 +176,6 @@
 	<property name="ant_task_repo_url"
 	value="${mvnrepo}/org/apache/maven/maven-ant-tasks/${ant-task.version}/maven-ant-tasks-${ant-task.version}.jar"/>
 	<property name="ivy_repo_url" value="${mvnrepo}/org/apache/ivy/ivy/${ivy.version}/ivy-${ivy.version}.jar"/>
-    <property name="guava.jar" value="guava-${guava.version}.jar"/>
     <property name="ivysettings.xml" location="${ivy.dir}/ivysettings.xml" />
     <property name="ivy.org" value="org.apache.pig"/>
     <property name="build.dir" location="build" />
@@ -224,7 +229,6 @@
     <!-- javadoc-classpath -->
     <path id="javadoc-classpath">
 	<fileset file="${lib.dir}/${automaton.jarfile}" />
-        <fileset file="${lib.dir}/${guava.jarfile}" />
 	<path refid="javadoc.classpath"/>	
     </path> 
 
@@ -513,9 +517,34 @@
         </jar>
     </target>
 
+    <!-- ================================================================== -->
+    <!-- Utility to build pig.jar and pig-withouthadoop.jar                 -->
+    <!-- ================================================================== -->
     <target name="jar-all" depends="jar,jar-withouthadoop" description="Create pig-all.jar and pig-withouthadoop.jar">
     </target>
 
+    <target name="include-meta" if="isHadoop23">
+        <copy todir="${build.classes}/META-INF">
+            <fileset dir="${src.dir}/META-INF" includes="**"/>  
+        </copy>
+        <move file="${output.jarfile}" tofile="${output.stage.jarfile}"/>
+        <sleep seconds="1"/>
+        <jar jarfile="${output.jarfile}" if="isHadoop23">
+            <manifest>
+                <attribute name="Main-Class" value="org.apache.pig.Main" />
+                <section name="org/apache/pig">
+                    <attribute name="Implementation-Vendor" value="Apache" />
+                    <attribute name="Implementation-Title" value="Pig" />
+                    <attribute name="Implementation-Version" value="${version}" />
+                    <attribute name="Build-TimeStamp" value="${timestamp}" />
+                    <attribute name="Svn-Revision" value="${svnString}" />
+                </section>
+            </manifest>
+            <zipfileset src="${output.stage.jarfile}"/>
+            <fileset dir="${build.classes}" includes="META-INF/**" />
+        </jar>
+    </target>
+
     <!-- ================================================================== -->
     <!-- Make pig.jar                                                       -->
     <!-- ================================================================== -->
@@ -565,23 +594,29 @@
                     <attribute name="Svn-Revision" value="${svnString}" />
                 </section>
             </manifest>
-            <zipfileset src="${ivy.lib.dir}/hadoop-core-${hadoop-core.version}.jar" />
             <zipfileset src="${lib.dir}/${automaton.jarfile}" />
-            <zipfileset src="${ivy.lib.dir}/antlr-runtime-${antlr.version}.jar" />
-            <zipfileset src="${ivy.lib.dir}/stringtemplate-${antlr.version}.jar" />
-            <zipfileset src="${ivy.lib.dir}/junit-${junit.version}.jar" />
-            <zipfileset src="${ivy.lib.dir}/jsch-${jsch.version}.jar" />
-            <zipfileset src="${ivy.lib.dir}/jline-${jline.version}.jar" />
-            <zipfileset src="${ivy.lib.dir}/jackson-mapper-asl-${jackson.version}.jar" />
-            <zipfileset src="${ivy.lib.dir}/jackson-core-asl-${jackson.version}.jar" />
-            <zipfileset src="${ivy.lib.dir}/joda-time-${joda-time.version}.jar" /> 
-            <zipfileset src="${ivy.lib.dir}/${guava.jar}" />
-            <zipgroupfileset dir="${ivy.lib.dir}" includes="hbase-${hbase.version}.jar" /> 
-            <zipgroupfileset dir="${ivy.lib.dir}" includes="commons*.jar"/>
-            <zipgroupfileset dir="${ivy.lib.dir}" includes="log4j*.jar"/>
-            <zipgroupfileset dir="${ivy.lib.dir}" includes="jsp-api*.jar"/>
+            <zipgroupfileset dir="${ivy.lib.dir}">
+                <include name="hadoop-core-${hadoop-core.version}.jar" />
+                <include name="hadoop-*-${hadoop-common.version}*.jar" />
+                <include name="antlr-runtime-${antlr.version}.jar" />
+                <include name="stringtemplate-${antlr.version}.jar" />
+                <include name="junit-${junit.version}.jar" />
+                <include name="jsch-${jsch.version}.jar" />
+                <include name="jline-${jline.version}.jar" />
+                <include name="jackson-mapper-asl-${jackson.version}.jar" />
+                <include name="jackson-core-asl-${jackson.version}.jar" />
+                <include name="joda-time-${joda-time.version}.jar" />
+                <include name="guava-${guava.version}.jar" />
+                <include name="avro-${avro.version}.jar"/>
+                <include name="hbase-${hbase.version}.jar" />
+                <include name="commons*.jar"/>
+                <include name="log4j*.jar"/>
+                <include name="jsp-api*.jar"/>
+                <include name="zookeeper*.jar"/>
+            </zipgroupfileset>
             <fileset file="${basedir}/conf/pig-default.properties" />
         </jar>
+        <antcall target="include-meta" inheritRefs="true" inheritall="true"/>
         <copy file="${output.jarfile}" tofile="${output.jarfile.backcompat}"/>
     </target>
 
@@ -650,12 +685,14 @@
                 </section>
             </manifest>
             <zipfileset src="${lib.dir}/${automaton.jarfile}" />
-            <zipfileset src="${ivy.lib.dir}/antlr-runtime-${antlr.version}.jar" />
-            <zipfileset src="${ivy.lib.dir}/stringtemplate-${antlr.version}.jar" />
-            <zipfileset src="${ivy.lib.dir}/jline-${jline.version}.jar" />
-            <zipfileset src="${ivy.lib.dir}/jackson-mapper-asl-${jackson.version}.jar" />
-            <zipfileset src="${ivy.lib.dir}/jackson-core-asl-${jackson.version}.jar" />
-            <zipfileset src="${ivy.lib.dir}/${guava.jar}" />
+            <zipgroupfileset dir="${ivy.lib.dir}">
+                <include name="antlr-runtime-${antlr.version}.jar" />
+                <include name="stringtemplate-${antlr.version}.jar" />
+                <include name="jline-${jline.version}.jar" />
+                <include name="jackson-mapper-asl-${jackson.version}.jar" />
+                <include name="jackson-core-asl-${jackson.version}.jar" />
+                <include name="guava-${guava.version}.jar" />
+            </zipgroupfileset>
             <fileset file="${basedir}/conf/pig-default.properties" />
         </jar>
         <copy file="${output.jarfile.withouthadoop}" tofile="${output.jarfile.backcompat.withouthadoop}"/>

Modified: pig/branches/branch-0.9/ivy.xml
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.9/ivy.xml?rev=1196767&r1=1196766&r2=1196767&view=diff
==============================================================================
--- pig/branches/branch-0.9/ivy.xml (original)
+++ pig/branches/branch-0.9/ivy.xml Wed Nov  2 19:20:18 2011
@@ -31,13 +31,15 @@
     <conf name="default" extends="master,runtime"/>
     <conf name="runtime" extends="compile,test" description="runtime but not the artifact" />
     <!--Private configurations. -->
-    <conf name="compile" visibility="private" description="compile artifacts"/>
+    <conf name="compile" extends="hadoop${hadoopversion}" visibility="private" description="compile artifacts"/>
     <conf name="test" extends="compile" visibility="private"/>
     <conf name="javadoc" visibility="private" extends="compile,test"/>
     <conf name="releaseaudit" visibility="private"/>
     <conf name="jdiff" visibility="private"/>
     <conf name="checkstyle" visibility="private"/>
     <conf name="buildJar" extends="compile,test" visibility="private"/>
+    <conf name="hadoop20" visibility="private"/>
+    <conf name="hadoop23" visibility="private"/>
   </configurations>
   <publications>
     <!--get the artifact from our module name-->
@@ -50,14 +52,56 @@
       conf="checkstyle->master"/> -->
     <dependency org="commons-beanutils" name="commons-beanutils-core" rev="${commons-beanutils.version}"
       conf="checkstyle->master"/>
+    <dependency org="commons-codec" name="commons-codec" rev="${commons-codec.version}"
+      conf="hadoop23->master"/>
     <dependency org="commons-el" name="commons-el" rev="${commons-el.version}"
       conf="compile->master"/>
+    <dependency org="commons-httpclient" name="commons-httpclient" rev="${commons-httpclient.version}"
+      conf="hadoop23->master"/>
+    <dependency org="commons-configuration" name="commons-configuration" rev="${commons-configuration.version}"
+      conf="hadoop23->master"/>
     <dependency org="log4j" name="log4j" rev="${log4j.version}"
       conf="compile->master"/>
     <dependency org="org.apache.hadoop" name="hadoop-core" rev="${hadoop-core.version}"
-      conf="compile->default;test->default"/>
+      conf="hadoop20->default"/>
     <dependency org="org.apache.hadoop" name="hadoop-test" rev="${hadoop-test.version}"
-      conf="compile->default;test->default"/>
+      conf="hadoop20->default"/>
+    <dependency org="org.apache.hadoop" name="hadoop-annotations" 
+      rev="${hadoop-common.version}" conf="hadoop23->master"/>
+    <dependency org="org.apache.hadoop" name="hadoop-auth" 
+      rev="${hadoop-common.version}" conf="hadoop23->master"/>
+    <dependency org="org.apache.hadoop" name="hadoop-common" 
+      rev="${hadoop-common.version}" conf="hadoop23->master">
+      <artifact name="hadoop-common" ext="jar" />
+      <artifact name="hadoop-common" type="tests" ext="jar" m:classifier="tests" />
+    </dependency>
+    <dependency org="org.apache.hadoop" name="hadoop-hdfs"
+      rev="${hadoop-hdfs.version}" conf="hadoop23->master">
+      <artifact name="hadoop-hdfs" ext="jar" />
+      <artifact name="hadoop-hdfs" type="tests" ext="jar" m:classifier="tests" />
+    </dependency>
+    <dependency org="org.apache.hadoop" name="hadoop-mapreduce-client-core" rev="${hadoop-mapreduce.version}"
+      conf="hadoop23->master"/>
+    <dependency org="org.apache.hadoop" name="hadoop-mapreduce-client-jobclient" rev="${hadoop-mapreduce.version}"
+      conf="hadoop23->master">
+        <artifact name="hadoop-mapreduce-client-jobclient" type="jar" m:classifier="tests"/>
+        <exclude org="commons-daemon" module="commons-daemon"/><!--bad POM-->
+        <exclude org="org.apache.commons" module="commons-daemon"/><!--bad POM-->
+    </dependency>
+    <dependency org="org.apache.hadoop" name="hadoop-yarn-server-tests" rev="${hadoop-mapreduce.version}"
+      conf="hadoop23->master">
+      <artifact name="hadoop-yarn-server-tests" type="jar" m:classifier="tests"/>
+    </dependency>
+    <dependency org="org.apache.hadoop" name="hadoop-mapreduce-client-common"
+      rev="${hadoop-mapreduce.version}" conf="hadoop23->master"/>
+    <dependency org="org.apache.hadoop" name="hadoop-yarn-api" 
+      rev="${hadoop-common.version}" conf="hadoop23->master"/>
+    <dependency org="org.apache.hadoop" name="hadoop-yarn-common" 
+      rev="${hadoop-common.version}" conf="hadoop23->master"/>
+    <dependency org="org.apache.hadoop" name="hadoop-yarn-server-nodemanager" 
+      rev="${hadoop-common.version}" conf="hadoop23->master"/>
+    <dependency org="org.apache.hadoop" name="hadoop-yarn-server-resourcemanager" 
+      rev="${hadoop-common.version}" conf="hadoop23->master"/>
     <dependency org="org.mortbay.jetty" name="jetty" rev="${jetty.version}"
       conf="compile->master"/>
     <dependency org="org.mortbay.jetty" name="jetty-util" rev="${jetty-util.version}"
@@ -67,7 +111,7 @@
      <dependency org="org.slf4j" name="slf4j-log4j12" rev="${slf4j-log4j12.version}"
       conf="compile->master;test->master"/>
     <dependency org="commons-cli" name="commons-cli" rev="${commons-cli.version}"
-      conf="checkstyle->master"/>
+      conf="compile->master;checkstyle->master"/>
     
     <dependency org="org.apache.avro" name="avro" rev="${avro.version}"
       conf="compile->master;checkstyle->master"/>
@@ -93,13 +137,13 @@
     <dependency org="org.codehaus.jackson" name="jackson-core-asl" rev="${jackson.version}"
       conf="compile->master"/>
     <dependency org="joda-time" name="joda-time" rev="${joda-time.version}" conf="compile->master"/>
-  	<dependency org="commons-lang" name="commons-lang" rev="${commons-lang.version}"
+    <dependency org="commons-lang" name="commons-lang" rev="${commons-lang.version}"
   	  conf="compile->master"/>
     <dependency org="com.google.guava" name="guava" rev="${guava.version}" conf="compile->master" />
-  	<dependency org="org.python" name="jython" rev="${jython.version}" conf="compile->master"/>
+    <dependency org="org.python" name="jython" rev="${jython.version}" conf="compile->master"/>
     <dependency org="rhino" name="js" rev="${rhino.version}" conf="compile->master"/>
-  	<dependency org="org.antlr" name="antlr" rev="${antlr.version}" conf="compile->master"/>
-  	<dependency org="org.antlr" name="antlr-runtime" rev="${antlr.version}" conf="compile->default"/>
+    <dependency org="org.antlr" name="antlr" rev="${antlr.version}" conf="compile->master"/>
+    <dependency org="org.antlr" name="antlr-runtime" rev="${antlr.version}" conf="compile->default"/>
     <dependency org="org.apache.zookeeper" name="zookeeper" rev="${zookeeper.version}" conf="compile->master"/>
     <dependency org="org.jboss.netty" name="netty" rev="3.2.2.Final" conf="compile->master"/>
 

Modified: pig/branches/branch-0.9/ivy/ivysettings.xml
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.9/ivy/ivysettings.xml?rev=1196767&r1=1196766&r2=1196767&view=diff
==============================================================================
--- pig/branches/branch-0.9/ivy/ivysettings.xml (original)
+++ pig/branches/branch-0.9/ivy/ivysettings.xml Wed Nov  2 19:20:18 2011
@@ -31,22 +31,32 @@
   -->
   <property name="repo.maven.org" value="${mvnrepo}" override="true"/>
   <property name="repo.jboss.org" value="http://repository.jboss.com/nexus/content/groups/public/" override="false"/>
-
+  <property name="repo.apache.snapshots" value="http://repository.apache.org/content/groups/snapshots-group/" override="false"/>
+  <property name="repo.dir" value="${user.home}/.m2/repository"/>
   <property name="maven2.pattern"  value="[organisation]/[module]/[revision]/[module]-[revision](-[classifier])"/> 
   <property name="maven2.pattern.ext" value="${maven2.pattern}.[ext]"/>
+  <property name="snapshot.pattern"  value="[organisation]/[module]/[revision]/[artifact]-[revision](-[classifier]).[ext]"/>
+  <property name="resolvers" value="default" override="false"/>
+  <property name="force-resolve" value="false" override="false"/>
   <!-- pull in the local repository -->
   <include url="${ivy.default.conf.dir}/ivyconf-local.xml"/>
-  <settings defaultResolver="default"/>
+  <settings defaultResolver="${resolvers}"/>
   <resolvers>
     <ibiblio name="maven2" root="${repo.maven.org}" pattern="${maven2.pattern.ext}" m2compatible="true"/>
     <ibiblio name="jboss-maven2" root="${repo.jboss.org}" pattern="${maven2.pattern.ext}" m2compatible="true"/>
+    <ibiblio name="apache-snapshots" root="${repo.apache.snapshots}" pattern="${snapshot.pattern}" checkmodified="true" m2compatible="true"/>
+    <filesystem name="fs" m2compatible="true" checkconsistency="false" force="${force-resolve}">
+       <artifact pattern="${repo.dir}/${maven2.pattern.ext}"/>
+       <ivy pattern="${repo.dir}/[organisation]/[module]/[revision]/[module]-[revision].pom"/>
+    </filesystem>
     <chain name="default" dual="true">
       <resolver ref="local"/>
       <resolver ref="maven2"/>
       <resolver ref="jboss-maven2"/>
+      <resolver ref="apache-snapshots"/>
     </chain>
     <chain name="internal">
-      <resolver ref="local"/>
+      <resolver ref="fs"/>
     </chain>
     <chain name="external">
       <resolver ref="maven2"/>

Modified: pig/branches/branch-0.9/ivy/libraries.properties
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.9/ivy/libraries.properties?rev=1196767&r1=1196766&r2=1196767&view=diff
==============================================================================
--- pig/branches/branch-0.9/ivy/libraries.properties (original)
+++ pig/branches/branch-0.9/ivy/libraries.properties Wed Nov  2 19:20:18 2011
@@ -21,14 +21,20 @@ pig.version=0.9.1
 avro.version=1.4.1
 commons-beanutils.version=1.7.0
 commons-cli.version=1.0
+commons-codec.version=1.4
 commons-el.version=1.0
 commons-logging.version=1.1.1
 commons-lang.version=2.4
+commons-configuration.version=1.6
+commons-httpclient.version=3.1
 checkstyle.version=4.2
 ivy.version=2.2.0
 guava.version=r06
 hadoop-core.version=0.20.2
 hadoop-test.version=0.20.2
+hadoop-common.version=0.23.0-SNAPSHOT
+hadoop-hdfs.version=0.23.0-SNAPSHOT
+hadoop-mapreduce.version=0.23.0-SNAPSHOT
 hbase.version=0.90.0
 hsqldb.version=1.8.0.10
 jackson.version=1.6.0
@@ -45,7 +51,7 @@ junit.version=4.5
 jython.version=2.5.0
 rhino.version=1.7R2
 antlr.version=3.2
-log4j.version=1.2.14
+log4j.version=1.2.16
 netty.version=3.2.2
 rats-lib.version=0.5.1
 slf4j-api.version=1.4.3

Modified: pig/branches/branch-0.9/shims/src/hadoop20/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigMapBase.java
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.9/shims/src/hadoop20/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigMapBase.java?rev=1196767&r1=1196766&r2=1196767&view=diff
==============================================================================
--- pig/branches/branch-0.9/shims/src/hadoop20/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigMapBase.java (original)
+++ pig/branches/branch-0.9/shims/src/hadoop20/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigMapBase.java Wed Nov  2 19:20:18 2011
@@ -26,6 +26,7 @@ import org.apache.hadoop.io.Text;
 import org.apache.hadoop.io.Writable;
 import org.apache.hadoop.mapreduce.InputSplit;
 import org.apache.hadoop.mapreduce.TaskAttemptID;
+import org.apache.hadoop.mapreduce.Mapper.Context;
 import org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.PigGenericMapBase;
 import org.apache.pig.data.DataBag;
 import org.apache.pig.data.Tuple;
@@ -52,6 +53,11 @@ abstract public class PigMapBase extends
         return new IllustratorContext(conf, input, output, split);
     }
     
+    @Override
+    public boolean inIllustrator(Context context) {
+        return (context instanceof PigMapBase.IllustratorContext);
+    }
+    
     public class IllustratorContext extends Context {
         private DataBag input;
         List<Pair<PigNullableWritable, Writable>> output;

Modified: pig/branches/branch-0.9/shims/src/hadoop20/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigMapReduce.java
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.9/shims/src/hadoop20/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigMapReduce.java?rev=1196767&r1=1196766&r2=1196767&view=diff
==============================================================================
--- pig/branches/branch-0.9/shims/src/hadoop20/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigMapReduce.java (original)
+++ pig/branches/branch-0.9/shims/src/hadoop20/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigMapReduce.java Wed Nov  2 19:20:18 2011
@@ -160,5 +160,17 @@ public class PigMapReduce extends PigGen
             public void progress() { 
             }
         }
+
+        @Override
+        public boolean inIllustrator(
+                org.apache.hadoop.mapreduce.Reducer.Context context) {
+            return (context instanceof PigMapReduce.Reduce.IllustratorContext);
+        }
+
+        @Override
+        public POPackage getPack(
+                org.apache.hadoop.mapreduce.Reducer.Context context) {
+            return ((PigMapReduce.Reduce.IllustratorContext) context).pack;
+        }
     }
 }

Modified: pig/branches/branch-0.9/shims/src/hadoop20/org/apache/pig/backend/hadoop/executionengine/shims/HadoopShims.java
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.9/shims/src/hadoop20/org/apache/pig/backend/hadoop/executionengine/shims/HadoopShims.java?rev=1196767&r1=1196766&r2=1196767&view=diff
==============================================================================
--- pig/branches/branch-0.9/shims/src/hadoop20/org/apache/pig/backend/hadoop/executionengine/shims/HadoopShims.java (original)
+++ pig/branches/branch-0.9/shims/src/hadoop20/org/apache/pig/backend/hadoop/executionengine/shims/HadoopShims.java Wed Nov  2 19:20:18 2011
@@ -58,4 +58,8 @@ public class HadoopShims {
     static public boolean isMap(TaskAttemptID taskAttemptID) {
         return taskAttemptID.isMap();
     }
+    
+    static public TaskAttemptID getNewTaskAttemptID() {
+        return new TaskAttemptID();
+    }
 }

Modified: pig/branches/branch-0.9/shims/src/hadoop23/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigMapBase.java
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.9/shims/src/hadoop23/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigMapBase.java?rev=1196767&r1=1196766&r2=1196767&view=diff
==============================================================================
--- pig/branches/branch-0.9/shims/src/hadoop23/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigMapBase.java (original)
+++ pig/branches/branch-0.9/shims/src/hadoop23/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigMapBase.java Wed Nov  2 19:20:18 2011
@@ -19,6 +19,7 @@ package org.apache.pig.backend.hadoop.ex
 
 
 import java.io.IOException;
+
 import java.net.URI;
 import java.util.Iterator;
 import java.util.List;
@@ -37,14 +38,21 @@ import org.apache.hadoop.mapreduce.Mappe
 import org.apache.hadoop.mapreduce.OutputCommitter;
 import org.apache.hadoop.mapreduce.OutputFormat;
 import org.apache.hadoop.mapreduce.Partitioner;
+import org.apache.hadoop.mapreduce.RecordReader;
+import org.apache.hadoop.mapreduce.RecordWriter;
 import org.apache.hadoop.mapreduce.Reducer;
+import org.apache.hadoop.mapreduce.StatusReporter;
 import org.apache.hadoop.mapreduce.TaskAttemptID;
+import org.apache.hadoop.mapreduce.Reducer.Context;
+import org.apache.hadoop.mapreduce.lib.map.WrappedMapper;
+import org.apache.hadoop.mapreduce.task.MapContextImpl;
 import org.apache.hadoop.security.Credentials;
 import org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.PigGenericMapBase;
 import org.apache.pig.data.DataBag;
 import org.apache.pig.data.Tuple;
 import org.apache.pig.impl.io.PigNullableWritable;
 import org.apache.pig.impl.util.Pair;
+import org.apache.hadoop.mapreduce.lib.map.WrappedMapper;
 
 abstract public class PigMapBase extends PigGenericMapBase {
     /**
@@ -63,10 +71,11 @@ abstract public class PigMapBase extends
     public Context getIllustratorContext(Configuration conf, DataBag input,
           List<Pair<PigNullableWritable, Writable>> output, InputSplit split)
           throws IOException, InterruptedException {
-        return new IllustratorContext(conf, input, output, split);
+    	org.apache.hadoop.mapreduce.Mapper.Context mapperContext = new WrappedMapper<Text, Tuple, PigNullableWritable, Writable>().getMapContext(new IllustratorContext(conf, input, output, split));
+        return mapperContext;
     }
     
-    public class IllustratorContext extends Context {
+    public class IllustratorContext extends MapContextImpl<Text, Tuple, PigNullableWritable, Writable> {
         private DataBag input;
         List<Pair<PigNullableWritable, Writable>> output;
         private Iterator<Tuple> it = null;
@@ -76,11 +85,12 @@ abstract public class PigMapBase extends
         public IllustratorContext(Configuration conf, DataBag input,
               List<Pair<PigNullableWritable, Writable>> output,
               InputSplit split) throws IOException, InterruptedException {
-              if (output == null)
-                  throw new IOException("Null output can not be used");
-              this.input = input; this.output = output;
+            super(conf, new TaskAttemptID(), null, null, null, null, split);
+            if (output == null)
+                throw new IOException("Null output can not be used");
+            this.input = input; this.output = output;
         }
-        
+
         @Override
         public boolean nextKeyValue() throws IOException, InterruptedException {
             if (input == null) {
@@ -118,263 +128,10 @@ abstract public class PigMapBase extends
         public void progress() {
           
         }
-
-        @Override
-        public InputSplit getInputSplit() {
-            // TODO Auto-generated method stub
-            return null;
-        }
-
-        @Override
-        public Counter getCounter(Enum<?> arg0) {
-            // TODO Auto-generated method stub
-            return null;
-        }
-
-        @Override
-        public Counter getCounter(String arg0, String arg1) {
-            // TODO Auto-generated method stub
-            return null;
-        }
-
-        @Override
-        public OutputCommitter getOutputCommitter() {
-            // TODO Auto-generated method stub
-            return null;
-        }
-
-        @Override
-        public String getStatus() {
-            // TODO Auto-generated method stub
-            return null;
-        }
-
-        @Override
-        public TaskAttemptID getTaskAttemptID() {
-            // TODO Auto-generated method stub
-            return null;
-        }
-
-        @Override
-        public void setStatus(String arg0) {
-            // TODO Auto-generated method stub
-            
-        }
-
-        @Override
-        public Path[] getArchiveClassPaths() {
-            // TODO Auto-generated method stub
-            return null;
-        }
-
-        @Override
-        public long[] getArchiveTimestamps() {
-            // TODO Auto-generated method stub
-            return null;
-        }
-
-        @Override
-        public URI[] getCacheArchives() throws IOException {
-            // TODO Auto-generated method stub
-            return null;
-        }
-
-        @Override
-        public URI[] getCacheFiles() throws IOException {
-            // TODO Auto-generated method stub
-            return null;
-        }
-
-        @Override
-        public Class<? extends Reducer<?, ?, ?, ?>> getCombinerClass()
-                throws ClassNotFoundException {
-            // TODO Auto-generated method stub
-            return null;
-        }
-
-        @Override
-        public Configuration getConfiguration() {
-            // TODO Auto-generated method stub
-            return null;
-        }
-
-        @Override
-        public Credentials getCredentials() {
-            // TODO Auto-generated method stub
-            return null;
-        }
-
-        @Override
-        public Path[] getFileClassPaths() {
-            // TODO Auto-generated method stub
-            return null;
-        }
-
-        @Override
-        public long[] getFileTimestamps() {
-            // TODO Auto-generated method stub
-            return null;
-        }
-
-        @Override
-        public RawComparator<?> getGroupingComparator() {
-            // TODO Auto-generated method stub
-            return null;
-        }
-
-        @Override
-        public Class<? extends InputFormat<?, ?>> getInputFormatClass()
-                throws ClassNotFoundException {
-            // TODO Auto-generated method stub
-            return null;
-        }
-
-        @Override
-        public String getJar() {
-            // TODO Auto-generated method stub
-            return null;
-        }
-
-        @Override
-        public JobID getJobID() {
-            // TODO Auto-generated method stub
-            return null;
-        }
-
-        @Override
-        public String getJobName() {
-            // TODO Auto-generated method stub
-            return null;
-        }
-
-        @Override
-        public boolean getJobSetupCleanupNeeded() {
-            // TODO Auto-generated method stub
-            return false;
-        }
-
-        @Override
-        public Path[] getLocalCacheArchives() throws IOException {
-            // TODO Auto-generated method stub
-            return null;
-        }
-
-        @Override
-        public Path[] getLocalCacheFiles() throws IOException {
-            // TODO Auto-generated method stub
-            return null;
-        }
-
-        @Override
-        public Class<?> getMapOutputKeyClass() {
-            // TODO Auto-generated method stub
-            return null;
-        }
-
-        @Override
-        public Class<?> getMapOutputValueClass() {
-            // TODO Auto-generated method stub
-            return null;
-        }
-
-        @Override
-        public Class<? extends Mapper<?, ?, ?, ?>> getMapperClass()
-                throws ClassNotFoundException {
-            // TODO Auto-generated method stub
-            return null;
-        }
-
-        @Override
-        public int getMaxMapAttempts() {
-            // TODO Auto-generated method stub
-            return 0;
-        }
-
-        @Override
-        public int getMaxReduceAttempts() {
-            // TODO Auto-generated method stub
-            return 0;
-        }
-
-        @Override
-        public int getNumReduceTasks() {
-            // TODO Auto-generated method stub
-            return 0;
-        }
-
-        @Override
-        public Class<? extends OutputFormat<?, ?>> getOutputFormatClass()
-                throws ClassNotFoundException {
-            // TODO Auto-generated method stub
-            return null;
-        }
-
-        @Override
-        public Class<?> getOutputKeyClass() {
-            // TODO Auto-generated method stub
-            return null;
-        }
-
-        @Override
-        public Class<?> getOutputValueClass() {
-            // TODO Auto-generated method stub
-            return null;
-        }
-
-        @Override
-        public Class<? extends Partitioner<?, ?>> getPartitionerClass()
-                throws ClassNotFoundException {
-            // TODO Auto-generated method stub
-            return null;
-        }
-
-        @Override
-        public boolean getProfileEnabled() {
-            // TODO Auto-generated method stub
-            return false;
-        }
-
-        @Override
-        public String getProfileParams() {
-            // TODO Auto-generated method stub
-            return null;
-        }
-
-        @Override
-        public IntegerRanges getProfileTaskRange(boolean arg0) {
-            // TODO Auto-generated method stub
-            return null;
-        }
-
-        @Override
-        public Class<? extends Reducer<?, ?, ?, ?>> getReducerClass()
-                throws ClassNotFoundException {
-            // TODO Auto-generated method stub
-            return null;
-        }
-
-        @Override
-        public RawComparator<?> getSortComparator() {
-            // TODO Auto-generated method stub
-            return null;
-        }
-
-        @Override
-        public boolean getSymlink() {
-            // TODO Auto-generated method stub
-            return false;
-        }
-
-        @Override
-        public String getUser() {
-            // TODO Auto-generated method stub
-            return null;
-        }
-
-        @Override
-        public Path getWorkingDirectory() throws IOException {
-            // TODO Auto-generated method stub
-            return null;
-        }
+    }
+    
+    @Override
+    public boolean inIllustrator(Context context) {
+        return (context instanceof WrappedMapper.Context);
     }
 }

Modified: pig/branches/branch-0.9/shims/src/hadoop23/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigMapReduce.java
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.9/shims/src/hadoop23/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigMapReduce.java?rev=1196767&r1=1196766&r2=1196767&view=diff
==============================================================================
--- pig/branches/branch-0.9/shims/src/hadoop23/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigMapReduce.java (original)
+++ pig/branches/branch-0.9/shims/src/hadoop23/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigMapReduce.java Wed Nov  2 19:20:18 2011
@@ -27,10 +27,12 @@ import java.util.Comparator;
 import java.util.Iterator;
 import java.util.List;
 
+import org.apache.hadoop.classification.InterfaceStability;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.conf.Configuration.IntegerRanges;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.io.RawComparator;
+import org.apache.hadoop.io.Text;
 import org.apache.hadoop.io.Writable;
 import org.apache.hadoop.mapred.jobcontrol.Job;
 import org.apache.hadoop.mapreduce.Counter;
@@ -40,17 +42,41 @@ import org.apache.hadoop.mapreduce.Mappe
 import org.apache.hadoop.mapreduce.OutputCommitter;
 import org.apache.hadoop.mapreduce.OutputFormat;
 import org.apache.hadoop.mapreduce.Partitioner;
+import org.apache.hadoop.mapreduce.ReduceContext;
 import org.apache.hadoop.mapreduce.Reducer;
 import org.apache.hadoop.mapreduce.TaskAttemptID;
 import org.apache.hadoop.mapreduce.Reducer.Context;
+import org.apache.hadoop.mapreduce.lib.reduce.WrappedReducer;
+import org.apache.hadoop.mapreduce.task.ReduceContextImpl;
 import org.apache.hadoop.security.Credentials;
+import org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.PigMapBase.IllustratorContext;
 import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POPackage;
+import org.apache.pig.data.Tuple;
 import org.apache.pig.impl.io.NullableTuple;
 import org.apache.pig.impl.io.PigNullableWritable;
 import org.apache.pig.impl.util.Pair;
 import org.apache.pig.pen.FakeRawKeyValueIterator;
 
 public class PigMapReduce extends PigGenericMapReduce {
+    
+    static class IllustrateReducerContext extends WrappedReducer<PigNullableWritable, NullableTuple, PigNullableWritable, Writable> {
+        public IllustratorContext 
+        getReducerContext(ReduceContext<PigNullableWritable, NullableTuple, PigNullableWritable, Writable> reduceContext) {
+            return new IllustratorContext(reduceContext);
+        }
+        
+        public class IllustratorContext 
+            extends WrappedReducer.Context {
+            public IllustratorContext(
+                    ReduceContext<PigNullableWritable, NullableTuple, PigNullableWritable, Writable> reduceContext) {
+                super(reduceContext);
+            }
+            public POPackage getPack() {
+                return ((Reduce.IllustratorContextImpl)reduceContext).pack;
+            }
+        }
+    }
+		  
     public static class Reduce extends PigGenericMapReduce.Reduce {
         /**
          * Get reducer's illustrator context
@@ -64,11 +90,13 @@ public class PigMapReduce extends PigGen
         @Override
         public Context getIllustratorContext(Job job,
                List<Pair<PigNullableWritable, Writable>> input, POPackage pkg) throws IOException, InterruptedException {
-            return new IllustratorContext(job, input, pkg);
+        	org.apache.hadoop.mapreduce.Reducer.Context reducerContext = new IllustrateReducerContext()
+        			.getReducerContext(new IllustratorContextImpl(job, input, pkg));
+        	return reducerContext;
         }
         
         @SuppressWarnings("unchecked")
-        public class IllustratorContext extends Context {
+        public class IllustratorContextImpl extends ReduceContextImpl<PigNullableWritable, NullableTuple, PigNullableWritable, Writable> {
             private PigNullableWritable currentKey = null, nextKey = null;
             private NullableTuple nextValue = null;
             private List<NullableTuple> currentValues = null;
@@ -76,12 +104,15 @@ public class PigMapReduce extends PigGen
             private final ByteArrayOutputStream bos;
             private final DataOutputStream dos;
             private final RawComparator sortComparator, groupingComparator;
-            POPackage pack = null;
+            public POPackage pack = null;
+            private IllustratorValueIterable iterable = new IllustratorValueIterable();
 
-            public IllustratorContext(Job job,
+            public IllustratorContextImpl(Job job,
                   List<Pair<PigNullableWritable, Writable>> input,
                   POPackage pkg
                   ) throws IOException, InterruptedException {
+                super(job.getJobConf(), new TaskAttemptID(), new FakeRawKeyValueIterator(input.iterator().hasNext()),
+                    null, null, null, null, null, null, PigNullableWritable.class, NullableTuple.class);
                 bos = new ByteArrayOutputStream();
                 dos = new DataOutputStream(bos);
                 org.apache.hadoop.mapreduce.Job nwJob = new org.apache.hadoop.mapreduce.Job(job.getJobConf());
@@ -116,6 +147,60 @@ public class PigMapReduce extends PigGen
                 pack = pkg;
             }
             
+            public class IllustratorValueIterator implements ReduceContext.ValueIterator<NullableTuple> {
+                
+                private int pos = -1;
+                private int mark = -1;
+
+                @Override
+                public void mark() throws IOException {
+                    mark=pos-1;
+                    if (mark<-1)
+                        mark=-1;
+                }
+
+                @Override
+                public void reset() throws IOException {
+                    pos=mark;
+                }
+
+                @Override
+                public void clearMark() throws IOException {
+                    mark=-1;
+                }
+
+                @Override
+                public boolean hasNext() {
+                    return pos<currentValues.size()-1;
+                }
+
+                @Override
+                public NullableTuple next() {
+                    pos++;
+                    return currentValues.get(pos);
+                }
+
+                @Override
+                public void remove() {
+                    throw new UnsupportedOperationException("remove not implemented");
+                }
+
+                @Override
+                public void resetBackupStore() throws IOException {
+                    pos=-1;
+                    mark=-1;
+                }
+                
+            }
+            
+            protected class IllustratorValueIterable implements Iterable<NullableTuple> {
+                private IllustratorValueIterator iterator = new IllustratorValueIterator();
+                @Override
+                public Iterator<NullableTuple> iterator() {
+                    return iterator;
+                } 
+            }
+            
             @Override
             public PigNullableWritable getCurrentKey() {
                 return currentKey;
@@ -160,7 +245,7 @@ public class PigMapReduce extends PigGen
             
             @Override
             public Iterable<NullableTuple> getValues() {
-                return currentValues;
+                return iterable;
             }
             
             @Override
@@ -170,272 +255,16 @@ public class PigMapReduce extends PigGen
             @Override
             public void progress() { 
             }
+        }
 
-            @Override
-            public Counter getCounter(Enum<?> arg0) {
-                // TODO Auto-generated method stub
-                return null;
-            }
-
-            @Override
-            public Counter getCounter(String arg0, String arg1) {
-                // TODO Auto-generated method stub
-                return null;
-            }
-
-            @Override
-            public NullableTuple getCurrentValue() throws IOException,
-                    InterruptedException {
-                // TODO Auto-generated method stub
-                return null;
-            }
-
-            @Override
-            public OutputCommitter getOutputCommitter() {
-                // TODO Auto-generated method stub
-                return null;
-            }
-
-            @Override
-            public boolean nextKeyValue() throws IOException,
-                    InterruptedException {
-                // TODO Auto-generated method stub
-                return false;
-            }
-
-            @Override
-            public String getStatus() {
-                // TODO Auto-generated method stub
-                return null;
-            }
-
-            @Override
-            public TaskAttemptID getTaskAttemptID() {
-                // TODO Auto-generated method stub
-                return null;
-            }
-
-            @Override
-            public void setStatus(String arg0) {
-                // TODO Auto-generated method stub
-                
-            }
-
-            @Override
-            public Path[] getArchiveClassPaths() {
-                // TODO Auto-generated method stub
-                return null;
-            }
-
-            @Override
-            public long[] getArchiveTimestamps() {
-                // TODO Auto-generated method stub
-                return null;
-            }
-
-            @Override
-            public URI[] getCacheArchives() throws IOException {
-                // TODO Auto-generated method stub
-                return null;
-            }
-
-            @Override
-            public URI[] getCacheFiles() throws IOException {
-                // TODO Auto-generated method stub
-                return null;
-            }
-
-            @Override
-            public Class<? extends Reducer<?, ?, ?, ?>> getCombinerClass()
-                    throws ClassNotFoundException {
-                // TODO Auto-generated method stub
-                return null;
-            }
-
-            @Override
-            public Configuration getConfiguration() {
-                // TODO Auto-generated method stub
-                return null;
-            }
-
-            @Override
-            public Credentials getCredentials() {
-                // TODO Auto-generated method stub
-                return null;
-            }
-
-            @Override
-            public Path[] getFileClassPaths() {
-                // TODO Auto-generated method stub
-                return null;
-            }
-
-            @Override
-            public long[] getFileTimestamps() {
-                // TODO Auto-generated method stub
-                return null;
-            }
-
-            @Override
-            public RawComparator<?> getGroupingComparator() {
-                // TODO Auto-generated method stub
-                return null;
-            }
-
-            @Override
-            public Class<? extends InputFormat<?, ?>> getInputFormatClass()
-                    throws ClassNotFoundException {
-                // TODO Auto-generated method stub
-                return null;
-            }
-
-            @Override
-            public String getJar() {
-                // TODO Auto-generated method stub
-                return null;
-            }
-
-            @Override
-            public JobID getJobID() {
-                // TODO Auto-generated method stub
-                return null;
-            }
-
-            @Override
-            public String getJobName() {
-                // TODO Auto-generated method stub
-                return null;
-            }
-
-            @Override
-            public boolean getJobSetupCleanupNeeded() {
-                // TODO Auto-generated method stub
-                return false;
-            }
-
-            @Override
-            public Path[] getLocalCacheArchives() throws IOException {
-                // TODO Auto-generated method stub
-                return null;
-            }
-
-            @Override
-            public Path[] getLocalCacheFiles() throws IOException {
-                // TODO Auto-generated method stub
-                return null;
-            }
-
-            @Override
-            public Class<?> getMapOutputKeyClass() {
-                // TODO Auto-generated method stub
-                return null;
-            }
-
-            @Override
-            public Class<?> getMapOutputValueClass() {
-                // TODO Auto-generated method stub
-                return null;
-            }
-
-            @Override
-            public Class<? extends Mapper<?, ?, ?, ?>> getMapperClass()
-                    throws ClassNotFoundException {
-                // TODO Auto-generated method stub
-                return null;
-            }
-
-            @Override
-            public int getMaxMapAttempts() {
-                // TODO Auto-generated method stub
-                return 0;
-            }
-
-            @Override
-            public int getMaxReduceAttempts() {
-                // TODO Auto-generated method stub
-                return 0;
-            }
-
-            @Override
-            public int getNumReduceTasks() {
-                // TODO Auto-generated method stub
-                return 0;
-            }
-
-            @Override
-            public Class<? extends OutputFormat<?, ?>> getOutputFormatClass()
-                    throws ClassNotFoundException {
-                // TODO Auto-generated method stub
-                return null;
-            }
-
-            @Override
-            public Class<?> getOutputKeyClass() {
-                // TODO Auto-generated method stub
-                return null;
-            }
-
-            @Override
-            public Class<?> getOutputValueClass() {
-                // TODO Auto-generated method stub
-                return null;
-            }
-
-            @Override
-            public Class<? extends Partitioner<?, ?>> getPartitionerClass()
-                    throws ClassNotFoundException {
-                // TODO Auto-generated method stub
-                return null;
-            }
-
-            @Override
-            public boolean getProfileEnabled() {
-                // TODO Auto-generated method stub
-                return false;
-            }
-
-            @Override
-            public String getProfileParams() {
-                // TODO Auto-generated method stub
-                return null;
-            }
-
-            @Override
-            public IntegerRanges getProfileTaskRange(boolean arg0) {
-                // TODO Auto-generated method stub
-                return null;
-            }
-
-            @Override
-            public Class<? extends Reducer<?, ?, ?, ?>> getReducerClass()
-                    throws ClassNotFoundException {
-                // TODO Auto-generated method stub
-                return null;
-            }
-
-            @Override
-            public RawComparator<?> getSortComparator() {
-                // TODO Auto-generated method stub
-                return null;
-            }
-
-            @Override
-            public boolean getSymlink() {
-                // TODO Auto-generated method stub
-                return false;
-            }
-
-            @Override
-            public String getUser() {
-                // TODO Auto-generated method stub
-                return null;
-            }
+        @Override
+        public boolean inIllustrator(org.apache.hadoop.mapreduce.Reducer.Context context) {
+            return (context instanceof PigMapReduce.IllustrateReducerContext.IllustratorContext);
+        }
 
-            @Override
-            public Path getWorkingDirectory() throws IOException {
-                // TODO Auto-generated method stub
-                return null;
-            }
+        @Override
+        public POPackage getPack(org.apache.hadoop.mapreduce.Reducer.Context context) {
+            return ((PigMapReduce.IllustrateReducerContext.IllustratorContext) context).getPack();
         }
     }
 }

Modified: pig/branches/branch-0.9/shims/src/hadoop23/org/apache/pig/backend/hadoop/executionengine/shims/HadoopShims.java
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.9/shims/src/hadoop23/org/apache/pig/backend/hadoop/executionengine/shims/HadoopShims.java?rev=1196767&r1=1196766&r2=1196767&view=diff
==============================================================================
--- pig/branches/branch-0.9/shims/src/hadoop23/org/apache/pig/backend/hadoop/executionengine/shims/HadoopShims.java (original)
+++ pig/branches/branch-0.9/shims/src/hadoop23/org/apache/pig/backend/hadoop/executionengine/shims/HadoopShims.java Wed Nov  2 19:20:18 2011
@@ -56,4 +56,10 @@ public class HadoopShims {
         
         return false;
     }
+    
+    static public TaskAttemptID getNewTaskAttemptID() {
+        TaskAttemptID taskAttemptID = new TaskAttemptID("", 1, TaskType.MAP, 
+                1, 1);
+        return taskAttemptID;
+    }
 }

Modified: pig/branches/branch-0.9/shims/test/hadoop23/org/apache/pig/test/MiniCluster.java
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.9/shims/test/hadoop23/org/apache/pig/test/MiniCluster.java?rev=1196767&r1=1196766&r2=1196767&view=diff
==============================================================================
--- pig/branches/branch-0.9/shims/test/hadoop23/org/apache/pig/test/MiniCluster.java (original)
+++ pig/branches/branch-0.9/shims/test/hadoop23/org/apache/pig/test/MiniCluster.java Wed Nov  2 19:20:18 2011
@@ -92,15 +92,18 @@ public class MiniCluster extends MiniGen
 				e.printStackTrace();
 			}
 			*/
-			m_dfs.getFileSystem().copyFromLocalFile(new Path("file:///hadoop-mapreduce-client-app-1.0-SNAPSHOT.jar"), new Path("/hadoop-mapreduce-client-app-1.0-SNAPSHOT.jar"));
-			m_dfs.getFileSystem().copyFromLocalFile(new Path("file:///hadoop-mapreduce-client-jobclient-1.0-SNAPSHOT.jar"), new Path("/hadoop-mapreduce-client-jobclient-1.0-SNAPSHOT.jar"));
+			m_dfs.getFileSystem().copyFromLocalFile(new Path("file:///hadoop-mapreduce-client-app-0.23.0-SNAPSHOT.jar"), new Path("/hadoop-mapreduce-client-app-0.23.0-SNAPSHOT.jar"));
+			m_dfs.getFileSystem().copyFromLocalFile(new Path("file:///hadoop-mapreduce-client-jobclient-0.23.0-SNAPSHOT.jar"), new Path("/hadoop-mapreduce-client-jobclient-0.23.0-SNAPSHOT.jar"));
 			m_dfs.getFileSystem().copyFromLocalFile(new Path("file:///pig.jar"), new Path("/pig.jar"));
 			m_dfs.getFileSystem().copyFromLocalFile(new Path("file:///pig-test.jar"), new Path("/pig-test.jar"));
 
-            DistributedCache.addFileToClassPath(new Path("/hadoop-mapreduce-client-app-1.0-SNAPSHOT.jar"), m_conf);
+            DistributedCache.addFileToClassPath(new Path("/hadoop-mapreduce-client-app-0.23.0-SNAPSHOT.jar"), m_conf);
             DistributedCache.addFileToClassPath(new Path("/pig.jar"), m_conf);
             DistributedCache.addFileToClassPath(new Path("/pig-test.jar"), m_conf);
-            DistributedCache.addFileToClassPath(new Path("/hadoop-mapreduce-client-jobclient-1.0-SNAPSHOT.jar"), m_conf);
+            DistributedCache.addFileToClassPath(new Path("/hadoop-mapreduce-client-jobclient-0.23.0-SNAPSHOT.jar"), m_conf);
+            String cachefile = m_conf.get("mapreduce.job.cache.files");
+            m_conf.set("alternative.mapreduce.job.cache.files", cachefile);
+            m_conf.unset("mapreduce.job.cache.files");
 
             //ConfigurationUtil.mergeConf(m_conf, m_dfs_conf);
             //ConfigurationUtil.mergeConf(m_conf, m_mr_conf);

Added: pig/branches/branch-0.9/src/META-INF/services/org.apache.hadoop.mapreduce.protocol.ClientProtocolProvider
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.9/src/META-INF/services/org.apache.hadoop.mapreduce.protocol.ClientProtocolProvider?rev=1196767&view=auto
==============================================================================
--- pig/branches/branch-0.9/src/META-INF/services/org.apache.hadoop.mapreduce.protocol.ClientProtocolProvider (added)
+++ pig/branches/branch-0.9/src/META-INF/services/org.apache.hadoop.mapreduce.protocol.ClientProtocolProvider Wed Nov  2 19:20:18 2011
@@ -0,0 +1,32 @@
+#
+#   Licensed under the Apache License, Version 2.0 (the "License");
+#   you may not use this file except in compliance with the License.
+#   You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS,
+#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#   See the License for the specific language governing permissions and
+#   limitations under the License.
+#
+org.apache.hadoop.mapred.LocalClientProtocolProvider
+org.apache.hadoop.mapred.YarnClientProtocolProvider
+org.apache.hadoop.mapred.LocalClientProtocolProvider
+org.apache.hadoop.mapred.YarnClientProtocolProvider
+#
+#   Licensed under the Apache License, Version 2.0 (the "License");
+#   you may not use this file except in compliance with the License.
+#   You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS,
+#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#   See the License for the specific language governing permissions and
+#   limitations under the License.
+#
+org.apache.hadoop.mapred.LocalClientProtocolProvider
+org.apache.hadoop.mapred.YarnClientProtocolProvider

Modified: pig/branches/branch-0.9/src/org/apache/pig/backend/hadoop/datastorage/ConfigurationUtil.java
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.9/src/org/apache/pig/backend/hadoop/datastorage/ConfigurationUtil.java?rev=1196767&r1=1196766&r2=1196767&view=diff
==============================================================================
--- pig/branches/branch-0.9/src/org/apache/pig/backend/hadoop/datastorage/ConfigurationUtil.java (original)
+++ pig/branches/branch-0.9/src/org/apache/pig/backend/hadoop/datastorage/ConfigurationUtil.java Wed Nov  2 19:20:18 2011
@@ -18,6 +18,8 @@
 
 package org.apache.pig.backend.hadoop.datastorage;
 
+import java.lang.reflect.InvocationTargetException;
+import java.lang.reflect.Method;
 import java.util.Enumeration;
 import java.util.Iterator;
 import java.util.Map;
@@ -73,6 +75,23 @@ public class ConfigurationUtil {
             localConf.addResource("core-default.xml");
         } else {
             localConf = new Configuration(true);
+            // It's really hacky, try to get unit test working under hadoop 23.
+            // Hadoop23 MiniMRCluster currently need setup Distributed cache before start, 
+            // so pigtest/conf/hadoop-site.xml contains such entry. This prevents some tests from 
+            // successful (They expect those files in hdfs), so we need to unset it in hadoop 23.
+            // This should go away once MiniMRCluster fix the distributed cache issue.
+            Method unsetMethod = null;
+            try {
+                unsetMethod = localConf.getClass().getMethod("unset", new Class[]{String.class});
+            } catch (Exception e) {
+            }
+            if (unsetMethod!=null) {
+                try {
+                    unsetMethod.invoke(localConf, new Object[]{"mapreduce.job.cache.files"});
+                } catch (Exception e) {
+                    // Should not happen
+                }
+            }
         }
         Properties props = ConfigurationUtil.toProperties(localConf);
         props.setProperty(MapRedUtil.FILE_SYSTEM_NAME, "file:///");

Modified: pig/branches/branch-0.9/src/org/apache/pig/backend/hadoop/executionengine/HExecutionEngine.java
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.9/src/org/apache/pig/backend/hadoop/executionengine/HExecutionEngine.java?rev=1196767&r1=1196766&r2=1196767&view=diff
==============================================================================
--- pig/branches/branch-0.9/src/org/apache/pig/backend/hadoop/executionengine/HExecutionEngine.java (original)
+++ pig/branches/branch-0.9/src/org/apache/pig/backend/hadoop/executionengine/HExecutionEngine.java Wed Nov  2 19:20:18 2011
@@ -174,6 +174,7 @@ public class HExecutionEngine {
             jc.addResource("mapred-default.xml");
             recomputeProperties(jc, properties);
             
+            properties.setProperty("mapreduce.framework.name", "local");
             properties.setProperty(JOB_TRACKER_LOCATION, LOCAL );
             properties.setProperty(FILE_SYSTEM_LOCATION, "file:///");
             properties.setProperty(ALTERNATIVE_FILE_SYSTEM_LOCATION, "file:///");

Modified: pig/branches/branch-0.9/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/JobControlCompiler.java
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.9/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/JobControlCompiler.java?rev=1196767&r1=1196766&r2=1196767&view=diff
==============================================================================
--- pig/branches/branch-0.9/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/JobControlCompiler.java (original)
+++ pig/branches/branch-0.9/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/JobControlCompiler.java Wed Nov  2 19:20:18 2011
@@ -701,7 +701,17 @@ public class JobControlCompiler{
             }
             if (maxCombinedSplitSize > 0)
                 conf.setLong("pig.maxCombinedSplitSize", maxCombinedSplitSize);
-                        
+            
+            // It's a hack to set distributed cache file for hadoop 23. Once MiniMRCluster do not require local
+            // jar on fixed location, this can be removed
+            if (pigContext.getExecType() == ExecType.MAPREDUCE) {
+                String newfiles = conf.get("alternative.mapreduce.job.cache.files");
+                if (newfiles!=null) {
+                    String files = conf.get("mapreduce.job.cache.files");
+                    conf.set("mapreduce.job.cache.files",
+                        files == null ? newfiles.toString() : files + "," + newfiles);
+                }
+            }
             // Serialize the UDF specific context info.
             UDFContext.getUDFContext().serialize(conf);
             Job cjob = new Job(new JobConf(nwJob.getConfiguration()), new ArrayList());

Modified: pig/branches/branch-0.9/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigGenericMapBase.java
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.9/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigGenericMapBase.java?rev=1196767&r1=1196766&r2=1196767&view=diff
==============================================================================
--- pig/branches/branch-0.9/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigGenericMapBase.java (original)
+++ pig/branches/branch-0.9/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigGenericMapBase.java Wed Nov  2 19:20:18 2011
@@ -164,7 +164,7 @@ public abstract class PigGenericMapBase 
         PigMapReduce.sJobContext = context;
         PigMapReduce.sJobConfInternal.set(context.getConfiguration());
         PigMapReduce.sJobConf = context.getConfiguration();
-        inIllustrator = (context instanceof PigMapBase.IllustratorContext);
+        inIllustrator = inIllustrator(context);
         
         PigContext.setPackageImportList((ArrayList<String>)ObjectSerializer.deserialize(job.get("udf.import.list")));
         pigContext = (PigContext)ObjectSerializer.deserialize(job.get("pig.pigContext"));
@@ -301,6 +301,8 @@ public abstract class PigGenericMapBase 
 
     abstract public void collect(Context oc, Tuple tuple) throws InterruptedException, IOException;
 
+    abstract public boolean inIllustrator(Context context);
+    
     /**
      * @return the keyType
      */

Modified: pig/branches/branch-0.9/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigGenericMapReduce.java
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.9/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigGenericMapReduce.java?rev=1196767&r1=1196766&r2=1196767&view=diff
==============================================================================
--- pig/branches/branch-0.9/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigGenericMapReduce.java (original)
+++ pig/branches/branch-0.9/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigGenericMapReduce.java Wed Nov  2 19:20:18 2011
@@ -35,6 +35,7 @@ import org.apache.hadoop.mapreduce.JobCo
 import org.apache.hadoop.mapred.jobcontrol.Job;
 import org.apache.hadoop.mapreduce.Reducer;
 import org.apache.hadoop.mapreduce.TaskAttemptID;
+import org.apache.hadoop.mapreduce.Mapper.Context;
 
 import org.apache.pig.PigException;
 import org.apache.pig.backend.executionengine.ExecException;
@@ -307,9 +308,9 @@ public class PigGenericMapReduce {
         @Override
         protected void setup(Context context) throws IOException, InterruptedException {
             super.setup(context);
-            inIllustrator = (context instanceof PigMapReduce.Reduce.IllustratorContext);
+            inIllustrator = inIllustrator(context);
             if (inIllustrator)
-                pack = ((PigMapReduce.Reduce.IllustratorContext) context).pack;
+                pack = getPack(context);
             Configuration jConf = context.getConfiguration();
             SpillableMemoryManager.configure(ConfigurationUtil.toProperties(jConf));
             sJobContext = context;
@@ -550,6 +551,10 @@ public class PigGenericMapReduce {
          */
         abstract public Context getIllustratorContext(Job job,
                List<Pair<PigNullableWritable, Writable>> input, POPackage pkg) throws IOException, InterruptedException;
+        
+        abstract public boolean inIllustrator(Context context);
+        
+        abstract public POPackage getPack(Context context);
     }
     
     /**

Modified: pig/branches/branch-0.9/src/org/apache/pig/impl/io/PigFile.java
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.9/src/org/apache/pig/impl/io/PigFile.java?rev=1196767&r1=1196766&r2=1196767&view=diff
==============================================================================
--- pig/branches/branch-0.9/src/org/apache/pig/impl/io/PigFile.java (original)
+++ pig/branches/branch-0.9/src/org/apache/pig/impl/io/PigFile.java Wed Nov  2 19:20:18 2011
@@ -80,7 +80,8 @@ public class PigFile {
         PigOutputFormat.setLocation(jc, store);
         OutputCommitter oc;
         // create a simulated TaskAttemptContext
-        TaskAttemptContext tac = HadoopShims.createTaskAttemptContext(conf, new TaskAttemptID());
+        
+        TaskAttemptContext tac = HadoopShims.createTaskAttemptContext(conf, HadoopShims.getNewTaskAttemptID());
         PigOutputFormat.setLocation(tac, store);
         RecordWriter<?,?> rw ;
         try {

Modified: pig/branches/branch-0.9/src/org/apache/pig/tools/pigstats/PigStatusReporter.java
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.9/src/org/apache/pig/tools/pigstats/PigStatusReporter.java?rev=1196767&r1=1196766&r2=1196767&view=diff
==============================================================================
--- pig/branches/branch-0.9/src/org/apache/pig/tools/pigstats/PigStatusReporter.java (original)
+++ pig/branches/branch-0.9/src/org/apache/pig/tools/pigstats/PigStatusReporter.java Wed Nov  2 19:20:18 2011
@@ -74,4 +74,7 @@ public class PigStatusReporter extends S
         }
     }
 
+    public float getProgress() {
+        return 0;
+    }
 }

Modified: pig/branches/branch-0.9/test/findbugsExcludeFile.xml
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.9/test/findbugsExcludeFile.xml?rev=1196767&r1=1196766&r2=1196767&view=diff
==============================================================================
--- pig/branches/branch-0.9/test/findbugsExcludeFile.xml (original)
+++ pig/branches/branch-0.9/test/findbugsExcludeFile.xml Wed Nov  2 19:20:18 2011
@@ -433,4 +433,8 @@
         <Class name = "org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.PigOutputCommitter" />
         <Bug pattern = "REC_CATCH_EXCEPTION" />
     </Match>
+    <Match>
+        <Class name = "org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.PigMapBase$IllustratorContext" />
+        <Bug pattern = "SIC_INNER_SHOULD_BE_STATIC" />
+    </Match>
 </FindBugsFilter>

Modified: pig/branches/branch-0.9/test/org/apache/pig/test/TestAlgebraicEvalLocal.java
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.9/test/org/apache/pig/test/TestAlgebraicEvalLocal.java?rev=1196767&r1=1196766&r2=1196767&view=diff
==============================================================================
--- pig/branches/branch-0.9/test/org/apache/pig/test/TestAlgebraicEvalLocal.java (original)
+++ pig/branches/branch-0.9/test/org/apache/pig/test/TestAlgebraicEvalLocal.java Wed Nov  2 19:20:18 2011
@@ -21,10 +21,12 @@ import java.io.File;
 import java.io.FileOutputStream;
 import java.io.PrintStream;
 import java.util.Iterator;
+import java.util.Properties;
 import java.util.Random;
 
 import junit.framework.TestCase;
 
+import org.apache.pig.ExecType;
 import org.apache.pig.PigServer;
 import org.apache.pig.builtin.PigStorage;
 import org.apache.pig.data.DataType;
@@ -42,7 +44,7 @@ public class TestAlgebraicEvalLocal exte
     @Before
     @Override
     protected void setUp() throws Exception {
-        pig = new PigServer("local");
+        pig = new PigServer(ExecType.LOCAL, new Properties());
     }
     
     Boolean[] nullFlags = new Boolean[]{ false, true};

Modified: pig/branches/branch-0.9/test/org/apache/pig/test/TestBZip.java
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.9/test/org/apache/pig/test/TestBZip.java?rev=1196767&r1=1196766&r2=1196767&view=diff
==============================================================================
--- pig/branches/branch-0.9/test/org/apache/pig/test/TestBZip.java (original)
+++ pig/branches/branch-0.9/test/org/apache/pig/test/TestBZip.java Wed Nov  2 19:20:18 2011
@@ -590,7 +590,9 @@ public class TestBZip {
         Util.createInputFile(cluster, inputFileName, inputData);
 
         String inputScript = "set mapred.output.compress true\n" +
+                "set mapreduce.output.fileoutputformat.compress true\n" +
                 "set mapred.output.compression.codec org.apache.hadoop.io.compress.BZip2Codec\n" +
+                "set mapreduce.output.fileoutputformat.compress.codec org.apache.hadoop.io.compress.BZip2Codec\n" +
                 "a = load '" + inputFileName + "';\n" +
                 "store a into 'output3.bz2';\n" +
                 "store a into 'output3';";

Modified: pig/branches/branch-0.9/test/org/apache/pig/test/TestCombiner.java
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.9/test/org/apache/pig/test/TestCombiner.java?rev=1196767&r1=1196766&r2=1196767&view=diff
==============================================================================
--- pig/branches/branch-0.9/test/org/apache/pig/test/TestCombiner.java (original)
+++ pig/branches/branch-0.9/test/org/apache/pig/test/TestCombiner.java Wed Nov  2 19:20:18 2011
@@ -37,12 +37,17 @@ import junit.framework.Assert;
 import org.apache.pig.EvalFunc;
 import org.apache.pig.ExecType;
 import org.apache.pig.PigServer;
+import org.apache.pig.ResourceSchema.ResourceFieldSchema;
 import org.apache.pig.builtin.PigStorage;
 import org.apache.pig.data.DataBag;
+import org.apache.pig.data.DataType;
 import org.apache.pig.data.DefaultDataBag;
 import org.apache.pig.data.Tuple;
 import org.apache.pig.impl.PigContext;
 import org.apache.pig.impl.io.FileLocalizer;
+import org.apache.pig.impl.util.Utils;
+import org.apache.pig.newplan.logical.relational.LogicalSchema;
+import org.apache.pig.newplan.logical.relational.LogicalSchema.LogicalFieldSchema;
 import org.junit.AfterClass;
 import org.junit.Test;
 
@@ -495,11 +500,7 @@ public class TestCombiner {
             assertFalse(baos.toString().matches("(?si).*combine plan.*"));
     
             Iterator<Tuple> it = pigServer.openIterator("c");
-            int count = 0;
-            while (it.hasNext()) {
-                Tuple t = it.next();
-                assertEquals(expected[count++], t.toString());
-            }
+            Util.checkQueryOutputsAfterSortRecursive(it, expected, "group:chararray,age:long,b:{t:(name:chararray,age:int,gpa:double)}");
         } catch (IOException e) {
             e.printStackTrace();
             Assert.fail();

Modified: pig/branches/branch-0.9/test/org/apache/pig/test/TestCommit.java
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.9/test/org/apache/pig/test/TestCommit.java?rev=1196767&r1=1196766&r2=1196767&view=diff
==============================================================================
--- pig/branches/branch-0.9/test/org/apache/pig/test/TestCommit.java (original)
+++ pig/branches/branch-0.9/test/org/apache/pig/test/TestCommit.java Wed Nov  2 19:20:18 2011
@@ -144,22 +144,21 @@ public class TestCommit extends TestCase
         pigServer.registerQuery("y = foreach x generate age, cnt, max, min;");
         Iterator<Tuple> iter = pigServer.openIterator("y");
         int count = 0;
+        boolean contain1=false, contain2=false;
         while(iter.hasNext()){
             Tuple t = iter.next();
             count++;
-            if (count == 1) {
-                assertTrue(t.get(0).equals(expected1.get(0)));
-                assertTrue(t.get(1).equals(expected1.get(1)));
-                assertTrue(t.get(2).equals(expected1.get(2)));
-                assertTrue(t.get(3).equals(expected1.get(3)));
-            } else if (count == 2){
-                assertTrue(t.get(0).equals(expected2.get(0)));
-                assertTrue(t.get(1).equals(expected2.get(1)));
-                assertTrue(t.get(2).equals(expected2.get(2)));
-                assertTrue(t.get(3).equals(expected2.get(3)));
+            if (t.get(0).equals(expected1.get(0)) && t.get(1).equals(expected1.get(1)) && t.get(2).equals(expected1.get(2)) && t.get(3).equals(expected1.get(3))) {
+                contain1 = true;
             }
+            
+            if (t.get(0).equals(expected2.get(0)) && t.get(1).equals(expected2.get(1)) && t.get(2).equals(expected2.get(2)) && t.get(3).equals(expected2.get(3))) {
+                contain2 = true;
+            }
+            
         }
         assertEquals(count, 2);
+        assertTrue(contain1 && contain2);
         Util.deleteFile(cluster, "testCheckin2-input.txt");
     }    
 }

Modified: pig/branches/branch-0.9/test/org/apache/pig/test/TestNewPlanOperatorPlan.java
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.9/test/org/apache/pig/test/TestNewPlanOperatorPlan.java?rev=1196767&r1=1196766&r2=1196767&view=diff
==============================================================================
--- pig/branches/branch-0.9/test/org/apache/pig/test/TestNewPlanOperatorPlan.java (original)
+++ pig/branches/branch-0.9/test/org/apache/pig/test/TestNewPlanOperatorPlan.java Wed Nov  2 19:20:18 2011
@@ -21,6 +21,7 @@ package org.apache.pig.test;
 import java.util.Collection;
 import java.util.List;
 
+import org.apache.hadoop.conf.Configuration;
 import org.apache.pig.FuncSpec;
 import org.apache.pig.builtin.PigStorage;
 import org.apache.pig.data.DataType;
@@ -1505,7 +1506,7 @@ public class TestNewPlanOperatorPlan ext
         aschema1.addField(new LogicalSchema.LogicalFieldSchema(
             "x", null, DataType.INTEGER));
         LOLoad A1 = new LOLoad(new FileSpec("/abc",
-            new FuncSpec(DummyLoad.class.getName(), new String[] {"x", "y"})), aschema1, lp1, null);
+            new FuncSpec(DummyLoad.class.getName(), new String[] {"x", "y"})), aschema1, lp1, new Configuration());
         lp1.add(A1);
         
         LogicalExpressionPlan fp1 = new LogicalExpressionPlan();
@@ -1523,7 +1524,7 @@ public class TestNewPlanOperatorPlan ext
         
         LogicalPlan lp2 = new LogicalPlan();
         LOLoad A2 = new LOLoad(new FileSpec("/abc",
-            new FuncSpec(DummyLoad.class.getName(), new String[] {"x", "z"})), null, lp2, null);
+            new FuncSpec(DummyLoad.class.getName(), new String[] {"x", "z"})), null, lp2, new Configuration());
         lp2.add(A2);
         
         LogicalExpressionPlan fp2 = new LogicalExpressionPlan();

Modified: pig/branches/branch-0.9/test/org/apache/pig/test/TestPigRunner.java
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.9/test/org/apache/pig/test/TestPigRunner.java?rev=1196767&r1=1196766&r2=1196767&view=diff
==============================================================================
--- pig/branches/branch-0.9/test/org/apache/pig/test/TestPigRunner.java (original)
+++ pig/branches/branch-0.9/test/org/apache/pig/test/TestPigRunner.java Wed Nov  2 19:20:18 2011
@@ -562,7 +562,8 @@ public class TestPigRunner {
         Assert.assertNotNull(ctx);
 
         assertTrue(ctx.extraJars.contains(ClassLoader.getSystemResource("pig-withouthadoop.jar")));
-        assertEquals("default", ctx.getProperties().getProperty("mapred.job.queue.name"));
+        assertTrue("default", ctx.getProperties().getProperty("mapred.job.queue.name")!=null && ctx.getProperties().getProperty("mapred.job.queue.name").equals("default")||
+                ctx.getProperties().getProperty("mapreduce.job.queuename")!=null && ctx.getProperties().getProperty("mapreduce.job.queuename").equals("default"));
        
     }
 

Modified: pig/branches/branch-0.9/test/org/apache/pig/test/TestProject.java
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.9/test/org/apache/pig/test/TestProject.java?rev=1196767&r1=1196766&r2=1196767&view=diff
==============================================================================
--- pig/branches/branch-0.9/test/org/apache/pig/test/TestProject.java (original)
+++ pig/branches/branch-0.9/test/org/apache/pig/test/TestProject.java Wed Nov  2 19:20:18 2011
@@ -339,7 +339,7 @@ public class TestProject extends  junit.
         String query = "a = load '" + inputFileName + "';" +
                 "b = group a all;" +
                 "c = foreach b generate flatten(a.($1, $2)),a.$2;";
-        
+            
         PigServer ps = new PigServer(ExecType.LOCAL);
         Util.registerMultiLineQuery(ps, query);
         Iterator<Tuple> it = ps.openIterator("c");
@@ -347,10 +347,16 @@ public class TestProject extends  junit.
                 (Tuple) Util.getPigConstant("('world', null, {(null),(null)})"),
                 (Tuple) Util.getPigConstant("('bye', null, {(null),(null)})")
         };
-        int i = 0;
+        boolean contains0 = false;
+        boolean contains1 = false;
         while(it.hasNext()) {
-            assertEquals(expectedResults[i++].toString(), it.next().toString());
+            String actualResult = it.next().toString();
+            if (actualResult.equals(expectedResults[0].toString()))
+                contains0 = true;
+            if (actualResult.equals(expectedResults[1].toString()))
+                contains1 = true;
         }
+        assertTrue(contains0&&contains1);
     }
 
     @Test

Modified: pig/branches/branch-0.9/test/org/apache/pig/test/TestPruneColumn.java
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.9/test/org/apache/pig/test/TestPruneColumn.java?rev=1196767&r1=1196766&r2=1196767&view=diff
==============================================================================
--- pig/branches/branch-0.9/test/org/apache/pig/test/TestPruneColumn.java (original)
+++ pig/branches/branch-0.9/test/org/apache/pig/test/TestPruneColumn.java Wed Nov  2 19:20:18 2011
@@ -573,13 +573,11 @@ public class TestPruneColumn extends Tes
         pigServer.registerQuery("C = foreach B generate $1;");
         Iterator<Tuple> iter = pigServer.openIterator("C");
         
+        String[] expected = new String[] {
+                "({(1,2,3),(2,5,2)})"
+        };
         assertTrue(iter.hasNext());
-        Tuple t = iter.next();
-        
-        assertTrue(t.size()==1);
-        assertTrue(t.get(0).toString().equals("{(1,2,3),(2,5,2)}"));
-        
-        assertFalse(iter.hasNext());
+        Util.checkQueryOutputsAfterSortRecursive(iter, expected, org.apache.pig.newplan.logical.Util.translateSchema(pigServer.dumpSchema("C")));   
         
         assertTrue(emptyLogFileMessage());
     }
@@ -1215,13 +1213,11 @@ public class TestPruneColumn extends Tes
         
         Iterator<Tuple> iter = pigServer.openIterator("C");
         
-        assertTrue(iter.hasNext());
-        Tuple t = iter.next();
-        assertTrue(t.size()==2);
-        assertTrue(t.get(0).toString().equals("all"));
-        assertTrue(t.get(1).toString().equals("{([key2#2,key1#1],1),([key2#4,key1#2],2)}"));
+        String[] expected = new String[] {
+                "(all,{([key2#2,key1#1],1),([key2#4,key1#2],2)})"
+        };
         
-        assertFalse(iter.hasNext());
+        Util.checkQueryOutputsAfterSortRecursive(iter, expected, org.apache.pig.newplan.logical.Util.translateSchema(pigServer.dumpSchema("C")));
         
         assertTrue(checkLogFileMessage(new String[]{"Columns pruned for A: $0"}));
     }
@@ -1920,21 +1916,14 @@ public class TestPruneColumn extends Tes
         pigServer.registerQuery("D = filter C by A::a1>=B::a1;");
         Iterator<Tuple> iter = pigServer.openIterator("D");
 
-        assertTrue(iter.hasNext());
-        Tuple t = iter.next();
-        assertTrue(t.toString().equals("(1,2,3,1,2)"));
+        String [] expected = new String[] {
+                "(1,2,3,1,2)",
+                "(1,3,2,1,2)",
+                "(1,3,2,1,3)",
+                "(2,5,2,2,5)"
+        };
         
-        assertTrue(iter.hasNext());
-        t = iter.next();
-        assertTrue(t.toString().equals("(1,3,2,1,2)"));
-        
-        assertTrue(iter.hasNext());
-        t = iter.next();
-        assertTrue(t.toString().equals("(1,3,2,1,3)"));
-
-        assertTrue(iter.hasNext());
-        t = iter.next();
-        assertTrue(t.toString().equals("(2,5,2,2,5)"));
+        Util.checkQueryOutputsAfterSortRecursive(iter, expected, org.apache.pig.newplan.logical.Util.translateSchema(pigServer.dumpSchema("D")));
         
         assertTrue(emptyLogFileMessage());
     }

Modified: pig/branches/branch-0.9/test/org/apache/pig/test/TestScalarAliases.java
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.9/test/org/apache/pig/test/TestScalarAliases.java?rev=1196767&r1=1196766&r2=1196767&view=diff
==============================================================================
--- pig/branches/branch-0.9/test/org/apache/pig/test/TestScalarAliases.java (original)
+++ pig/branches/branch-0.9/test/org/apache/pig/test/TestScalarAliases.java Wed Nov  2 19:20:18 2011
@@ -410,16 +410,13 @@ public class TestScalarAliases  {
 
         Iterator<Tuple> iter = pigServer.openIterator("Y");
 
-        Tuple t = iter.next();
-        assertTrue(t.toString().equals("(1,5,Total3,three)"));
-
-        t = iter.next();
-        assertTrue(t.toString().equals("(2,10,Total3,three)"));
-
-        t = iter.next();
-        assertTrue(t.toString().equals("(3,20,Total3,three)"));
-
-        assertFalse(iter.hasNext());
+        String[] expected = new String[] {
+                "(1,5,Total3,three)",
+                "(2,10,Total3,three)",
+                "(3,20,Total3,three)"
+        };
+        
+        Util.checkQueryOutputsAfterSortRecursive(iter, expected, org.apache.pig.newplan.logical.Util.translateSchema(pigServer.dumpSchema("Y")));
     }
 
     // See PIG-1434

Modified: pig/branches/branch-0.9/test/org/apache/pig/test/TestScriptUDF.java
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.9/test/org/apache/pig/test/TestScriptUDF.java?rev=1196767&r1=1196766&r2=1196767&view=diff
==============================================================================
--- pig/branches/branch-0.9/test/org/apache/pig/test/TestScriptUDF.java (original)
+++ pig/branches/branch-0.9/test/org/apache/pig/test/TestScriptUDF.java Wed Nov  2 19:20:18 2011
@@ -270,7 +270,7 @@ public class TestScriptUDF{
     public void testPythonScriptUDFBagInput() throws Exception{
         String[] script = {
                 "#!/usr/bin/python",
-                "@outputSchema(\"bag:{(y:{t:(word:chararray)})}\")",
+                "@outputSchema(\"bag:{(y:{t:(len:int,word:chararray)})}\")",
                 "def collect(bag):" ,
                 "\toutBag = []",
                 "\tfor word in bag:",
@@ -296,42 +296,13 @@ public class TestScriptUDF{
         pigServer.registerQuery("C = foreach B generate pig.collect(A);");
 
         Iterator<Tuple> iter = pigServer.openIterator("C");
-        Assert.assertTrue(iter.hasNext());
-        Tuple t = iter.next();
         
-        DataBag bag; 
-        Tuple tup;
-        bag = BagFactory.getInstance().newDefaultBag();
-        tup = TupleFactory.getInstance().newTuple();
-        tup.append(3);
-        tup.append("hello");
-        bag.add(tup);
-        tup = TupleFactory.getInstance().newTuple();
-        tup.append(3);
-        tup.append("world");
-        bag.add(tup);
-        tup = TupleFactory.getInstance().newTuple();
-        tup.append(3);
-        tup.append("program");
-        bag.add(tup);
-    
-        Assert.assertTrue(t.get(0).toString().equals(bag.toString()));
-        
-        Assert.assertTrue(iter.hasNext());
-        t = iter.next();
-        
-        bag = BagFactory.getInstance().newDefaultBag();
-        tup = TupleFactory.getInstance().newTuple();
-        tup.append(2);
-        tup.append("pig");
-        bag.add(tup);
-        tup = TupleFactory.getInstance().newTuple();
-        tup.append(2);
-        tup.append("hadoop");
-        bag.add(tup);
-        
-        Assert.assertTrue(t.get(0).toString().equals(bag.toString()));
-    }
+        String[] expected = new String[] {
+        		"({(3,hello),(3,world),(3,program)})",
+        		"({(2,hadoop),(2,pig)})"
+        };
+        Util.checkQueryOutputsAfterSortRecursive(iter, expected, "y: {(len:int, word:chararray)}");
+        }
     
     @Test
     public void testPythonScriptUDFMapInput() throws Exception{



Mime
View raw message