incubator-bigtop-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From r..@apache.org
Subject svn commit: r1196333 [2/3] - in /incubator/bigtop/branches/hadoop-0.23: bigtop-packages/src/common/pig/do-component-build bigtop-packages/src/common/pig/patch bigtop-packages/src/deb/pig/rules bigtop-packages/src/rpm/pig/SPECS/pig.spec bigtop.mk
Date Tue, 01 Nov 2011 22:29:24 GMT
Added: incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/common/pig/patch
URL: http://svn.apache.org/viewvc/incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/common/pig/patch?rev=1196333&view=auto
==============================================================================
--- incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/common/pig/patch (added)
+++ incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/common/pig/patch Tue Nov  1 22:29:24 2011
@@ -0,0 +1,2024 @@
+Index: test/org/apache/pig/test/TestScalarAliases.java
+===================================================================
+--- test/org/apache/pig/test/TestScalarAliases.java	(revision 1190473)
++++ test/org/apache/pig/test/TestScalarAliases.java	(working copy)
+@@ -410,16 +410,13 @@
+ 
+         Iterator<Tuple> iter = pigServer.openIterator("Y");
+ 
+-        Tuple t = iter.next();
+-        assertTrue(t.toString().equals("(1,5,Total3,three)"));
+-
+-        t = iter.next();
+-        assertTrue(t.toString().equals("(2,10,Total3,three)"));
+-
+-        t = iter.next();
+-        assertTrue(t.toString().equals("(3,20,Total3,three)"));
+-
+-        assertFalse(iter.hasNext());
++        String[] expected = new String[] {
++                "(1,5,Total3,three)",
++                "(2,10,Total3,three)",
++                "(3,20,Total3,three)"
++        };
++        
++        Util.checkQueryOutputsAfterSortRecursive(iter, expected, org.apache.pig.newplan.logical.Util.translateSchema(pigServer.dumpSchema("Y")));
+     }
+ 
+     // See PIG-1434
+Index: test/org/apache/pig/test/TestScriptUDF.java
+===================================================================
+--- test/org/apache/pig/test/TestScriptUDF.java	(revision 1190473)
++++ test/org/apache/pig/test/TestScriptUDF.java	(working copy)
+@@ -270,7 +270,7 @@
+     public void testPythonScriptUDFBagInput() throws Exception{
+         String[] script = {
+                 "#!/usr/bin/python",
+-                "@outputSchema(\"bag:{(y:{t:(word:chararray)})}\")",
++                "@outputSchema(\"bag:{(y:{t:(len:int,word:chararray)})}\")",
+                 "def collect(bag):" ,
+                 "\toutBag = []",
+                 "\tfor word in bag:",
+@@ -296,43 +296,14 @@
+         pigServer.registerQuery("C = foreach B generate pig.collect(A);");
+ 
+         Iterator<Tuple> iter = pigServer.openIterator("C");
+-        Assert.assertTrue(iter.hasNext());
+-        Tuple t = iter.next();
+         
+-        DataBag bag; 
+-        Tuple tup;
+-        bag = BagFactory.getInstance().newDefaultBag();
+-        tup = TupleFactory.getInstance().newTuple();
+-        tup.append(3);
+-        tup.append("hello");
+-        bag.add(tup);
+-        tup = TupleFactory.getInstance().newTuple();
+-        tup.append(3);
+-        tup.append("world");
+-        bag.add(tup);
+-        tup = TupleFactory.getInstance().newTuple();
+-        tup.append(3);
+-        tup.append("program");
+-        bag.add(tup);
++        String[] expected = new String[] {
++        		"({(3,hello),(3,world),(3,program)})",
++        		"({(2,hadoop),(2,pig)})"
++        };
++        Util.checkQueryOutputsAfterSortRecursive(iter, expected, "y: {(len:int, word:chararray)}");
++        }
+     
+-        Assert.assertTrue(t.get(0).toString().equals(bag.toString()));
+-        
+-        Assert.assertTrue(iter.hasNext());
+-        t = iter.next();
+-        
+-        bag = BagFactory.getInstance().newDefaultBag();
+-        tup = TupleFactory.getInstance().newTuple();
+-        tup.append(2);
+-        tup.append("pig");
+-        bag.add(tup);
+-        tup = TupleFactory.getInstance().newTuple();
+-        tup.append(2);
+-        tup.append("hadoop");
+-        bag.add(tup);
+-        
+-        Assert.assertTrue(t.get(0).toString().equals(bag.toString()));
+-    }
+-    
+     @Test
+     public void testPythonScriptUDFMapInput() throws Exception{
+         String[] script = {
+Index: test/org/apache/pig/test/TestBZip.java
+===================================================================
+--- test/org/apache/pig/test/TestBZip.java	(revision 1190473)
++++ test/org/apache/pig/test/TestBZip.java	(working copy)
+@@ -590,7 +590,9 @@
+         Util.createInputFile(cluster, inputFileName, inputData);
+ 
+         String inputScript = "set mapred.output.compress true\n" +
++                "set mapreduce.output.fileoutputformat.compress true\n" +
+                 "set mapred.output.compression.codec org.apache.hadoop.io.compress.BZip2Codec\n" +
++                "set mapreduce.output.fileoutputformat.compress.codec org.apache.hadoop.io.compress.BZip2Codec\n" +
+                 "a = load '" + inputFileName + "';\n" +
+                 "store a into 'output3.bz2';\n" +
+                 "store a into 'output3';";
+Index: test/org/apache/pig/test/TestStreamingLocal.java
+===================================================================
+--- test/org/apache/pig/test/TestStreamingLocal.java	(revision 1190473)
++++ test/org/apache/pig/test/TestStreamingLocal.java	(working copy)
+@@ -197,7 +197,7 @@
+             }
+ 
+             // Run the query and check the results
+-            Util.checkQueryOutputs(pigServer.openIterator("OP"), expectedResults);
++            Util.checkQueryOutputsAfterSort(pigServer.openIterator("OP"), expectedResults);
+         }
+     }
+ 
+Index: test/org/apache/pig/test/TestCombiner.java
+===================================================================
+--- test/org/apache/pig/test/TestCombiner.java	(revision 1190473)
++++ test/org/apache/pig/test/TestCombiner.java	(working copy)
+@@ -37,12 +37,17 @@
+ import org.apache.pig.EvalFunc;
+ import org.apache.pig.ExecType;
+ import org.apache.pig.PigServer;
++import org.apache.pig.ResourceSchema.ResourceFieldSchema;
+ import org.apache.pig.builtin.PigStorage;
+ import org.apache.pig.data.DataBag;
++import org.apache.pig.data.DataType;
+ import org.apache.pig.data.DefaultDataBag;
+ import org.apache.pig.data.Tuple;
+ import org.apache.pig.impl.PigContext;
+ import org.apache.pig.impl.io.FileLocalizer;
++import org.apache.pig.impl.util.Utils;
++import org.apache.pig.newplan.logical.relational.LogicalSchema;
++import org.apache.pig.newplan.logical.relational.LogicalSchema.LogicalFieldSchema;
+ import org.junit.AfterClass;
+ import org.junit.Test;
+ 
+@@ -495,11 +500,7 @@
+             assertFalse(baos.toString().matches("(?si).*combine plan.*"));
+     
+             Iterator<Tuple> it = pigServer.openIterator("c");
+-            int count = 0;
+-            while (it.hasNext()) {
+-                Tuple t = it.next();
+-                assertEquals(expected[count++], t.toString());
+-            }
++            Util.checkQueryOutputsAfterSortRecursive(it, expected, "group:chararray,age:long,b:{t:(name:chararray,age:int,gpa:double)}");
+         } catch (IOException e) {
+             e.printStackTrace();
+             Assert.fail();
+Index: test/org/apache/pig/test/TestPigRunner.java
+===================================================================
+--- test/org/apache/pig/test/TestPigRunner.java	(revision 1190473)
++++ test/org/apache/pig/test/TestPigRunner.java	(working copy)
+@@ -562,7 +562,8 @@
+         Assert.assertNotNull(ctx);
+ 
+         assertTrue(ctx.extraJars.contains(ClassLoader.getSystemResource("pig-withouthadoop.jar")));
+-        assertEquals("default", ctx.getProperties().getProperty("mapred.job.queue.name"));
++        assertTrue("default", ctx.getProperties().getProperty("mapred.job.queue.name")!=null && ctx.getProperties().getProperty("mapred.job.queue.name").equals("default")||
++                ctx.getProperties().getProperty("mapreduce.job.queuename")!=null && ctx.getProperties().getProperty("mapreduce.job.queuename").equals("default"));
+        
+     }
+ 
+Index: test/org/apache/pig/test/TestAlgebraicEvalLocal.java
+===================================================================
+--- test/org/apache/pig/test/TestAlgebraicEvalLocal.java	(revision 1190473)
++++ test/org/apache/pig/test/TestAlgebraicEvalLocal.java	(working copy)
+@@ -21,10 +21,12 @@
+ import java.io.FileOutputStream;
+ import java.io.PrintStream;
+ import java.util.Iterator;
++import java.util.Properties;
+ import java.util.Random;
+ 
+ import junit.framework.TestCase;
+ 
++import org.apache.pig.ExecType;
+ import org.apache.pig.PigServer;
+ import org.apache.pig.builtin.PigStorage;
+ import org.apache.pig.data.DataType;
+@@ -42,7 +44,7 @@
+     @Before
+     @Override
+     protected void setUp() throws Exception {
+-        pig = new PigServer("local");
++        pig = new PigServer(ExecType.LOCAL, new Properties());
+     }
+     
+     Boolean[] nullFlags = new Boolean[]{ false, true};
+Index: test/org/apache/pig/test/TestProject.java
+===================================================================
+--- test/org/apache/pig/test/TestProject.java	(revision 1190473)
++++ test/org/apache/pig/test/TestProject.java	(working copy)
+@@ -339,7 +339,7 @@
+         String query = "a = load '" + inputFileName + "';" +
+                 "b = group a all;" +
+                 "c = foreach b generate flatten(a.($1, $2)),a.$2;";
+-        
++            
+         PigServer ps = new PigServer(ExecType.LOCAL);
+         Util.registerMultiLineQuery(ps, query);
+         Iterator<Tuple> it = ps.openIterator("c");
+@@ -347,10 +347,16 @@
+                 (Tuple) Util.getPigConstant("('world', null, {(null),(null)})"),
+                 (Tuple) Util.getPigConstant("('bye', null, {(null),(null)})")
+         };
+-        int i = 0;
++        boolean contains0 = false;
++        boolean contains1 = false;
+         while(it.hasNext()) {
+-            assertEquals(expectedResults[i++].toString(), it.next().toString());
++            String actualResult = it.next().toString();
++            if (actualResult.equals(expectedResults[0].toString()))
++                contains0 = true;
++            if (actualResult.equals(expectedResults[1].toString()))
++                contains1 = true;
+         }
++        assertTrue(contains0&&contains1);
+     }
+ 
+     @Test
+Index: test/org/apache/pig/test/TestStreaming.java
+===================================================================
+--- test/org/apache/pig/test/TestStreaming.java	(revision 1190473)
++++ test/org/apache/pig/test/TestStreaming.java	(working copy)
+@@ -214,7 +214,7 @@
+             }
+     		
+     		// Run the query and check the results
+-    		Util.checkQueryOutputs(pigServer.openIterator("OP"), expectedResults);
++    		Util.checkQueryOutputsAfterSort(pigServer.openIterator("OP"), expectedResults);
+         }
+ 	}
+ 
+Index: test/org/apache/pig/test/TestNewPlanOperatorPlan.java
+===================================================================
+--- test/org/apache/pig/test/TestNewPlanOperatorPlan.java	(revision 1190473)
++++ test/org/apache/pig/test/TestNewPlanOperatorPlan.java	(working copy)
+@@ -1521,7 +1521,7 @@
+         aschema1.addField(new LogicalSchema.LogicalFieldSchema(
+             "x", null, DataType.INTEGER));
+         LOLoad A1 = new LOLoad(new FileSpec("/abc",
+-            new FuncSpec(DummyLoad.class.getName(), new String[] {"x", "y"})), aschema1, lp1, null);
++            new FuncSpec(DummyLoad.class.getName(), new String[] {"x", "y"})), aschema1, lp1, new Configuration());
+         lp1.add(A1);
+         
+         LogicalExpressionPlan fp1 = new LogicalExpressionPlan();
+@@ -1539,7 +1539,7 @@
+         
+         LogicalPlan lp2 = new LogicalPlan();
+         LOLoad A2 = new LOLoad(new FileSpec("/abc",
+-            new FuncSpec(DummyLoad.class.getName(), new String[] {"x", "z"})), null, lp2, null);
++            new FuncSpec(DummyLoad.class.getName(), new String[] {"x", "z"})), null, lp2, new Configuration());
+         lp2.add(A2);
+         
+         LogicalExpressionPlan fp2 = new LogicalExpressionPlan();
+Index: test/org/apache/pig/test/TestSecondarySort.java
+===================================================================
+--- test/org/apache/pig/test/TestSecondarySort.java	(revision 1190473)
++++ test/org/apache/pig/test/TestSecondarySort.java	(working copy)
+@@ -38,6 +38,7 @@
+ import org.apache.pig.data.Tuple;
+ import org.apache.pig.impl.PigContext;
+ import org.apache.pig.impl.io.FileLocalizer;
++import org.apache.pig.impl.logicalLayer.schema.Schema;
+ import org.junit.AfterClass;
+ import org.junit.Before;
+ import org.junit.Test;
+@@ -462,16 +463,20 @@
+         ps1.println("1\t2\t4");
+         ps1.println("2\t3\t4");
+         ps1.close();
++        
++        String expected[] = {
++                "(2,{(2,3,4)})",
++                "(1,{(1,2,3),(1,2,4),(1,2,4),(1,2,4),(1,3,4)})"
++        };
++        
+         Util.copyFromLocalToCluster(cluster, tmpFile1.getCanonicalPath(), tmpFile1.getCanonicalPath());
+         pigServer.registerQuery("A = LOAD '" + tmpFile1.getCanonicalPath() + "' AS (a0, a1, a2);");
+         pigServer.registerQuery("B = group A by $0 parallel 2;");
+         pigServer.registerQuery("C = foreach B { D = limit A 10; E = order D by $1; generate group, E;};");
+         Iterator<Tuple> iter = pigServer.openIterator("C");
+-        assertTrue(iter.hasNext());
+-        assertTrue(iter.next().toString().equals("(2,{(2,3,4)})"));
+-        assertTrue(iter.hasNext());
+-        assertTrue(iter.next().toString().equals("(1,{(1,2,3),(1,2,4),(1,2,4),(1,2,4),(1,3,4)})"));
+-        assertFalse(iter.hasNext());
++        Schema s = pigServer.dumpSchema("C");
++        
++        Util.checkQueryOutputsAfterSortRecursive(iter, expected, org.apache.pig.newplan.logical.Util.translateSchema(s));
+         Util.deleteFile(cluster, tmpFile1.getCanonicalPath());
+     }
+ 
+@@ -486,16 +491,20 @@
+         ps1.println("1\t8\t4");
+         ps1.println("2\t3\t4");
+         ps1.close();
++        
++        String expected[] = {
++                "(2,{(2,3,4)})",
++                "(1,{(1,8,4),(1,4,4),(1,3,4),(1,2,3),(1,2,4)})"
++        };
++        
+         Util.copyFromLocalToCluster(cluster, tmpFile1.getCanonicalPath(), tmpFile1.getCanonicalPath());
+         pigServer.registerQuery("A = LOAD '" + tmpFile1.getCanonicalPath() + "' AS (a0, a1, a2);");
+         pigServer.registerQuery("B = group A by $0 parallel 2;");
+         pigServer.registerQuery("C = foreach B { D = order A by a1 desc; generate group, D;};");
+         Iterator<Tuple> iter = pigServer.openIterator("C");
+-        assertTrue(iter.hasNext());
+-        assertEquals("(2,{(2,3,4)})", iter.next().toString());
+-        assertTrue(iter.hasNext());
+-        assertEquals("(1,{(1,8,4),(1,4,4),(1,3,4),(1,2,3),(1,2,4)})", iter.next().toString());
+-        assertFalse(iter.hasNext());
++        Schema s = pigServer.dumpSchema("C");
++        
++        Util.checkQueryOutputsAfterSortRecursive(iter, expected, org.apache.pig.newplan.logical.Util.translateSchema(s));
+         Util.deleteFile(cluster, tmpFile1.getCanonicalPath());
+     }
+ 
+@@ -571,18 +580,21 @@
+         ps1.println("1\t2\t4");
+         ps1.println("2\t3\t4");
+         ps1.close();
++        
++        String expected[] = {
++                "((1,2),{(2,3),(2,4),(2,4),(2,4)})",
++                "((1,3),{(3,4)})",
++                "((2,3),{(3,4)})"
++        };
+         Util.copyFromLocalToCluster(cluster, tmpFile1.getCanonicalPath(), tmpFile1.getCanonicalPath());
+         pigServer.registerQuery("A = LOAD '" + tmpFile1.getCanonicalPath() + "' AS (a0, a1, a2);");
+         pigServer.registerQuery("B = group A by (a0, a1);");
+         pigServer.registerQuery("C = foreach B { C1 = A.(a1,a2); generate group, C1;};");
+         Iterator<Tuple> iter = pigServer.openIterator("C");
+-        assertTrue(iter.hasNext());
+-        assertEquals("((1,2),{(2,3),(2,4),(2,4),(2,4)})", iter.next().toString());
+-        assertTrue(iter.hasNext());
+-        assertEquals("((1,3),{(3,4)})", iter.next().toString());
+-        assertTrue(iter.hasNext());
+-        assertEquals("((2,3),{(3,4)})", iter.next().toString());
+-        assertFalse(iter.hasNext());
++        Schema s = pigServer.dumpSchema("C");
++        
++        Util.checkQueryOutputsAfterSortRecursive(iter, expected, org.apache.pig.newplan.logical.Util.translateSchema(s));
++        
+         Util.deleteFile(cluster, tmpFile1.getCanonicalPath());
+     }
+ }
+\ No newline at end of file
+Index: test/org/apache/pig/test/TestPruneColumn.java
+===================================================================
+--- test/org/apache/pig/test/TestPruneColumn.java	(revision 1190473)
++++ test/org/apache/pig/test/TestPruneColumn.java	(working copy)
+@@ -573,14 +573,12 @@
+         pigServer.registerQuery("C = foreach B generate $1;");
+         Iterator<Tuple> iter = pigServer.openIterator("C");
+         
++        String[] expected = new String[] {
++                "({(1,2,3),(2,5,2)})"
++        };
+         assertTrue(iter.hasNext());
+-        Tuple t = iter.next();
++        Util.checkQueryOutputsAfterSortRecursive(iter, expected, org.apache.pig.newplan.logical.Util.translateSchema(pigServer.dumpSchema("C")));   
+         
+-        assertTrue(t.size()==1);
+-        assertTrue(t.get(0).toString().equals("{(1,2,3),(2,5,2)}"));
+-        
+-        assertFalse(iter.hasNext());
+-        
+         assertTrue(emptyLogFileMessage());
+     }
+     
+@@ -1215,13 +1213,11 @@
+         
+         Iterator<Tuple> iter = pigServer.openIterator("C");
+         
+-        assertTrue(iter.hasNext());
+-        Tuple t = iter.next();
+-        assertTrue(t.size()==2);
+-        assertTrue(t.get(0).toString().equals("all"));
+-        assertTrue(t.get(1).toString().equals("{([key2#2,key1#1],1),([key2#4,key1#2],2)}"));
++        String[] expected = new String[] {
++                "(all,{([key2#2,key1#1],1),([key2#4,key1#2],2)})"
++        };
+         
+-        assertFalse(iter.hasNext());
++        Util.checkQueryOutputsAfterSortRecursive(iter, expected, org.apache.pig.newplan.logical.Util.translateSchema(pigServer.dumpSchema("C")));
+         
+         assertTrue(checkLogFileMessage(new String[]{"Columns pruned for A: $0"}));
+     }
+@@ -1920,22 +1916,15 @@
+         pigServer.registerQuery("D = filter C by A::a1>=B::a1;");
+         Iterator<Tuple> iter = pigServer.openIterator("D");
+ 
+-        assertTrue(iter.hasNext());
+-        Tuple t = iter.next();
+-        assertTrue(t.toString().equals("(1,2,3,1,2)"));
++        String [] expected = new String[] {
++                "(1,2,3,1,2)",
++                "(1,3,2,1,2)",
++                "(1,3,2,1,3)",
++                "(2,5,2,2,5)"
++        };
+         
+-        assertTrue(iter.hasNext());
+-        t = iter.next();
+-        assertTrue(t.toString().equals("(1,3,2,1,2)"));
++        Util.checkQueryOutputsAfterSortRecursive(iter, expected, org.apache.pig.newplan.logical.Util.translateSchema(pigServer.dumpSchema("D")));
+         
+-        assertTrue(iter.hasNext());
+-        t = iter.next();
+-        assertTrue(t.toString().equals("(1,3,2,1,3)"));
+-
+-        assertTrue(iter.hasNext());
+-        t = iter.next();
+-        assertTrue(t.toString().equals("(2,5,2,2,5)"));
+-        
+         assertTrue(emptyLogFileMessage());
+     }
+ 
+@@ -2080,20 +2069,10 @@
+         pigServer.registerQuery("F = foreach E generate a0;");
+         Iterator<Tuple> iter = pigServer.openIterator("F");
+         
+-        assertTrue(iter.hasNext());
+-        Tuple t = iter.next();
++        String[] expected = new String[] {"(1)", "(2)"};
+         
+-        assertTrue(t.size()==1);
+-        assertTrue(t.toString().equals("(1)"));
+-        
+-        assertTrue(iter.hasNext());
+-        t = iter.next();
+-        
+-        assertTrue(t.size()==1);
+-        assertTrue(t.toString().equals("(2)"));
++        Util.checkQueryOutputsAfterSortRecursive(iter, expected, org.apache.pig.newplan.logical.Util.translateSchema(pigServer.dumpSchema("F")));
+ 
+-        assertFalse(iter.hasNext());
+-        
+         assertTrue(checkLogFileMessage(new String[]{"Columns pruned for A: $1"}));
+     }
+ 
+Index: test/org/apache/pig/test/Util.java
+===================================================================
+--- test/org/apache/pig/test/Util.java	(revision 1190473)
++++ test/org/apache/pig/test/Util.java	(working copy)
+@@ -58,8 +58,10 @@
+ import org.apache.log4j.Logger;
+ import org.apache.log4j.SimpleLayout;
+ import org.apache.pig.ExecType;
++import org.apache.pig.LoadCaster;
+ import org.apache.pig.PigException;
+ import org.apache.pig.PigServer;
++import org.apache.pig.ResourceSchema.ResourceFieldSchema;
+ import org.apache.pig.backend.executionengine.ExecException;
+ import org.apache.pig.backend.hadoop.datastorage.ConfigurationUtil;
+ import org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.MRCompiler;
+@@ -69,10 +71,13 @@
+ import org.apache.pig.backend.hadoop.executionengine.physicalLayer.plans.PhysicalPlan;
+ import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POStore;
+ import org.apache.pig.backend.hadoop.executionengine.util.MapRedUtil;
++import org.apache.pig.builtin.Utf8StorageConverter;
+ import org.apache.pig.data.BagFactory;
+ import org.apache.pig.data.DataBag;
+ import org.apache.pig.data.DataByteArray;
+ import org.apache.pig.data.DataType;
++import org.apache.pig.data.DefaultBagFactory;
++import org.apache.pig.data.SortedDataBag;
+ import org.apache.pig.data.Tuple;
+ import org.apache.pig.data.TupleFactory;
+ import org.apache.pig.impl.PigContext;
+@@ -87,6 +92,8 @@
+ import org.apache.pig.newplan.logical.relational.LOStore;
+ import org.apache.pig.newplan.logical.relational.LogToPhyTranslationVisitor;
+ import org.apache.pig.newplan.logical.relational.LogicalPlan;
++import org.apache.pig.newplan.logical.relational.LogicalSchema;
++import org.apache.pig.newplan.logical.relational.LogicalSchema.LogicalFieldSchema;
+ import org.apache.pig.newplan.logical.optimizer.UidResetter;
+ import org.apache.pig.newplan.logical.rules.LoadStoreFuncDupSignatureValidator;
+ import org.apache.pig.newplan.logical.visitor.CastLineageSetter;
+@@ -1038,7 +1045,86 @@
+         }
+     }
+ 
+-  
++    
++    static public void checkQueryOutputsAfterSort(Iterator<Tuple> actualResultsIt, 
++            Tuple[] expectedResArray) {
++        List<Tuple> list = new ArrayList<Tuple>();
++        Collections.addAll(list, expectedResArray);
++        checkQueryOutputsAfterSort(actualResultsIt, list);
++    }
++     
++    
++    static private void convertBagToSortedBag(Tuple t) {
++        for (int i=0;i<t.size();i++) {
++           Object obj = null;
++           try {
++               obj = t.get(i);
++           } catch (ExecException e) {
++               // shall not happen
++           }
++           if (obj instanceof DataBag) {
++                DataBag bag = (DataBag)obj;
++                Iterator<Tuple> iter = bag.iterator();
++                DataBag sortedBag = DefaultBagFactory.getInstance().newSortedBag(null);
++                while (iter.hasNext()) {
++                    Tuple t2 = iter.next();
++                    sortedBag.add(t2);
++                    convertBagToSortedBag(t2);
++                }
++                try {
++                    t.set(i, sortedBag);
++                } catch (ExecException e) {
++                    // shall not happen
++                }
++           }
++        }
++    }
++    
++    static public void checkQueryOutputsAfterSortRecursive(Iterator<Tuple> actualResultsIt, 
++            String[] expectedResArray, String schemaString) throws IOException {
++        LogicalSchema resultSchema = org.apache.pig.impl.util.Utils.parseSchema(schemaString);
++        checkQueryOutputsAfterSortRecursive(actualResultsIt, expectedResArray, resultSchema);
++    }
++          /**
++     * Helper function to check if the result of a Pig Query is in line with 
++     * expected results. It sorts actual and expected string results before comparison
++     * 
++     * @param actualResultsIt Result of the executed Pig query
++     * @param expectedResArray Expected string results to validate against
++     * @param fs fieldSchema of expecteResArray
++     * @throws IOException 
++     */
++    static public void checkQueryOutputsAfterSortRecursive(Iterator<Tuple> actualResultsIt, 
++            String[] expectedResArray, LogicalSchema schema) throws IOException {
++        LogicalFieldSchema fs = new LogicalFieldSchema("tuple", schema, DataType.TUPLE);
++        ResourceFieldSchema rfs = new ResourceFieldSchema(fs);
++        
++        LoadCaster caster = new Utf8StorageConverter();
++        List<Tuple> actualResList = new ArrayList<Tuple>();
++        while(actualResultsIt.hasNext()){
++            actualResList.add(actualResultsIt.next());
++        }
++        
++        List<Tuple> expectedResList = new ArrayList<Tuple>();
++        for (String str : expectedResArray) {
++            Tuple newTuple = caster.bytesToTuple(str.getBytes(), rfs);
++            expectedResList.add(newTuple);
++        }
++        
++        for (Tuple t : actualResList) {
++            convertBagToSortedBag(t);
++        }
++        
++        for (Tuple t : expectedResList) {
++            convertBagToSortedBag(t);
++        }
++        
++        Collections.sort(actualResList);
++        Collections.sort(expectedResList);
++        
++        Assert.assertEquals("Comparing actual and expected results. ",
++                expectedResList, actualResList);
++    }
+ 
+     
+ }
+Index: test/org/apache/pig/test/TestCommit.java
+===================================================================
+--- test/org/apache/pig/test/TestCommit.java	(revision 1190473)
++++ test/org/apache/pig/test/TestCommit.java	(working copy)
+@@ -144,22 +144,21 @@
+         pigServer.registerQuery("y = foreach x generate age, cnt, max, min;");
+         Iterator<Tuple> iter = pigServer.openIterator("y");
+         int count = 0;
++        boolean contain1=false, contain2=false;
+         while(iter.hasNext()){
+             Tuple t = iter.next();
+             count++;
+-            if (count == 1) {
+-                assertTrue(t.get(0).equals(expected1.get(0)));
+-                assertTrue(t.get(1).equals(expected1.get(1)));
+-                assertTrue(t.get(2).equals(expected1.get(2)));
+-                assertTrue(t.get(3).equals(expected1.get(3)));
+-            } else if (count == 2){
+-                assertTrue(t.get(0).equals(expected2.get(0)));
+-                assertTrue(t.get(1).equals(expected2.get(1)));
+-                assertTrue(t.get(2).equals(expected2.get(2)));
+-                assertTrue(t.get(3).equals(expected2.get(3)));
++            if (t.get(0).equals(expected1.get(0)) && t.get(1).equals(expected1.get(1)) && t.get(2).equals(expected1.get(2)) && t.get(3).equals(expected1.get(3))) {
++                contain1 = true;
+             }
++            
++            if (t.get(0).equals(expected2.get(0)) && t.get(1).equals(expected2.get(1)) && t.get(2).equals(expected2.get(2)) && t.get(3).equals(expected2.get(3))) {
++                contain2 = true;
++            }
++            
+         }
+         assertEquals(count, 2);
++        assertTrue(contain1 && contain2);
+         Util.deleteFile(cluster, "testCheckin2-input.txt");
+     }    
+ }
+Index: shims/test/hadoop23/org/apache/pig/test/MiniCluster.java
+===================================================================
+--- shims/test/hadoop23/org/apache/pig/test/MiniCluster.java	(revision 1190473)
++++ shims/test/hadoop23/org/apache/pig/test/MiniCluster.java	(working copy)
+@@ -92,15 +92,18 @@
+ 				e.printStackTrace();
+ 			}
+ 			*/
+-			m_dfs.getFileSystem().copyFromLocalFile(new Path("file:///hadoop-mapreduce-client-app-1.0-SNAPSHOT.jar"), new Path("/hadoop-mapreduce-client-app-1.0-SNAPSHOT.jar"));
+-			m_dfs.getFileSystem().copyFromLocalFile(new Path("file:///hadoop-mapreduce-client-jobclient-1.0-SNAPSHOT.jar"), new Path("/hadoop-mapreduce-client-jobclient-1.0-SNAPSHOT.jar"));
++			m_dfs.getFileSystem().copyFromLocalFile(new Path("file:///hadoop-mapreduce-client-app-0.23.0-SNAPSHOT.jar"), new Path("/hadoop-mapreduce-client-app-0.23.0-SNAPSHOT.jar"));
++			m_dfs.getFileSystem().copyFromLocalFile(new Path("file:///hadoop-mapreduce-client-jobclient-0.23.0-SNAPSHOT.jar"), new Path("/hadoop-mapreduce-client-jobclient-0.23.0-SNAPSHOT.jar"));
+ 			m_dfs.getFileSystem().copyFromLocalFile(new Path("file:///pig.jar"), new Path("/pig.jar"));
+ 			m_dfs.getFileSystem().copyFromLocalFile(new Path("file:///pig-test.jar"), new Path("/pig-test.jar"));
+ 
+-            DistributedCache.addFileToClassPath(new Path("/hadoop-mapreduce-client-app-1.0-SNAPSHOT.jar"), m_conf);
++            DistributedCache.addFileToClassPath(new Path("/hadoop-mapreduce-client-app-0.23.0-SNAPSHOT.jar"), m_conf);
+             DistributedCache.addFileToClassPath(new Path("/pig.jar"), m_conf);
+             DistributedCache.addFileToClassPath(new Path("/pig-test.jar"), m_conf);
+-            DistributedCache.addFileToClassPath(new Path("/hadoop-mapreduce-client-jobclient-1.0-SNAPSHOT.jar"), m_conf);
++            DistributedCache.addFileToClassPath(new Path("/hadoop-mapreduce-client-jobclient-0.23.0-SNAPSHOT.jar"), m_conf);
++            String cachefile = m_conf.get("mapreduce.job.cache.files");
++            m_conf.set("alternative.mapreduce.job.cache.files", cachefile);
++            m_conf.unset("mapreduce.job.cache.files");
+ 
+             //ConfigurationUtil.mergeConf(m_conf, m_dfs_conf);
+             //ConfigurationUtil.mergeConf(m_conf, m_mr_conf);
+Index: shims/src/hadoop23/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigMapReduce.java
+===================================================================
+--- shims/src/hadoop23/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigMapReduce.java	(revision 1190473)
++++ shims/src/hadoop23/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigMapReduce.java	(working copy)
+@@ -27,10 +27,12 @@
+ import java.util.Iterator;
+ import java.util.List;
+ 
++import org.apache.hadoop.classification.InterfaceStability;
+ import org.apache.hadoop.conf.Configuration;
+ import org.apache.hadoop.conf.Configuration.IntegerRanges;
+ import org.apache.hadoop.fs.Path;
+ import org.apache.hadoop.io.RawComparator;
++import org.apache.hadoop.io.Text;
+ import org.apache.hadoop.io.Writable;
+ import org.apache.hadoop.mapred.jobcontrol.Job;
+ import org.apache.hadoop.mapreduce.Counter;
+@@ -40,17 +42,41 @@
+ import org.apache.hadoop.mapreduce.OutputCommitter;
+ import org.apache.hadoop.mapreduce.OutputFormat;
+ import org.apache.hadoop.mapreduce.Partitioner;
++import org.apache.hadoop.mapreduce.ReduceContext;
+ import org.apache.hadoop.mapreduce.Reducer;
+ import org.apache.hadoop.mapreduce.TaskAttemptID;
+ import org.apache.hadoop.mapreduce.Reducer.Context;
++import org.apache.hadoop.mapreduce.lib.reduce.WrappedReducer;
++import org.apache.hadoop.mapreduce.task.ReduceContextImpl;
+ import org.apache.hadoop.security.Credentials;
++import org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.PigMapBase.IllustratorContext;
+ import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POPackage;
++import org.apache.pig.data.Tuple;
+ import org.apache.pig.impl.io.NullableTuple;
+ import org.apache.pig.impl.io.PigNullableWritable;
+ import org.apache.pig.impl.util.Pair;
+ import org.apache.pig.pen.FakeRawKeyValueIterator;
+ 
+ public class PigMapReduce extends PigGenericMapReduce {
++    
++    static class IllustrateReducerContext extends WrappedReducer<PigNullableWritable, NullableTuple, PigNullableWritable, Writable> {
++        public IllustratorContext 
++        getReducerContext(ReduceContext<PigNullableWritable, NullableTuple, PigNullableWritable, Writable> reduceContext) {
++            return new IllustratorContext(reduceContext);
++        }
++        
++        public class IllustratorContext 
++            extends WrappedReducer.Context {
++            public IllustratorContext(
++                    ReduceContext<PigNullableWritable, NullableTuple, PigNullableWritable, Writable> reduceContext) {
++                super(reduceContext);
++            }
++            public POPackage getPack() {
++                return ((Reduce.IllustratorContextImpl)reduceContext).pack;
++            }
++        }
++    }
++		  
+     public static class Reduce extends PigGenericMapReduce.Reduce {
+         /**
+          * Get reducer's illustrator context
+@@ -64,11 +90,13 @@
+         @Override
+         public Context getIllustratorContext(Job job,
+                List<Pair<PigNullableWritable, Writable>> input, POPackage pkg) throws IOException, InterruptedException {
+-            return new IllustratorContext(job, input, pkg);
++        	org.apache.hadoop.mapreduce.Reducer.Context reducerContext = new IllustrateReducerContext()
++        			.getReducerContext(new IllustratorContextImpl(job, input, pkg));
++        	return reducerContext;
+         }
+         
+         @SuppressWarnings("unchecked")
+-        public class IllustratorContext extends Context {
++        public class IllustratorContextImpl extends ReduceContextImpl<PigNullableWritable, NullableTuple, PigNullableWritable, Writable> {
+             private PigNullableWritable currentKey = null, nextKey = null;
+             private NullableTuple nextValue = null;
+             private List<NullableTuple> currentValues = null;
+@@ -76,12 +104,15 @@
+             private final ByteArrayOutputStream bos;
+             private final DataOutputStream dos;
+             private final RawComparator sortComparator, groupingComparator;
+-            POPackage pack = null;
++            public POPackage pack = null;
++            private IllustratorValueIterable iterable = new IllustratorValueIterable();
+ 
+-            public IllustratorContext(Job job,
++            public IllustratorContextImpl(Job job,
+                   List<Pair<PigNullableWritable, Writable>> input,
+                   POPackage pkg
+                   ) throws IOException, InterruptedException {
++                super(job.getJobConf(), new TaskAttemptID(), new FakeRawKeyValueIterator(input.iterator().hasNext()),
++                    null, null, null, null, null, null, PigNullableWritable.class, NullableTuple.class);
+                 bos = new ByteArrayOutputStream();
+                 dos = new DataOutputStream(bos);
+                 org.apache.hadoop.mapreduce.Job nwJob = new org.apache.hadoop.mapreduce.Job(job.getJobConf());
+@@ -116,6 +147,60 @@
+                 pack = pkg;
+             }
+             
++            public class IllustratorValueIterator implements ReduceContext.ValueIterator<NullableTuple> {
++                
++                private int pos = -1;
++                private int mark = -1;
++
++                @Override
++                public void mark() throws IOException {
++                    mark=pos-1;
++                    if (mark<-1)
++                        mark=-1;
++                }
++
++                @Override
++                public void reset() throws IOException {
++                    pos=mark;
++                }
++
++                @Override
++                public void clearMark() throws IOException {
++                    mark=-1;
++                }
++
++                @Override
++                public boolean hasNext() {
++                    return pos<currentValues.size()-1;
++                }
++
++                @Override
++                public NullableTuple next() {
++                    pos++;
++                    return currentValues.get(pos);
++                }
++
++                @Override
++                public void remove() {
++                    throw new UnsupportedOperationException("remove not implemented");
++                }
++
++                @Override
++                public void resetBackupStore() throws IOException {
++                    pos=-1;
++                    mark=-1;
++                }
++                
++            }
++            
++            protected class IllustratorValueIterable implements Iterable<NullableTuple> {
++                private IllustratorValueIterator iterator = new IllustratorValueIterator();
++                @Override
++                public Iterator<NullableTuple> iterator() {
++                    return iterator;
++                } 
++            }
++            
+             @Override
+             public PigNullableWritable getCurrentKey() {
+                 return currentKey;
+@@ -160,7 +245,7 @@
+             
+             @Override
+             public Iterable<NullableTuple> getValues() {
+-                return currentValues;
++                return iterable;
+             }
+             
+             @Override
+@@ -170,272 +255,16 @@
+             @Override
+             public void progress() { 
+             }
++        }
+ 
+-            @Override
+-            public Counter getCounter(Enum<?> arg0) {
+-                // TODO Auto-generated method stub
+-                return null;
+-            }
++        @Override
++        public boolean inIllustrator(org.apache.hadoop.mapreduce.Reducer.Context context) {
++            return (context instanceof PigMapReduce.IllustrateReducerContext.IllustratorContext);
++        }
+ 
+-            @Override
+-            public Counter getCounter(String arg0, String arg1) {
+-                // TODO Auto-generated method stub
+-                return null;
+-            }
+-
+-            @Override
+-            public NullableTuple getCurrentValue() throws IOException,
+-                    InterruptedException {
+-                // TODO Auto-generated method stub
+-                return null;
+-            }
+-
+-            @Override
+-            public OutputCommitter getOutputCommitter() {
+-                // TODO Auto-generated method stub
+-                return null;
+-            }
+-
+-            @Override
+-            public boolean nextKeyValue() throws IOException,
+-                    InterruptedException {
+-                // TODO Auto-generated method stub
+-                return false;
+-            }
+-
+-            @Override
+-            public String getStatus() {
+-                // TODO Auto-generated method stub
+-                return null;
+-            }
+-
+-            @Override
+-            public TaskAttemptID getTaskAttemptID() {
+-                // TODO Auto-generated method stub
+-                return null;
+-            }
+-
+-            @Override
+-            public void setStatus(String arg0) {
+-                // TODO Auto-generated method stub
+-                
+-            }
+-
+-            @Override
+-            public Path[] getArchiveClassPaths() {
+-                // TODO Auto-generated method stub
+-                return null;
+-            }
+-
+-            @Override
+-            public long[] getArchiveTimestamps() {
+-                // TODO Auto-generated method stub
+-                return null;
+-            }
+-
+-            @Override
+-            public URI[] getCacheArchives() throws IOException {
+-                // TODO Auto-generated method stub
+-                return null;
+-            }
+-
+-            @Override
+-            public URI[] getCacheFiles() throws IOException {
+-                // TODO Auto-generated method stub
+-                return null;
+-            }
+-
+-            @Override
+-            public Class<? extends Reducer<?, ?, ?, ?>> getCombinerClass()
+-                    throws ClassNotFoundException {
+-                // TODO Auto-generated method stub
+-                return null;
+-            }
+-
+-            @Override
+-            public Configuration getConfiguration() {
+-                // TODO Auto-generated method stub
+-                return null;
+-            }
+-
+-            @Override
+-            public Credentials getCredentials() {
+-                // TODO Auto-generated method stub
+-                return null;
+-            }
+-
+-            @Override
+-            public Path[] getFileClassPaths() {
+-                // TODO Auto-generated method stub
+-                return null;
+-            }
+-
+-            @Override
+-            public long[] getFileTimestamps() {
+-                // TODO Auto-generated method stub
+-                return null;
+-            }
+-
+-            @Override
+-            public RawComparator<?> getGroupingComparator() {
+-                // TODO Auto-generated method stub
+-                return null;
+-            }
+-
+-            @Override
+-            public Class<? extends InputFormat<?, ?>> getInputFormatClass()
+-                    throws ClassNotFoundException {
+-                // TODO Auto-generated method stub
+-                return null;
+-            }
+-
+-            @Override
+-            public String getJar() {
+-                // TODO Auto-generated method stub
+-                return null;
+-            }
+-
+-            @Override
+-            public JobID getJobID() {
+-                // TODO Auto-generated method stub
+-                return null;
+-            }
+-
+-            @Override
+-            public String getJobName() {
+-                // TODO Auto-generated method stub
+-                return null;
+-            }
+-
+-            @Override
+-            public boolean getJobSetupCleanupNeeded() {
+-                // TODO Auto-generated method stub
+-                return false;
+-            }
+-
+-            @Override
+-            public Path[] getLocalCacheArchives() throws IOException {
+-                // TODO Auto-generated method stub
+-                return null;
+-            }
+-
+-            @Override
+-            public Path[] getLocalCacheFiles() throws IOException {
+-                // TODO Auto-generated method stub
+-                return null;
+-            }
+-
+-            @Override
+-            public Class<?> getMapOutputKeyClass() {
+-                // TODO Auto-generated method stub
+-                return null;
+-            }
+-
+-            @Override
+-            public Class<?> getMapOutputValueClass() {
+-                // TODO Auto-generated method stub
+-                return null;
+-            }
+-
+-            @Override
+-            public Class<? extends Mapper<?, ?, ?, ?>> getMapperClass()
+-                    throws ClassNotFoundException {
+-                // TODO Auto-generated method stub
+-                return null;
+-            }
+-
+-            @Override
+-            public int getMaxMapAttempts() {
+-                // TODO Auto-generated method stub
+-                return 0;
+-            }
+-
+-            @Override
+-            public int getMaxReduceAttempts() {
+-                // TODO Auto-generated method stub
+-                return 0;
+-            }
+-
+-            @Override
+-            public int getNumReduceTasks() {
+-                // TODO Auto-generated method stub
+-                return 0;
+-            }
+-
+-            @Override
+-            public Class<? extends OutputFormat<?, ?>> getOutputFormatClass()
+-                    throws ClassNotFoundException {
+-                // TODO Auto-generated method stub
+-                return null;
+-            }
+-
+-            @Override
+-            public Class<?> getOutputKeyClass() {
+-                // TODO Auto-generated method stub
+-                return null;
+-            }
+-
+-            @Override
+-            public Class<?> getOutputValueClass() {
+-                // TODO Auto-generated method stub
+-                return null;
+-            }
+-
+-            @Override
+-            public Class<? extends Partitioner<?, ?>> getPartitionerClass()
+-                    throws ClassNotFoundException {
+-                // TODO Auto-generated method stub
+-                return null;
+-            }
+-
+-            @Override
+-            public boolean getProfileEnabled() {
+-                // TODO Auto-generated method stub
+-                return false;
+-            }
+-
+-            @Override
+-            public String getProfileParams() {
+-                // TODO Auto-generated method stub
+-                return null;
+-            }
+-
+-            @Override
+-            public IntegerRanges getProfileTaskRange(boolean arg0) {
+-                // TODO Auto-generated method stub
+-                return null;
+-            }
+-
+-            @Override
+-            public Class<? extends Reducer<?, ?, ?, ?>> getReducerClass()
+-                    throws ClassNotFoundException {
+-                // TODO Auto-generated method stub
+-                return null;
+-            }
+-
+-            @Override
+-            public RawComparator<?> getSortComparator() {
+-                // TODO Auto-generated method stub
+-                return null;
+-            }
+-
+-            @Override
+-            public boolean getSymlink() {
+-                // TODO Auto-generated method stub
+-                return false;
+-            }
+-
+-            @Override
+-            public String getUser() {
+-                // TODO Auto-generated method stub
+-                return null;
+-            }
+-
+-            @Override
+-            public Path getWorkingDirectory() throws IOException {
+-                // TODO Auto-generated method stub
+-                return null;
+-            }
++        @Override
++        public POPackage getPack(org.apache.hadoop.mapreduce.Reducer.Context context) {
++            return ((PigMapReduce.IllustrateReducerContext.IllustratorContext) context).getPack();
+         }
+     }
+ }
+Index: shims/src/hadoop23/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigMapBase.java
+===================================================================
+--- shims/src/hadoop23/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigMapBase.java	(revision 1190473)
++++ shims/src/hadoop23/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigMapBase.java	(working copy)
+@@ -19,6 +19,7 @@
+ 
+ 
+ import java.io.IOException;
++
+ import java.net.URI;
+ import java.util.Iterator;
+ import java.util.List;
+@@ -37,14 +38,21 @@
+ import org.apache.hadoop.mapreduce.OutputCommitter;
+ import org.apache.hadoop.mapreduce.OutputFormat;
+ import org.apache.hadoop.mapreduce.Partitioner;
++import org.apache.hadoop.mapreduce.RecordReader;
++import org.apache.hadoop.mapreduce.RecordWriter;
+ import org.apache.hadoop.mapreduce.Reducer;
++import org.apache.hadoop.mapreduce.StatusReporter;
+ import org.apache.hadoop.mapreduce.TaskAttemptID;
++import org.apache.hadoop.mapreduce.Reducer.Context;
++import org.apache.hadoop.mapreduce.lib.map.WrappedMapper;
++import org.apache.hadoop.mapreduce.task.MapContextImpl;
+ import org.apache.hadoop.security.Credentials;
+ import org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.PigGenericMapBase;
+ import org.apache.pig.data.DataBag;
+ import org.apache.pig.data.Tuple;
+ import org.apache.pig.impl.io.PigNullableWritable;
+ import org.apache.pig.impl.util.Pair;
++import org.apache.hadoop.mapreduce.lib.map.WrappedMapper;
+ 
+ abstract public class PigMapBase extends PigGenericMapBase {
+     /**
+@@ -63,10 +71,11 @@
+     public Context getIllustratorContext(Configuration conf, DataBag input,
+           List<Pair<PigNullableWritable, Writable>> output, InputSplit split)
+           throws IOException, InterruptedException {
+-        return new IllustratorContext(conf, input, output, split);
++    	org.apache.hadoop.mapreduce.Mapper.Context mapperContext = new WrappedMapper<Text, Tuple, PigNullableWritable, Writable>().getMapContext(new IllustratorContext(conf, input, output, split));
++        return mapperContext;
+     }
+     
+-    public class IllustratorContext extends Context {
++    public class IllustratorContext extends MapContextImpl<Text, Tuple, PigNullableWritable, Writable> {
+         private DataBag input;
+         List<Pair<PigNullableWritable, Writable>> output;
+         private Iterator<Tuple> it = null;
+@@ -76,11 +85,12 @@
+         public IllustratorContext(Configuration conf, DataBag input,
+               List<Pair<PigNullableWritable, Writable>> output,
+               InputSplit split) throws IOException, InterruptedException {
+-              if (output == null)
+-                  throw new IOException("Null output can not be used");
+-              this.input = input; this.output = output;
++            super(conf, new TaskAttemptID(), null, null, null, null, split);
++            if (output == null)
++                throw new IOException("Null output can not be used");
++            this.input = input; this.output = output;
+         }
+-        
++
+         @Override
+         public boolean nextKeyValue() throws IOException, InterruptedException {
+             if (input == null) {
+@@ -118,263 +128,10 @@
+         public void progress() {
+           
+         }
+-
+-        @Override
+-        public InputSplit getInputSplit() {
+-            // TODO Auto-generated method stub
+-            return null;
+-        }
+-
+-        @Override
+-        public Counter getCounter(Enum<?> arg0) {
+-            // TODO Auto-generated method stub
+-            return null;
+-        }
+-
+-        @Override
+-        public Counter getCounter(String arg0, String arg1) {
+-            // TODO Auto-generated method stub
+-            return null;
+-        }
+-
+-        @Override
+-        public OutputCommitter getOutputCommitter() {
+-            // TODO Auto-generated method stub
+-            return null;
+-        }
+-
+-        @Override
+-        public String getStatus() {
+-            // TODO Auto-generated method stub
+-            return null;
+-        }
+-
+-        @Override
+-        public TaskAttemptID getTaskAttemptID() {
+-            // TODO Auto-generated method stub
+-            return null;
+-        }
+-
+-        @Override
+-        public void setStatus(String arg0) {
+-            // TODO Auto-generated method stub
+-            
+-        }
+-
+-        @Override
+-        public Path[] getArchiveClassPaths() {
+-            // TODO Auto-generated method stub
+-            return null;
+-        }
+-
+-        @Override
+-        public long[] getArchiveTimestamps() {
+-            // TODO Auto-generated method stub
+-            return null;
+-        }
+-
+-        @Override
+-        public URI[] getCacheArchives() throws IOException {
+-            // TODO Auto-generated method stub
+-            return null;
+-        }
+-
+-        @Override
+-        public URI[] getCacheFiles() throws IOException {
+-            // TODO Auto-generated method stub
+-            return null;
+-        }
+-
+-        @Override
+-        public Class<? extends Reducer<?, ?, ?, ?>> getCombinerClass()
+-                throws ClassNotFoundException {
+-            // TODO Auto-generated method stub
+-            return null;
+-        }
+-
+-        @Override
+-        public Configuration getConfiguration() {
+-            // TODO Auto-generated method stub
+-            return null;
+-        }
+-
+-        @Override
+-        public Credentials getCredentials() {
+-            // TODO Auto-generated method stub
+-            return null;
+-        }
+-
+-        @Override
+-        public Path[] getFileClassPaths() {
+-            // TODO Auto-generated method stub
+-            return null;
+-        }
+-
+-        @Override
+-        public long[] getFileTimestamps() {
+-            // TODO Auto-generated method stub
+-            return null;
+-        }
+-
+-        @Override
+-        public RawComparator<?> getGroupingComparator() {
+-            // TODO Auto-generated method stub
+-            return null;
+-        }
+-
+-        @Override
+-        public Class<? extends InputFormat<?, ?>> getInputFormatClass()
+-                throws ClassNotFoundException {
+-            // TODO Auto-generated method stub
+-            return null;
+-        }
+-
+-        @Override
+-        public String getJar() {
+-            // TODO Auto-generated method stub
+-            return null;
+-        }
+-
+-        @Override
+-        public JobID getJobID() {
+-            // TODO Auto-generated method stub
+-            return null;
+-        }
+-
+-        @Override
+-        public String getJobName() {
+-            // TODO Auto-generated method stub
+-            return null;
+-        }
+-
+-        @Override
+-        public boolean getJobSetupCleanupNeeded() {
+-            // TODO Auto-generated method stub
+-            return false;
+-        }
+-
+-        @Override
+-        public Path[] getLocalCacheArchives() throws IOException {
+-            // TODO Auto-generated method stub
+-            return null;
+-        }
+-
+-        @Override
+-        public Path[] getLocalCacheFiles() throws IOException {
+-            // TODO Auto-generated method stub
+-            return null;
+-        }
+-
+-        @Override
+-        public Class<?> getMapOutputKeyClass() {
+-            // TODO Auto-generated method stub
+-            return null;
+-        }
+-
+-        @Override
+-        public Class<?> getMapOutputValueClass() {
+-            // TODO Auto-generated method stub
+-            return null;
+-        }
+-
+-        @Override
+-        public Class<? extends Mapper<?, ?, ?, ?>> getMapperClass()
+-                throws ClassNotFoundException {
+-            // TODO Auto-generated method stub
+-            return null;
+-        }
+-
+-        @Override
+-        public int getMaxMapAttempts() {
+-            // TODO Auto-generated method stub
+-            return 0;
+-        }
+-
+-        @Override
+-        public int getMaxReduceAttempts() {
+-            // TODO Auto-generated method stub
+-            return 0;
+-        }
+-
+-        @Override
+-        public int getNumReduceTasks() {
+-            // TODO Auto-generated method stub
+-            return 0;
+-        }
+-
+-        @Override
+-        public Class<? extends OutputFormat<?, ?>> getOutputFormatClass()
+-                throws ClassNotFoundException {
+-            // TODO Auto-generated method stub
+-            return null;
+-        }
+-
+-        @Override
+-        public Class<?> getOutputKeyClass() {
+-            // TODO Auto-generated method stub
+-            return null;
+-        }
+-
+-        @Override
+-        public Class<?> getOutputValueClass() {
+-            // TODO Auto-generated method stub
+-            return null;
+-        }
+-
+-        @Override
+-        public Class<? extends Partitioner<?, ?>> getPartitionerClass()
+-                throws ClassNotFoundException {
+-            // TODO Auto-generated method stub
+-            return null;
+-        }
+-
+-        @Override
+-        public boolean getProfileEnabled() {
+-            // TODO Auto-generated method stub
+-            return false;
+-        }
+-
+-        @Override
+-        public String getProfileParams() {
+-            // TODO Auto-generated method stub
+-            return null;
+-        }
+-
+-        @Override
+-        public IntegerRanges getProfileTaskRange(boolean arg0) {
+-            // TODO Auto-generated method stub
+-            return null;
+-        }
+-
+-        @Override
+-        public Class<? extends Reducer<?, ?, ?, ?>> getReducerClass()
+-                throws ClassNotFoundException {
+-            // TODO Auto-generated method stub
+-            return null;
+-        }
+-
+-        @Override
+-        public RawComparator<?> getSortComparator() {
+-            // TODO Auto-generated method stub
+-            return null;
+-        }
+-
+-        @Override
+-        public boolean getSymlink() {
+-            // TODO Auto-generated method stub
+-            return false;
+-        }
+-
+-        @Override
+-        public String getUser() {
+-            // TODO Auto-generated method stub
+-            return null;
+-        }
+-
+-        @Override
+-        public Path getWorkingDirectory() throws IOException {
+-            // TODO Auto-generated method stub
+-            return null;
+-        }
+     }
++    
++    @Override
++    public boolean inIllustrator(Context context) {
++        return (context instanceof WrappedMapper.Context);
++    }
+ }
+Index: shims/src/hadoop23/org/apache/pig/backend/hadoop/executionengine/shims/HadoopShims.java
+===================================================================
+--- shims/src/hadoop23/org/apache/pig/backend/hadoop/executionengine/shims/HadoopShims.java	(revision 1190473)
++++ shims/src/hadoop23/org/apache/pig/backend/hadoop/executionengine/shims/HadoopShims.java	(working copy)
+@@ -56,4 +56,10 @@
+         
+         return false;
+     }
++    
++    static public TaskAttemptID getNewTaskAttemptID() {
++        TaskAttemptID taskAttemptID = new TaskAttemptID("", 1, TaskType.MAP, 
++                1, 1);
++        return taskAttemptID;
++    }
+ }
+Index: shims/src/hadoop20/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigMapReduce.java
+===================================================================
+--- shims/src/hadoop20/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigMapReduce.java	(revision 1190473)
++++ shims/src/hadoop20/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigMapReduce.java	(working copy)
+@@ -160,5 +160,17 @@
+             public void progress() { 
+             }
+         }
++
++        @Override
++        public boolean inIllustrator(
++                org.apache.hadoop.mapreduce.Reducer.Context context) {
++            return (context instanceof PigMapReduce.Reduce.IllustratorContext);
++        }
++
++        @Override
++        public POPackage getPack(
++                org.apache.hadoop.mapreduce.Reducer.Context context) {
++            return ((PigMapReduce.Reduce.IllustratorContext) context).pack;
++        }
+     }
+ }
+Index: shims/src/hadoop20/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigMapBase.java
+===================================================================
+--- shims/src/hadoop20/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigMapBase.java	(revision 1190473)
++++ shims/src/hadoop20/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigMapBase.java	(working copy)
+@@ -29,6 +29,7 @@
+ import org.apache.hadoop.mapreduce.InputSplit;
+ import org.apache.hadoop.mapreduce.StatusReporter;
+ import org.apache.hadoop.mapreduce.TaskAttemptID;
++import org.apache.hadoop.mapreduce.Mapper.Context;
+ import org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.PigGenericMapBase;
+ import org.apache.pig.data.DataBag;
+ import org.apache.pig.data.Tuple;
+@@ -55,7 +56,10 @@
+         return new IllustratorContext(conf, input, output, split);
+     }
+     
+-    
++    @Override
++    public boolean inIllustrator(Context context) {
++        return (context instanceof PigMapBase.IllustratorContext);
++    }
+ 
+     /**
+      * Dummy implementation of StatusReporter for illustrate mode
+Index: shims/src/hadoop20/org/apache/pig/backend/hadoop/executionengine/shims/HadoopShims.java
+===================================================================
+--- shims/src/hadoop20/org/apache/pig/backend/hadoop/executionengine/shims/HadoopShims.java	(revision 1190473)
++++ shims/src/hadoop20/org/apache/pig/backend/hadoop/executionengine/shims/HadoopShims.java	(working copy)
+@@ -58,4 +58,8 @@
+     static public boolean isMap(TaskAttemptID taskAttemptID) {
+         return taskAttemptID.isMap();
+     }
++    
++    static public TaskAttemptID getNewTaskAttemptID() {
++        return new TaskAttemptID();
++    }
+ }
+Index: src/META-INF/services/org.apache.hadoop.mapreduce.protocol.ClientProtocolProvider
+===================================================================
+--- src/META-INF/services/org.apache.hadoop.mapreduce.protocol.ClientProtocolProvider	(revision 0)
++++ src/META-INF/services/org.apache.hadoop.mapreduce.protocol.ClientProtocolProvider	(revision 0)
+@@ -0,0 +1,17 @@
++#
++#   Licensed under the Apache License, Version 2.0 (the "License");
++#   you may not use this file except in compliance with the License.
++#   You may obtain a copy of the License at
++#
++#       http://www.apache.org/licenses/LICENSE-2.0
++#
++#   Unless required by applicable law or agreed to in writing, software
++#   distributed under the License is distributed on an "AS IS" BASIS,
++#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
++#   See the License for the specific language governing permissions and
++#   limitations under the License.
++#
++org.apache.hadoop.mapred.LocalClientProtocolProvider
++org.apache.hadoop.mapred.YarnClientProtocolProvider
++org.apache.hadoop.mapred.LocalClientProtocolProvider
++org.apache.hadoop.mapred.YarnClientProtocolProvider
+Index: src/org/apache/pig/backend/hadoop/executionengine/HExecutionEngine.java
+===================================================================
+--- src/org/apache/pig/backend/hadoop/executionengine/HExecutionEngine.java	(revision 1190473)
++++ src/org/apache/pig/backend/hadoop/executionengine/HExecutionEngine.java	(working copy)
+@@ -174,6 +174,7 @@
+             jc.addResource("mapred-default.xml");
+             recomputeProperties(jc, properties);
+             
++            properties.setProperty("mapreduce.framework.name", "local");
+             properties.setProperty(JOB_TRACKER_LOCATION, LOCAL );
+             properties.setProperty(FILE_SYSTEM_LOCATION, "file:///");
+             properties.setProperty(ALTERNATIVE_FILE_SYSTEM_LOCATION, "file:///");
+Index: src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigGenericMapReduce.java
+===================================================================
+--- src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigGenericMapReduce.java	(revision 1190473)
++++ src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigGenericMapReduce.java	(working copy)
+@@ -35,6 +35,7 @@
+ import org.apache.hadoop.mapred.jobcontrol.Job;
+ import org.apache.hadoop.mapreduce.Reducer;
+ import org.apache.hadoop.mapreduce.TaskAttemptID;
++import org.apache.hadoop.mapreduce.Mapper.Context;
+ 
+ import org.apache.pig.PigException;
+ import org.apache.pig.backend.executionengine.ExecException;
+@@ -307,9 +308,9 @@
+         @Override
+         protected void setup(Context context) throws IOException, InterruptedException {
+             super.setup(context);
+-            inIllustrator = (context instanceof PigMapReduce.Reduce.IllustratorContext);
++            inIllustrator = inIllustrator(context);
+             if (inIllustrator)
+-                pack = ((PigMapReduce.Reduce.IllustratorContext) context).pack;
++                pack = getPack(context);
+             Configuration jConf = context.getConfiguration();
+             SpillableMemoryManager.configure(ConfigurationUtil.toProperties(jConf));
+             sJobContext = context;
+@@ -550,6 +551,10 @@
+          */
+         abstract public Context getIllustratorContext(Job job,
+                List<Pair<PigNullableWritable, Writable>> input, POPackage pkg) throws IOException, InterruptedException;
++        
++        abstract public boolean inIllustrator(Context context);
++        
++        abstract public POPackage getPack(Context context);
+     }
+     
+     /**
+Index: src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigGenericMapBase.java
+===================================================================
+--- src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigGenericMapBase.java	(revision 1190473)
++++ src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigGenericMapBase.java	(working copy)
+@@ -164,7 +164,7 @@
+         PigMapReduce.sJobContext = context;
+         PigMapReduce.sJobConfInternal.set(context.getConfiguration());
+         PigMapReduce.sJobConf = context.getConfiguration();
+-        inIllustrator = (context instanceof PigMapBase.IllustratorContext);
++        inIllustrator = inIllustrator(context);
+         
+         PigContext.setPackageImportList((ArrayList<String>)ObjectSerializer.deserialize(job.get("udf.import.list")));
+         pigContext = (PigContext)ObjectSerializer.deserialize(job.get("pig.pigContext"));
+@@ -301,6 +301,8 @@
+ 
+     abstract public void collect(Context oc, Tuple tuple) throws InterruptedException, IOException;
+ 
++    abstract public boolean inIllustrator(Context context);
++    
+     /**
+      * @return the keyType
+      */
+Index: src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/JobControlCompiler.java
+===================================================================
+--- src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/JobControlCompiler.java	(revision 1190473)
++++ src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/JobControlCompiler.java	(working copy)
+@@ -703,7 +703,17 @@
+             }
+             if (maxCombinedSplitSize > 0)
+                 conf.setLong("pig.maxCombinedSplitSize", maxCombinedSplitSize);
+-                        
++            
++            // It's a hack to set distributed cache file for hadoop 23. Once MiniMRCluster do not require local
++            // jar on fixed location, this can be removed
++            if (pigContext.getExecType() == ExecType.MAPREDUCE) {
++                String newfiles = conf.get("alternative.mapreduce.job.cache.files");
++                if (newfiles!=null) {
++                    String files = conf.get("mapreduce.job.cache.files");
++                    conf.set("mapreduce.job.cache.files",
++                        files == null ? newfiles.toString() : files + "," + newfiles);
++                }
++            }
+             // Serialize the UDF specific context info.
+             UDFContext.getUDFContext().serialize(conf);
+             Job cjob = new Job(new JobConf(nwJob.getConfiguration()), new ArrayList());
+Index: src/org/apache/pig/backend/hadoop/datastorage/ConfigurationUtil.java
+===================================================================
+--- src/org/apache/pig/backend/hadoop/datastorage/ConfigurationUtil.java	(revision 1190473)
++++ src/org/apache/pig/backend/hadoop/datastorage/ConfigurationUtil.java	(working copy)
+@@ -18,6 +18,8 @@
+ 
+ package org.apache.pig.backend.hadoop.datastorage;
+ 
++import java.lang.reflect.InvocationTargetException;
++import java.lang.reflect.Method;
+ import java.util.Enumeration;
+ import java.util.Iterator;
+ import java.util.Map;
+@@ -73,6 +75,23 @@
+             localConf.addResource("core-default.xml");
+         } else {
+             localConf = new Configuration(true);
++            // It's really hacky, try to get unit test working under hadoop 23.
++            // Hadoop23 MiniMRCluster currently need setup Distributed cache before start, 
++            // so pigtest/conf/hadoop-site.xml contains such entry. This prevents some tests from 
++            // successful (They expect those files in hdfs), so we need to unset it in hadoop 23.
++            // This should go away once MiniMRCluster fix the distributed cache issue.
++            Method unsetMethod = null;
++            try {
++                unsetMethod = localConf.getClass().getMethod("unset", new Class[]{String.class});
++            } catch (Exception e) {
++            }
++            if (unsetMethod!=null) {
++                try {
++                    unsetMethod.invoke(localConf, new Object[]{"mapreduce.job.cache.files"});
++                } catch (Exception e) {
++                    // Should not happen
++                }
++            }
+         }
+         Properties props = ConfigurationUtil.toProperties(localConf);
+         props.setProperty(MapRedUtil.FILE_SYSTEM_NAME, "file:///");
+Index: src/org/apache/pig/tools/pigstats/PigStatusReporter.java
+===================================================================
+--- src/org/apache/pig/tools/pigstats/PigStatusReporter.java	(revision 1190473)
++++ src/org/apache/pig/tools/pigstats/PigStatusReporter.java	(working copy)
+@@ -74,4 +74,7 @@
+         }
+     }
+ 
++    public float getProgress() {
++        return 0;
++    }
+ }
+Index: src/org/apache/pig/impl/io/PigFile.java
+===================================================================
+--- src/org/apache/pig/impl/io/PigFile.java	(revision 1190473)
++++ src/org/apache/pig/impl/io/PigFile.java	(working copy)
+@@ -80,7 +80,8 @@
+         PigOutputFormat.setLocation(jc, store);
+         OutputCommitter oc;
+         // create a simulated TaskAttemptContext
+-        TaskAttemptContext tac = HadoopShims.createTaskAttemptContext(conf, new TaskAttemptID());
++        
++        TaskAttemptContext tac = HadoopShims.createTaskAttemptContext(conf, HadoopShims.getNewTaskAttemptID());
+         PigOutputFormat.setLocation(tac, store);
+         RecordWriter<?,?> rw ;
+         try {
+Index: build.xml
+===================================================================
+--- build.xml	(revision 1190473)
++++ build.xml	(working copy)
+@@ -20,8 +20,6 @@
+ 	xmlns:ivy="antlib:org.apache.ivy.ant">
+     <!-- Load all the default properties, and any the user wants    -->
+     <!-- to contribute (without having to type -D or edit this file -->
+-    <taskdef resource="net/sf/antcontrib/antcontrib.properties"/>
+-
+     <property file="${user.home}/build.properties" />
+     <property file="${basedir}/build.properties" />
+ 
+@@ -68,7 +66,9 @@
+ 
+     <!-- jar names. TODO we might want to use the svn reversion name in the name in case it is a dev version -->
+     <property name="output.jarfile" value="${build.dir}/${final.name}.jar" />
++    <property name="output.stage.jarfile" value="${build.dir}/${final.name}.stage.jar" />
+     <property name="output.jarfile.withouthadoop" value="${build.dir}/${final.name}-withouthadoop.jar" />
++    <property name="output.stage.jarfile.withouthadoop" value="${build.dir}/${final.name}-withouthadoop.stage.jar" />
+     <property name="output.jarfile.core" value="${build.dir}/${final.name}-core.jar" />
+     <property name="output.jarfile.sources" value="${build.dir}/${final.name}-sources.jar" />
+     <property name="output.jarfile.javadoc" value="${build.dir}/${final.name}-javadoc.jar" />
+@@ -158,6 +158,10 @@
+            select="\1" /-->
+     <property name="hadoopversion" value="20" />
+ 
++    <condition property="isHadoop23">
++        <equals arg1="hadoopversion" arg2="23"/>
++    </condition>
++
+     <property name="src.shims.dir" value="${basedir}/shims/src/hadoop${hadoopversion}" />
+     <property name="src.shims.test.dir" value="${basedir}/shims/test/hadoop${hadoopversion}" />
+ 
+@@ -171,7 +175,6 @@
+ 	<property name="ant_task_repo_url"
+ 	value="${mvnrepo}/org/apache/maven/maven-ant-tasks/${ant-task.version}/maven-ant-tasks-${ant-task.version}.jar"/>
+ 	<property name="ivy_repo_url" value="${mvnrepo}/org/apache/ivy/ivy/${ivy.version}/ivy-${ivy.version}.jar"/>
+-    <property name="guava.jar" value="guava-${guava.version}.jar"/>
+     <property name="ivysettings.xml" location="${ivy.dir}/ivysettings.xml" />
+     <property name="ivy.org" value="org.apache.pig"/>
+     <property name="build.dir" location="build" />
+@@ -225,7 +228,6 @@
+     <!-- javadoc-classpath -->
+     <path id="javadoc-classpath">
+ 	<fileset file="${lib.dir}/${automaton.jarfile}" />
+-        <fileset file="${lib.dir}/${guava.jarfile}" />
+ 	<path refid="javadoc.classpath"/>	
+     </path> 
+ 
+@@ -514,9 +516,34 @@
+         </jar>
+     </target>
+ 
++    <!-- ================================================================== -->
++    <!-- Utility to build pig.jar and pig-withouthadoop.jar                 -->
++    <!-- ================================================================== -->
+     <target name="jar-all" depends="jar,jar-withouthadoop" description="Create pig-all.jar and pig-withouthadoop.jar">
+     </target>
+ 
++    <target name="include-meta" if="isHadoop23">
++        <copy todir="${build.classes}/META-INF">
++            <fileset dir="${src.dir}/META-INF" includes="**"/>  
++        </copy>
++        <move file="${output.jarfile}" tofile="${output.stage.jarfile}"/>
++        <sleep seconds="1"/>
++        <jar jarfile="${output.jarfile}" if="isHadoop23">
++            <manifest>
++                <attribute name="Main-Class" value="org.apache.pig.Main" />
++                <section name="org/apache/pig">
++                    <attribute name="Implementation-Vendor" value="Apache" />
++                    <attribute name="Implementation-Title" value="Pig" />
++                    <attribute name="Implementation-Version" value="${version}" />
++                    <attribute name="Build-TimeStamp" value="${timestamp}" />
++                    <attribute name="Svn-Revision" value="${svnString}" />
++                </section>
++            </manifest>
++            <zipfileset src="${output.stage.jarfile}"/>
++            <fileset dir="${build.classes}" includes="META-INF/**" />
++        </jar>
++    </target>
++
+     <!-- ================================================================== -->
+     <!-- Make pig.jar                                                       -->
+     <!-- ================================================================== -->
+@@ -566,24 +593,31 @@
+                     <attribute name="Svn-Revision" value="${svnString}" />
+                 </section>
+             </manifest>
+-            <zipfileset src="${ivy.lib.dir}/hadoop-core-${hadoop-core.version}.jar" />
+             <zipfileset src="${lib.dir}/${automaton.jarfile}" />
+-            <zipfileset src="${ivy.lib.dir}/antlr-runtime-${antlr.version}.jar" />
+-            <zipfileset src="${ivy.lib.dir}/ST4-${stringtemplate.version}.jar" />
+-            <zipfileset src="${ivy.lib.dir}/junit-${junit.version}.jar" />
+-            <zipfileset src="${ivy.lib.dir}/jsch-${jsch.version}.jar" />
+-            <zipfileset src="${ivy.lib.dir}/jline-${jline.version}.jar" />
+-            <zipfileset src="${ivy.lib.dir}/jackson-mapper-asl-${jackson.version}.jar" />
+-            <zipfileset src="${ivy.lib.dir}/jackson-core-asl-${jackson.version}.jar" />
+-            <zipfileset src="${ivy.lib.dir}/joda-time-${joda-time.version}.jar" /> 
+-            <zipfileset src="${ivy.lib.dir}/${guava.jar}" />
+-            <zipgroupfileset dir="${ivy.lib.dir}" includes="hbase-${hbase.version}.jar" /> 
+-            <zipgroupfileset dir="${ivy.lib.dir}" includes="commons*.jar"/>
+-            <zipgroupfileset dir="${ivy.lib.dir}" includes="log4j*.jar"/>
+-            <zipgroupfileset dir="${ivy.lib.dir}" includes="jsp-api*.jar"/>
+-            <zipgroupfileset dir="${ivy.lib.dir}" includes="zookeeper*.jar"/>
++            <!-- This will be removed once we get hadoop-mapreduce.jar from maven-->
++            <zipgroupfileset dir="${lib.dir}" includes="hadoop-*${hadoopversion}*.jar" />
++            <zipgroupfileset dir="${ivy.lib.dir}">
++                <include name="hadoop-core-${hadoop-core.version}.jar" />
++                <include name="hadoop-*-${hadoop-common.version}*.jar" />
++                <include name="antlr-runtime-${antlr.version}.jar" />
++                <include name="ST4-${stringtemplate.version}.jar" />
++                <include name="junit-${junit.version}.jar" />
++                <include name="jsch-${jsch.version}.jar" />
++                <include name="jline-${jline.version}.jar" />
++                <include name="jackson-mapper-asl-${jackson.version}.jar" />
++                <include name="jackson-core-asl-${jackson.version}.jar" />
++                <include name="joda-time-${joda-time.version}.jar" />
++                <include name="guava-${guava.version}.jar" />
++                <include name="avro-${avro.version}.jar"/>
++                <include name="hbase-${hbase.version}.jar" />
++                <include name="commons*.jar"/>
++                <include name="log4j*.jar"/>
++                <include name="jsp-api*.jar"/>
++                <include name="zookeeper*.jar"/>
++            </zipgroupfileset>
+             <fileset file="${basedir}/conf/pig-default.properties" />
+         </jar>
++        <antcall target="include-meta" inheritRefs="true" inheritall="true"/>
+         <copy file="${output.jarfile}" tofile="${output.jarfile.backcompat}"/>
+     </target>
+ 
+@@ -653,12 +687,14 @@
+                 </section>
+             </manifest>
+             <zipfileset src="${lib.dir}/${automaton.jarfile}" />
+-            <zipfileset src="${ivy.lib.dir}/antlr-runtime-${antlr.version}.jar" />
+-            <zipfileset src="${ivy.lib.dir}/ST4-${stringtemplate.version}.jar" />
+-            <zipfileset src="${ivy.lib.dir}/jline-${jline.version}.jar" />
+-            <zipfileset src="${ivy.lib.dir}/jackson-mapper-asl-${jackson.version}.jar" />
+-            <zipfileset src="${ivy.lib.dir}/jackson-core-asl-${jackson.version}.jar" />
+-            <zipfileset src="${ivy.lib.dir}/${guava.jar}" />
++            <zipgroupfileset dir="${ivy.lib.dir}">
++                <include name="antlr-runtime-${antlr.version}.jar" />
++                <include name="ST4-${stringtemplate.version}.jar" />
++                <include name="jline-${jline.version}.jar" />
++                <include name="jackson-mapper-asl-${jackson.version}.jar" />
++                <include name="jackson-core-asl-${jackson.version}.jar" />
++                <include name="guava-${guava.version}.jar" />
++            </zipgroupfileset>
+             <fileset file="${basedir}/conf/pig-default.properties" />
+         </jar>
+         <copy file="${output.jarfile.withouthadoop}" tofile="${output.jarfile.backcompat.withouthadoop}"/>
+Index: ivy.xml
+===================================================================
+--- ivy.xml	(revision 1190473)
++++ ivy.xml	(working copy)
+@@ -31,13 +31,15 @@
+     <conf name="default" extends="master,runtime"/>
+     <conf name="runtime" extends="compile,test" description="runtime but not the artifact" />
+     <!--Private configurations. -->
+-    <conf name="compile" visibility="private" description="compile artifacts"/>
++    <conf name="compile" extends="hadoop${hadoopversion}" visibility="private" description="compile artifacts"/>
+     <conf name="test" extends="compile" visibility="private"/>
+     <conf name="javadoc" visibility="private" extends="compile,test"/>
+     <conf name="releaseaudit" visibility="private"/>
+     <conf name="jdiff" visibility="private"/>
+     <conf name="checkstyle" visibility="private"/>
+     <conf name="buildJar" extends="compile,test" visibility="private"/>
++    <conf name="hadoop20" visibility="private"/>
++    <conf name="hadoop23" visibility="private"/>
+   </configurations>
+   <publications>
+     <!--get the artifact from our module name-->
+@@ -50,14 +52,56 @@
+       conf="checkstyle->master"/> -->
+     <dependency org="commons-beanutils" name="commons-beanutils-core" rev="${commons-beanutils.version}"
+       conf="checkstyle->master"/>
++    <dependency org="commons-codec" name="commons-codec" rev="${commons-codec.version}"
++      conf="hadoop23->master"/>
+     <dependency org="commons-el" name="commons-el" rev="${commons-el.version}"
+       conf="compile->master"/>
++    <dependency org="commons-httpclient" name="commons-httpclient" rev="${commons-httpclient.version}"
++      conf="hadoop23->master"/>
++    <dependency org="commons-configuration" name="commons-configuration" rev="${commons-configuration.version}"
++      conf="hadoop23->master"/>
+     <dependency org="log4j" name="log4j" rev="${log4j.version}"
+       conf="compile->master"/>
+     <dependency org="org.apache.hadoop" name="hadoop-core" rev="${hadoop-core.version}"
+-      conf="compile->default;test->default"/>
++      conf="hadoop20->default"/>
+     <dependency org="org.apache.hadoop" name="hadoop-test" rev="${hadoop-test.version}"
+-      conf="compile->default;test->default"/>
++      conf="hadoop20->default"/>
++    <dependency org="org.apache.hadoop" name="hadoop-annotations" 
++      rev="${hadoop-common.version}" conf="hadoop23->master"/>
++    <dependency org="org.apache.hadoop" name="hadoop-auth" 
++      rev="${hadoop-common.version}" conf="hadoop23->master"/>
++    <dependency org="org.apache.hadoop" name="hadoop-common" 
++      rev="${hadoop-common.version}" conf="hadoop23->master">
++      <artifact name="hadoop-common" ext="jar" />
++      <artifact name="hadoop-common" type="tests" ext="jar" m:classifier="tests" />
++    </dependency>
++    <dependency org="org.apache.hadoop" name="hadoop-hdfs"
++      rev="${hadoop-hdfs.version}" conf="hadoop23->master">
++      <artifact name="hadoop-hdfs" ext="jar" />
++      <artifact name="hadoop-hdfs" type="tests" ext="jar" m:classifier="tests" />
++    </dependency>
++    <dependency org="org.apache.hadoop" name="hadoop-mapreduce-client-core" rev="${hadoop-mapreduce.version}"
++      conf="hadoop23->master"/>
++    <dependency org="org.apache.hadoop" name="hadoop-mapreduce-client-jobclient" rev="${hadoop-mapreduce.version}"
++      conf="hadoop23->master">
++        <artifact name="hadoop-mapreduce-client-jobclient" type="jar" m:classifier="tests"/>
++        <exclude org="commons-daemon" module="commons-daemon"/><!--bad POM-->
++        <exclude org="org.apache.commons" module="commons-daemon"/><!--bad POM-->
++    </dependency>
++    <dependency org="org.apache.hadoop" name="hadoop-yarn-server-tests" rev="${hadoop-mapreduce.version}"
++      conf="hadoop23->master">
++      <artifact name="hadoop-yarn-server-tests" type="jar" m:classifier="tests"/>
++    </dependency>
++    <dependency org="org.apache.hadoop" name="hadoop-mapreduce" 
++      rev="${hadoop-common.version}" conf="hadoop23->master"/>
++    <dependency org="org.apache.hadoop" name="hadoop-yarn-api" 
++      rev="${hadoop-common.version}" conf="hadoop23->master"/>
++    <dependency org="org.apache.hadoop" name="hadoop-yarn-common" 
++      rev="${hadoop-common.version}" conf="hadoop23->master"/>
++    <dependency org="org.apache.hadoop" name="hadoop-yarn-server-nodemanager" 
++      rev="${hadoop-common.version}" conf="hadoop23->master"/>
++    <dependency org="org.apache.hadoop" name="hadoop-yarn-server-resourcemanager" 
++      rev="${hadoop-common.version}" conf="hadoop23->master"/>
+     <dependency org="org.mortbay.jetty" name="jetty" rev="${jetty.version}"
+       conf="compile->master"/>
+     <dependency org="org.mortbay.jetty" name="jetty-util" rev="${jetty-util.version}"
+@@ -67,7 +111,7 @@
+      <dependency org="org.slf4j" name="slf4j-log4j12" rev="${slf4j-log4j12.version}"
+       conf="compile->master;test->master"/>
+     <dependency org="commons-cli" name="commons-cli" rev="${commons-cli.version}"
+-      conf="checkstyle->master"/>
++      conf="compile->master;checkstyle->master"/>
+     
+     <dependency org="org.apache.avro" name="avro" rev="${avro.version}"
+       conf="compile->default;checkstyle->master"/>
+@@ -93,14 +137,14 @@
+     <dependency org="org.codehaus.jackson" name="jackson-core-asl" rev="${jackson.version}"
+       conf="compile->master"/>
+     <dependency org="joda-time" name="joda-time" rev="${joda-time.version}" conf="compile->master"/>
+-  	<dependency org="commons-lang" name="commons-lang" rev="${commons-lang.version}"
++    <dependency org="commons-lang" name="commons-lang" rev="${commons-lang.version}"
+   	  conf="compile->master"/>
+     <dependency org="com.google.guava" name="guava" rev="${guava.version}" conf="compile->master" />
+-  	<dependency org="org.python" name="jython" rev="${jython.version}" conf="compile->master"/>
++    <dependency org="org.python" name="jython" rev="${jython.version}" conf="compile->master"/>
+     <dependency org="rhino" name="js" rev="${rhino.version}" conf="compile->master"/>
+-  	<dependency org="org.antlr" name="antlr" rev="${antlr.version}" conf="compile->master"/>
+-  	<dependency org="org.antlr" name="antlr-runtime" rev="${antlr.version}" conf="compile->default"/>
+-	<dependency org="org.antlr" name="ST4" rev="${stringtemplate.version}" conf="compile->default"/>
++    <dependency org="org.antlr" name="antlr" rev="${antlr.version}" conf="compile->master"/>
++    <dependency org="org.antlr" name="antlr-runtime" rev="${antlr.version}" conf="compile->default"/>
++    <dependency org="org.antlr" name="ST4" rev="${stringtemplate.version}" conf="compile->default"/>
+     <dependency org="org.apache.zookeeper" name="zookeeper" rev="${zookeeper.version}" conf="compile->master"/>
+     <dependency org="org.jboss.netty" name="netty" rev="3.2.2.Final" conf="compile->master"/>
+ 
+Index: ivy/ivysettings.xml
+===================================================================
+--- ivy/ivysettings.xml	(revision 1190473)
++++ ivy/ivysettings.xml	(working copy)
+@@ -31,22 +31,32 @@
+   -->
+   <property name="repo.maven.org" value="${mvnrepo}" override="true"/>
+   <property name="repo.jboss.org" value="http://repository.jboss.com/nexus/content/groups/public/" override="false"/>
+-
++  <property name="repo.apache.snapshots" value="http://repository.apache.org/content/groups/snapshots-group/" override="false"/>
++  <property name="repo.dir" value="${user.home}/.m2/repository"/>
+   <property name="maven2.pattern"  value="[organisation]/[module]/[revision]/[module]-[revision](-[classifier])"/> 
+   <property name="maven2.pattern.ext" value="${maven2.pattern}.[ext]"/>
++  <property name="snapshot.pattern"  value="[organisation]/[module]/[revision]/[artifact]-[revision](-[classifier]).[ext]"/>
++  <property name="resolvers" value="default" override="false"/>
++  <property name="force-resolve" value="false" override="false"/>
+   <!-- pull in the local repository -->
+   <include url="${ivy.default.conf.dir}/ivyconf-local.xml"/>
+-  <settings defaultResolver="default"/>
++  <settings defaultResolver="${resolvers}"/>
+   <resolvers>
+     <ibiblio name="maven2" root="${repo.maven.org}" pattern="${maven2.pattern.ext}" m2compatible="true"/>
+     <ibiblio name="jboss-maven2" root="${repo.jboss.org}" pattern="${maven2.pattern.ext}" m2compatible="true"/>
++    <ibiblio name="apache-snapshots" root="${repo.apache.snapshots}" pattern="${snapshot.pattern}" checkmodified="true" m2compatible="true"/>
++    <filesystem name="fs" m2compatible="true" checkconsistency="false" force="${force-resolve}">
++       <artifact pattern="${repo.dir}/${maven2.pattern.ext}"/>
++       <ivy pattern="${repo.dir}/[organisation]/[module]/[revision]/[module]-[revision].pom"/>
++    </filesystem>
+     <chain name="default" dual="true">
+       <resolver ref="local"/>
+       <resolver ref="maven2"/>
+       <resolver ref="jboss-maven2"/>
++      <resolver ref="apache-snapshots"/>
+     </chain>
+     <chain name="internal">
+-      <resolver ref="local"/>
++      <resolver ref="fs"/>
+     </chain>
+     <chain name="external">
+       <resolver ref="maven2"/>
+Index: ivy/libraries.properties
+===================================================================
+--- ivy/libraries.properties	(revision 1190473)
++++ ivy/libraries.properties	(working copy)
+@@ -18,14 +18,20 @@
+ avro.version=1.5.3
+ commons-beanutils.version=1.7.0
+ commons-cli.version=1.0
++commons-codec.version=1.4
+ commons-el.version=1.0
+ commons-logging.version=1.1.1
+ commons-lang.version=2.4
++commons-configuration.version=1.6
++commons-httpclient.version=3.1
+ checkstyle.version=4.2
+ ivy.version=2.2.0
+ guava.version=r06
+ hadoop-core.version=0.20.2
+ hadoop-test.version=0.20.2
++hadoop-common.version=0.23.0-SNAPSHOT
++hadoop-hdfs.version=0.23.0-SNAPSHOT
++hadoop-mapreduce.version=0.23.0-SNAPSHOT
+ hbase.version=0.90.0
+ hsqldb.version=1.8.0.10
+ jackson.version=1.7.3
+@@ -43,11 +49,11 @@
+ rhino.version=1.7R2
+ antlr.version=3.4
+ stringtemplate.version=4.0.4
+-log4j.version=1.2.14
++log4j.version=1.2.16
+ netty.version=3.2.2
+ rats-lib.version=0.5.1
+-slf4j-api.version=1.4.3
+-slf4j-log4j12.version=1.4.3
++slf4j-api.version=1.6.1
++slf4j-log4j12.version=1.6.1
+ xerces.version=1.4.4
+ wagon-http.version=1.0-beta-2
+ zookeeper.version=3.3.3
+Index: bin/pig
+===================================================================
+--- bin/pig	(revision 1190473)
++++ bin/pig	(working copy)
+@@ -115,7 +115,10 @@
+ shopt -s extglob
+ shopt -s nullglob
+ 
+-for f in $PIG_HOME/lib/*.jar; do
++# exclude hadoop class in lib, this is temporary. We put hadoop-mapreduce.jar
++# into lib, since it is not in maven yet. We shall include all jar in lib once
++# hadoop-mapreduce.jar is in maven
++for f in $PIG_HOME/lib/!(hadoop*).jar; do
+     CLASSPATH=${CLASSPATH}:$f;
+ done
+ 

Modified: incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/deb/pig/rules
URL: http://svn.apache.org/viewvc/incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/deb/pig/rules?rev=1196333&r1=1196332&r2=1196333&view=diff
==============================================================================
--- incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/deb/pig/rules (original)
+++ incubator/bigtop/branches/hadoop-0.23/bigtop-packages/src/deb/pig/rules Tue Nov  1 22:29:24 2011
@@ -25,6 +25,7 @@ export DH_OPTIONS
 
 patch: patch-stamp
 patch-stamp:	
+	patch -p0 < debian/patch
 	touch $@
 
 clean:	



Mime
View raw message