pig-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From cheol...@apache.org
Subject svn commit: r1578718 [2/2] - in /pig/branches/tez: ./ conf/ src/ src/docs/src/documentation/content/xdocs/ src/org/apache/pig/ src/org/apache/pig/backend/hadoop/datastorage/ src/org/apache/pig/backend/hadoop/executionengine/ src/org/apache/pig/backend/...
Date Tue, 18 Mar 2014 02:19:25 GMT
Modified: pig/branches/tez/test/org/apache/pig/test/TestLogicalPlanBuilder.java
URL: http://svn.apache.org/viewvc/pig/branches/tez/test/org/apache/pig/test/TestLogicalPlanBuilder.java?rev=1578718&r1=1578717&r2=1578718&view=diff
==============================================================================
--- pig/branches/tez/test/org/apache/pig/test/TestLogicalPlanBuilder.java (original)
+++ pig/branches/tez/test/org/apache/pig/test/TestLogicalPlanBuilder.java Tue Mar 18 02:19:24
2014
@@ -2168,9 +2168,11 @@ public class TestLogicalPlanBuilder {
         LogicalPlanData lData = pigServer.getLogicalPlanData();
         assertEquals("LoadFunc must be PigStorage", "org.apache.pig.builtin.PigStorage",
lData.getLoadFuncs().get(0));
         assertEquals("StoreFunc must be PigStorageWithSchema", "org.apache.pig.test.PigStorageWithSchema",
lData.getStoreFuncs().get(0));
-        assertEquals("Number of sources must be 2", lData.getSources().size(), 2);
+        assertEquals("Number of sources must be 2", lData.getNumSources(), 2);
+        assertEquals("Number of sinks must be 1", lData.getNumSinks(), 1);
         assertTrue("Source must end with input.txt", lData.getSources().get(0).endsWith("input.txt"));
         assertTrue("Sink must end with output", lData.getSinks().get(0).endsWith("output"));
+        assertEquals("Number of logical relational operators must be 4", lData.getNumLogicalRelationOperators(),
4);
     }
     
     /**

Modified: pig/branches/tez/test/org/apache/pig/test/TestMultiQueryCompiler.java
URL: http://svn.apache.org/viewvc/pig/branches/tez/test/org/apache/pig/test/TestMultiQueryCompiler.java?rev=1578718&r1=1578717&r2=1578718&view=diff
==============================================================================
--- pig/branches/tez/test/org/apache/pig/test/TestMultiQueryCompiler.java (original)
+++ pig/branches/tez/test/org/apache/pig/test/TestMultiQueryCompiler.java Tue Mar 18 02:19:24
2014
@@ -1234,9 +1234,8 @@ public class TestMultiQueryCompiler {
                           + "describe b;"
                           + "store b into '/tmp/output1';\n";
 
-            GruntParser parser = new GruntParser(new StringReader(script));
+            GruntParser parser = new GruntParser(new StringReader(script), myPig);
             parser.setInteractive(false);
-            parser.setParams(myPig);
             parser.parseStopOnError();
 
         } catch (Exception e) {
@@ -1257,9 +1256,8 @@ public class TestMultiQueryCompiler {
                           + "illustrate b;"
                           + "store b into '/tmp/output1';\n";
 
-            GruntParser parser = new GruntParser(new StringReader(script));
+            GruntParser parser = new GruntParser(new StringReader(script), myPig);
             parser.setInteractive(false);
-            parser.setParams(myPig);
             parser.parseStopOnError();
 
         } catch (Exception e) {
@@ -1280,9 +1278,8 @@ public class TestMultiQueryCompiler {
                           + "explain b;"
                           + "store b into '/tmp/output1';\n";
 
-            GruntParser parser = new GruntParser(new StringReader(script));
+            GruntParser parser = new GruntParser(new StringReader(script), myPig);
             parser.setInteractive(false);
-            parser.setParams(myPig);
             parser.parseStopOnError();
 
         } catch (Exception e) {
@@ -1303,9 +1300,8 @@ public class TestMultiQueryCompiler {
                           + "dump b;"
                           + "store b into '/tmp/output1';\n";
 
-            GruntParser parser = new GruntParser(new StringReader(script));
+            GruntParser parser = new GruntParser(new StringReader(script), myPig);
             parser.setInteractive(false);
-            parser.setParams(myPig);
             parser.parseStopOnError();
 
         } catch (Exception e) {
@@ -1336,9 +1332,8 @@ public class TestMultiQueryCompiler {
     public void testLoadStoreLoop() {
         try {
             String script = "a = load 'dummy'; b = filter a by $0 == 1; store b into 'dummy';\n";
-            GruntParser parser = new GruntParser(new StringReader(script));
+            GruntParser parser = new GruntParser(new StringReader(script), myPig);
             parser.setInteractive(false);
-            parser.setParams(myPig);
             myPig.getPigContext().inExplain = true;
             parser.parseStopOnError();
         } catch (Exception e) {

Modified: pig/branches/tez/test/org/apache/pig/test/TestMultiQueryLocal.java
URL: http://svn.apache.org/viewvc/pig/branches/tez/test/org/apache/pig/test/TestMultiQueryLocal.java?rev=1578718&r1=1578717&r2=1578718&view=diff
==============================================================================
--- pig/branches/tez/test/org/apache/pig/test/TestMultiQueryLocal.java (original)
+++ pig/branches/tez/test/org/apache/pig/test/TestMultiQueryLocal.java Tue Mar 18 02:19:24
2014
@@ -443,9 +443,8 @@ public class TestMultiQueryLocal {
                           + "explain b;"
                           + "store b into '" + TMP_DIR + "/Pig-TestMultiQueryLocal1';\n";
 
-            GruntParser parser = new GruntParser(new StringReader(script));
+            GruntParser parser = new GruntParser(new StringReader(script), myPig);
             parser.setInteractive(false);
-            parser.setParams(myPig);
             parser.parseStopOnError();
 
         } catch (Exception e) {
@@ -468,9 +467,8 @@ public class TestMultiQueryLocal {
                           + "dump b;"
                           + "store b into '" + TMP_DIR + "/Pig-TestMultiQueryLocal1';\n";
 
-            GruntParser parser = new GruntParser(new StringReader(script));
+            GruntParser parser = new GruntParser(new StringReader(script), myPig);
             parser.setInteractive(false);
-            parser.setParams(myPig);
             parser.parseStopOnError();
 
         } catch (Exception e) {
@@ -493,9 +491,8 @@ public class TestMultiQueryLocal {
                           + "describe b;"
                           + "store b into '" + TMP_DIR + "/Pig-TestMultiQueryLocal1';\n";
 
-            GruntParser parser = new GruntParser(new StringReader(script));
+            GruntParser parser = new GruntParser(new StringReader(script), myPig);
             parser.setInteractive(false);
-            parser.setParams(myPig);
             parser.parseStopOnError();
 
         } catch (Exception e) {
@@ -518,10 +515,9 @@ public class TestMultiQueryLocal {
                           + "illustrate b;"
                           + "store b into '" + TMP_DIR + "/Pig-TestMultiQueryLocal1';\n";
 
-            GruntParser parser = new GruntParser(new StringReader(script));
+            GruntParser parser = new GruntParser(new StringReader(script), myPig);
             parser.setInteractive(false);
             myPig.getPigContext().getProperties().setProperty("pig.usenewlogicalplan", "true");
-            parser.setParams(myPig);
             parser.parseStopOnError();
 
         } catch (Exception e) {

Modified: pig/branches/tez/test/org/apache/pig/test/TestNewPlanPushDownForeachFlatten.java
URL: http://svn.apache.org/viewvc/pig/branches/tez/test/org/apache/pig/test/TestNewPlanPushDownForeachFlatten.java?rev=1578718&r1=1578717&r2=1578718&view=diff
==============================================================================
--- pig/branches/tez/test/org/apache/pig/test/TestNewPlanPushDownForeachFlatten.java (original)
+++ pig/branches/tez/test/org/apache/pig/test/TestNewPlanPushDownForeachFlatten.java Tue Mar
18 02:19:24 2014
@@ -1111,6 +1111,25 @@ public class TestNewPlanPushDownForeachF
                   ((LOLoad)load).getSchema().getField("a1") != null );
     }
 
+    // See PIG-3782
+    @Test
+    public void testForeachJoinWithUserDefinedSchemaAndPruning() throws Exception {
+        String query =
+        "a = load '1.txt' as (a0:int, a1, a2:bag{});" +
+        "b = load '2.txt' as (b0:int, b1);" +
+        "c = foreach a generate a0, flatten(a2) as (q1, q2);" +
+        "d = join c by a0, b by b0;" +
+        "e = foreach d generate a0, q1, q2;" +
+        "f = foreach e generate a0, (int)q1, (int)q2;" +
+        "store f into 'output';" ;
+
+        LogicalPlan newLogicalPlan = migrateAndOptimizePlanWithPruning( query );
+        //In the original issue, Exception is thrown during the Pruning due to
+        //incorrect UID assignment by the PushDownForEachFlatten failing this
+        //test
+
+    }
+
     public class MyPlanOptimizerWithPruning extends LogicalPlanOptimizer {
         protected MyPlanOptimizerWithPruning (OperatorPlan p,  int iterations) {
             super(p, iterations, new HashSet<String>());

Modified: pig/branches/tez/test/org/apache/pig/test/TestPigRunner.java
URL: http://svn.apache.org/viewvc/pig/branches/tez/test/org/apache/pig/test/TestPigRunner.java?rev=1578718&r1=1578717&r2=1578718&view=diff
==============================================================================
--- pig/branches/tez/test/org/apache/pig/test/TestPigRunner.java (original)
+++ pig/branches/tez/test/org/apache/pig/test/TestPigRunner.java Tue Mar 18 02:19:24 2014
@@ -52,7 +52,7 @@ import org.apache.pig.tools.pigstats.Out
 import org.apache.pig.tools.pigstats.PigProgressNotificationListener;
 import org.apache.pig.tools.pigstats.PigStats;
 import org.apache.pig.tools.pigstats.PigStatsUtil;
-import org.apache.pig.tools.pigstats.SimpleFetchPigStats;
+import org.apache.pig.tools.pigstats.EmptyPigStats;
 import org.apache.pig.tools.pigstats.mapreduce.MRJobStats;
 import org.apache.pig.tools.pigstats.mapreduce.MRPigStatsUtil;
 import org.junit.AfterClass;
@@ -193,7 +193,7 @@ public class TestPigRunner {
             String[] args = { "-Dstop.on.failure=true", "-Dopt.multiquery=false", "-Daggregate.warning=false",
PIG_FILE };
             PigStats stats = PigRunner.run(args, new TestNotificationListener());
 
-            assertTrue(stats instanceof SimpleFetchPigStats);
+            assertTrue(stats instanceof EmptyPigStats);
             assertTrue(stats.isSuccessful());
             assertEquals(0, stats.getNumberJobs());
             assertEquals(stats.getJobGraph().size(), 0);

Modified: pig/branches/tez/test/org/apache/pig/test/TestPigServer.java
URL: http://svn.apache.org/viewvc/pig/branches/tez/test/org/apache/pig/test/TestPigServer.java?rev=1578718&r1=1578717&r2=1578718&view=diff
==============================================================================
--- pig/branches/tez/test/org/apache/pig/test/TestPigServer.java (original)
+++ pig/branches/tez/test/org/apache/pig/test/TestPigServer.java Tue Mar 18 02:19:24 2014
@@ -958,9 +958,8 @@ public class TestPigServer {
             PigServer pigServer = new PigServer(pigContext);
             data = resetData(pigServer);
             data.set("foo", tuple("a", 1, "b"), tuple("b", 2, "c"), tuple("c", 3, "d"));
-            GruntParser grunt = new GruntParser(in);
+            GruntParser grunt = new GruntParser(in, pigServer);
             grunt.setInteractive(false);
-            grunt.setParams(pigServer);
             grunt.parseStopOnError(true); //not batch
         }
 

Modified: pig/branches/tez/test/org/apache/pig/test/Util.java
URL: http://svn.apache.org/viewvc/pig/branches/tez/test/org/apache/pig/test/Util.java?rev=1578718&r1=1578717&r2=1578718&view=diff
==============================================================================
--- pig/branches/tez/test/org/apache/pig/test/Util.java (original)
+++ pig/branches/tez/test/org/apache/pig/test/Util.java Tue Mar 18 02:19:24 2014
@@ -613,10 +613,8 @@ public class Util {
         }
         PigServer ps = new PigServer(ExecType.MAPREDUCE, cluster.getProperties());
         String script = getMkDirCommandForHadoop2_0(fileNameOnCluster) + "fs -put " + localFileName
+ " " + fileNameOnCluster;
-
-	    GruntParser parser = new GruntParser(new StringReader(script));
+        GruntParser parser = new GruntParser(new StringReader(script), ps);
         parser.setInteractive(false);
-        parser.setParams(ps);
         try {
             parser.parseStopOnError();
         } catch (org.apache.pig.tools.pigscript.parser.ParseException e) {
@@ -635,9 +633,8 @@ public class Util {
 
         new File(toLocalFileName).deleteOnExit();
 
-        GruntParser parser = new GruntParser(new StringReader(script));
+        GruntParser parser = new GruntParser(new StringReader(script), ps);
         parser.setInteractive(false);
-        parser.setParams(ps);
         try {
             parser.parseStopOnError();
         } catch (org.apache.pig.tools.pigscript.parser.ParseException e) {
@@ -1289,4 +1286,19 @@ public class Util {
             }
         };
     }
+
+    /**
+     * 
+     * @param expected
+     *            Exception class that is expected to be thrown
+     * @param found
+     *            Exception that occurred in the test
+     * @param message
+     *            expected String to verify against
+     */
+    public static void assertExceptionAndMessage(Class<?> expected,
+            Exception found, String message) {
+        assertEquals(expected, found.getClass());
+        assertEquals(found.getMessage(), message);
+    }
 }

Propchange: pig/branches/tez/test/org/apache/pig/test/data/bzipdir1.bz2/bzipdir2.bz2/recordLossblockHeaderEndsAt136500.txt.bz2
------------------------------------------------------------------------------
  Merged /pig/trunk/test/org/apache/pig/test/data/bzipdir1.bz2/bzipdir2.bz2/recordLossblockHeaderEndsAt136500.txt.bz2:r1571422-1578679

Modified: pig/branches/tez/test/org/apache/pig/test/pigunit/pig/TestGruntParser.java
URL: http://svn.apache.org/viewvc/pig/branches/tez/test/org/apache/pig/test/pigunit/pig/TestGruntParser.java?rev=1578718&r1=1578717&r2=1578718&view=diff
==============================================================================
--- pig/branches/tez/test/org/apache/pig/test/pigunit/pig/TestGruntParser.java (original)
+++ pig/branches/tez/test/org/apache/pig/test/pigunit/pig/TestGruntParser.java Tue Mar 18
02:19:24 2014
@@ -19,7 +19,10 @@ import java.util.Map;
 
 import junit.framework.Assert;
 
+import org.apache.pig.ExecType;
+import org.apache.pig.backend.executionengine.ExecException;
 import org.apache.pig.pigunit.pig.GruntParser;
+import org.apache.pig.pigunit.pig.PigServer;
 import org.junit.Before;
 import org.junit.Test;
 
@@ -30,12 +33,13 @@ public class TestGruntParser {
 
   @SuppressWarnings("serial")
   @Before
-  public void setUp() {
+  public void setUp() throws ExecException {
     override = new HashMap<String, String>() {{
       put("STORE", "");
       put("DUMP", "");
     }};
-    parser = new GruntParser(new StringReader(""), override);
+    PigServer pigServer = new PigServer(ExecType.LOCAL);
+    parser = new GruntParser(new StringReader(""), pigServer, override);
   }
 
   @Test

Modified: pig/branches/tez/test/org/apache/pig/tools/grunt/TestGruntParser.java
URL: http://svn.apache.org/viewvc/pig/branches/tez/test/org/apache/pig/tools/grunt/TestGruntParser.java?rev=1578718&r1=1578717&r2=1578718&view=diff
==============================================================================
--- pig/branches/tez/test/org/apache/pig/tools/grunt/TestGruntParser.java (original)
+++ pig/branches/tez/test/org/apache/pig/tools/grunt/TestGruntParser.java Tue Mar 18 02:19:24
2014
@@ -39,18 +39,17 @@ public class TestGruntParser {
     public void setup() throws IOException {
         pig = new PigServer(ExecType.LOCAL);
     }
-    
+
     @Test
     public void testProcessRemove() throws IOException {
         File tmpFile = File.createTempFile("TestGruntParser", "testProcessRemove");
         String dummyScript = "";
-        parser = new GruntParser(new ByteArrayInputStream(dummyScript.getBytes()));
-        parser.setParams(pig);
-        
+        parser = new GruntParser(new ByteArrayInputStream(dummyScript.getBytes()), pig);
+
         //Delete existing file and check that it doesn't exist
         parser.processRemove(tmpFile.getAbsolutePath(), "");
         assertFalse(tmpFile.exists());
-        
+
         //Delete non-existing file and check for exception
         try {
             parser.processRemove(tmpFile.getAbsolutePath(), "");

Modified: pig/branches/tez/test/perf/pigmix/bin/runpigmix.pl
URL: http://svn.apache.org/viewvc/pig/branches/tez/test/perf/pigmix/bin/runpigmix.pl?rev=1578718&r1=1578717&r2=1578718&view=diff
==============================================================================
--- pig/branches/tez/test/perf/pigmix/bin/runpigmix.pl (original)
+++ pig/branches/tez/test/perf/pigmix/bin/runpigmix.pl Tue Mar 18 02:19:24 2014
@@ -80,7 +80,10 @@ for(my $i = 1; $i <= 17; $i++) {
         $mr_times = int($mr_times + 0.5);
         $total_mr_times = $total_mr_times + $mr_times;
 
-        my $multiplier = $pig_times/$mr_times;
+
+        my $multiplier=0;
+        if ($mr_times!=0) 
+            $multiplier = $pig_times/$mr_times;
         print "PigMix_$i pig run time: $pig_times, java run time: $mr_times, multiplier:
$multiplier\n";
     }
 }



Mime
View raw message