pig-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From da...@apache.org
Subject svn commit: r1746331 - in /pig/branches/branch-0.16: CHANGES.txt src/org/apache/pig/backend/hadoop/executionengine/tez/TezDagBuilder.java test/org/apache/pig/tez/TestTezGraceParallelism.java
Date Tue, 31 May 2016 19:46:33 GMT
Author: daijy
Date: Tue May 31 19:46:33 2016
New Revision: 1746331

URL: http://svn.apache.org/viewvc?rev=1746331&view=rev
Log:
PIG-4786: CROSS will not work correctly with Grace Parallelism

Modified:
    pig/branches/branch-0.16/CHANGES.txt
    pig/branches/branch-0.16/src/org/apache/pig/backend/hadoop/executionengine/tez/TezDagBuilder.java
    pig/branches/branch-0.16/test/org/apache/pig/tez/TestTezGraceParallelism.java

Modified: pig/branches/branch-0.16/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.16/CHANGES.txt?rev=1746331&r1=1746330&r2=1746331&view=diff
==============================================================================
--- pig/branches/branch-0.16/CHANGES.txt (original)
+++ pig/branches/branch-0.16/CHANGES.txt Tue May 31 19:46:33 2016
@@ -131,6 +131,8 @@ PIG-4639: Add better parser for Apache H
 
 BUG FIXES
 
+PIG-4786: CROSS will not work correctly with Grace Parallelism (daijy)
+
 PIG-3227: SearchEngineExtractor does not work for bing (dannyant via daijy)
 
 PIG-4902: Fix UT failures on 0.16 branch: TestTezGraceParallelism, TestPigScriptParser (daijy)

Modified: pig/branches/branch-0.16/src/org/apache/pig/backend/hadoop/executionengine/tez/TezDagBuilder.java
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.16/src/org/apache/pig/backend/hadoop/executionengine/tez/TezDagBuilder.java?rev=1746331&r1=1746330&r2=1746331&view=diff
==============================================================================
--- pig/branches/branch-0.16/src/org/apache/pig/backend/hadoop/executionengine/tez/TezDagBuilder.java
(original)
+++ pig/branches/branch-0.16/src/org/apache/pig/backend/hadoop/executionengine/tez/TezDagBuilder.java
Tue May 31 19:46:33 2016
@@ -807,7 +807,8 @@ public class TezDagBuilder extends TezOp
                     // Use auto-parallelism feature of ShuffleVertexManager to dynamically
                     // reduce the parallelism of the vertex
                     if (payloadConf.getBoolean(PigConfiguration.PIG_TEZ_GRACE_PARALLELISM,
true)
-                            && !TezOperPlan.getGrandParentsForGraceParallelism(getPlan(),
tezOp).isEmpty()) {
+                            && !TezOperPlan.getGrandParentsForGraceParallelism(getPlan(),
tezOp).isEmpty()
+                            && tezOp.getCrossKeys() == null) {
                         vmPluginName = PigGraceShuffleVertexManager.class.getName();
                         tezOp.setUseGraceParallelism(true);
                         vmPluginConf.set("pig.tez.plan", getSerializedTezPlan());

Modified: pig/branches/branch-0.16/test/org/apache/pig/tez/TestTezGraceParallelism.java
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.16/test/org/apache/pig/tez/TestTezGraceParallelism.java?rev=1746331&r1=1746330&r2=1746331&view=diff
==============================================================================
--- pig/branches/branch-0.16/test/org/apache/pig/tez/TestTezGraceParallelism.java (original)
+++ pig/branches/branch-0.16/test/org/apache/pig/tez/TestTezGraceParallelism.java Tue May
31 19:46:33 2016
@@ -322,4 +322,33 @@ public class TestTezGraceParallelism {
             super.setStoreLocation(location, job);
         }
     }
+
+    @Test
+    // See PIG-4786
+    public void testCross() throws IOException{
+        // scope-90 is the cross vertex. It should not use PigGraceShuffleVertexManager
+        NodeIdGenerator.reset();
+        PigServer.resetScope();
+        StringWriter writer = new StringWriter();
+        Util.createLogAppender("testCross", writer, PigGraceShuffleVertexManager.class);
+        File outputDir = File.createTempFile("intemediate", "txt");
+        outputDir.delete();
+        pigServer.getPigContext().getProperties().setProperty("mapreduce.input.fileinputformat.split.maxsize",
"3000");
+        pigServer.getPigContext().getProperties().setProperty("pig.noSplitCombination", "true");
+        pigServer.registerQuery("A = load '" + INPUT_DIR + "/" + INPUT_FILE2 + "' as (name:chararray,
gender:chararray);");
+        pigServer.registerQuery("B = order A by name;");
+        pigServer.registerQuery("C = distinct B;");
+        pigServer.registerQuery("D = load '" + INPUT_DIR + "/" + INPUT_FILE1 + "' as (name:chararray,
age:int);");
+        pigServer.registerQuery("E = group D by name;");
+        pigServer.registerQuery("F = foreach E generate group as name, AVG(D.age) as avg_age;");
+        pigServer.registerQuery("G = cross C, F;");
+        Iterator<Tuple> iter = pigServer.openIterator("G");
+        int count = 0;
+        while (iter.hasNext()) {
+            iter.next();
+            count++;
+        }
+        assertEquals(count, 400);
+        assertFalse(writer.toString().contains("scope-90"));
+    }
 }



Mime
View raw message