Return-Path: X-Original-To: apmail-pig-commits-archive@www.apache.org Delivered-To: apmail-pig-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id D301317C14 for ; Tue, 7 Apr 2015 23:21:03 +0000 (UTC) Received: (qmail 45589 invoked by uid 500); 7 Apr 2015 23:21:03 -0000 Delivered-To: apmail-pig-commits-archive@pig.apache.org Received: (qmail 45411 invoked by uid 500); 7 Apr 2015 23:21:03 -0000 Mailing-List: contact commits-help@pig.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@pig.apache.org Delivered-To: mailing list commits@pig.apache.org Received: (qmail 45295 invoked by uid 99); 7 Apr 2015 23:21:03 -0000 Received: from eris.apache.org (HELO hades.apache.org) (140.211.11.105) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 07 Apr 2015 23:21:03 +0000 Received: from hades.apache.org (localhost [127.0.0.1]) by hades.apache.org (ASF Mail Server at hades.apache.org) with ESMTP id 55BE6AC0454 for ; Tue, 7 Apr 2015 23:21:03 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r1671973 [2/3] - in /pig/trunk: ./ src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/ src/org/apache/pig/backend/hadoop/executionengine/tez/ src/org/apache/pig/backend/hadoop/executionengine/tez/plan/ src/o... Date: Tue, 07 Apr 2015 23:21:02 -0000 To: commits@pig.apache.org From: rohini@apache.org X-Mailer: svnmailer-1.0.9 Message-Id: <20150407232103.55BE6AC0454@hades.apache.org> Modified: pig/trunk/test/e2e/pig/tests/nightly.conf URL: http://svn.apache.org/viewvc/pig/trunk/test/e2e/pig/tests/nightly.conf?rev=1671973&r1=1671972&r2=1671973&view=diff ============================================================================== --- pig/trunk/test/e2e/pig/tests/nightly.conf (original) +++ pig/trunk/test/e2e/pig/tests/nightly.conf Tue Apr 7 23:21:01 2015 @@ -1380,7 +1380,8 @@ store g into ':OUTPATH:';\, { 'name' => 'Union', 'tests' => [ - { + { + # Simple store 'num' => 1, 'pig' => q\a = load ':INPATH:/singlefile/studenttab10k' as (name, age, gpa); b = load ':INPATH:/singlefile/votertab10k' as (name, age, registration, contributions); @@ -1389,7 +1390,8 @@ d = foreach b generate name, age; e = union c, d; store e into ':OUTPATH:';\, }, - { + { + # Union + Groupby + Combiner 'num' => 2, 'pig' => q\a = load ':INPATH:/singlefile/studenttab10k' as (name, age, gpa); b = load ':INPATH:/singlefile/studentcolon10k' using PigStorage(':') as (name, age, gpa); @@ -1399,6 +1401,7 @@ e = foreach d generate group, SUM(c.age) store e into ':OUTPATH:';\, }, { + # Union + Groupby + Secondary key partitioner 'num' => 3, 'pig' => q\a = load ':INPATH:/singlefile/studenttab10k' as (name, age, gpa); b = load ':INPATH:/singlefile/studentcolon10k' using PigStorage(':') as (name, age, gpa); @@ -1408,6 +1411,7 @@ e = foreach d { f = order c by $1,$2; ge store e into ':OUTPATH:';\, }, { + # Union + Orderby 'num' => 4, 'pig' => q\a = load ':INPATH:/singlefile/studenttab10k' as (name, age, gpa); b = load ':INPATH:/singlefile/studentcolon10k' using PigStorage(':') as (name, age, gpa); @@ -1417,6 +1421,7 @@ store d into ':OUTPATH:';\, 'sortArgs' => ['-t', ' ', '-k', '1,1'], }, { + # Simple split + Union 'num' => 5, 'pig' => q\a = load ':INPATH:/singlefile/studenttab10k' as (name, age, gpa); b = load ':INPATH:/singlefile/studentcolon10k' using PigStorage(':') as (name, age, gpa); @@ -1427,6 +1432,7 @@ store a2 into ':OUTPATH:.1'; store d into ':OUTPATH:.2';\, }, { + # Union + Join 'num' => 6, 'pig' => q\a = load ':INPATH:/singlefile/studenttab10k' as (name, age, gpa); b = load ':INPATH:/singlefile/studentcolon10k' using PigStorage(':') as (name, age, gpa); @@ -1436,6 +1442,7 @@ e = join c by name, d by name PARALLEL 2 store e into ':OUTPATH:';\, }, { + # Union + Replicate Join left 'num' => 7, 'pig' => q\a = load ':INPATH:/singlefile/studenttab10k' as (name, age, gpa); b = load ':INPATH:/singlefile/studentcolon10k' using PigStorage(':') as (name, age, gpa); @@ -1445,6 +1452,7 @@ e = join c by name, d by name using 'rep store e into ':OUTPATH:';\, }, { + # Union + Replicate Join right 'num' => 8, 'pig' => q\a = load ':INPATH:/singlefile/studenttab10k' as (name, age, gpa); b = load ':INPATH:/singlefile/studentcolon10k' using PigStorage(':') as (name, age, gpa); @@ -1454,6 +1462,7 @@ e = join d by name, c by name using 'rep store e into ':OUTPATH:';\, }, { + # Union + Skewed Join left 'num' => 9, 'pig' => q\a = load ':INPATH:/singlefile/studenttab10k' as (name, age, gpa); b = load ':INPATH:/singlefile/studentcolon10k' using PigStorage(':') as (name, age, gpa); @@ -1463,6 +1472,7 @@ e = join c by name, d by name using 'ske store e into ':OUTPATH:';\, }, { + # Union + Skewed Join right 'num' => 10, 'pig' => q\a = load ':INPATH:/singlefile/studenttab10k' as (name, age, gpa); b = load ':INPATH:/singlefile/studentcolon10k' using PigStorage(':') as (name, age, gpa); @@ -1486,6 +1496,7 @@ i = foreach i generate group, SUM(h.age) store i into ':OUTPATH:';\, }, { + # Union + operators 'num' => 12, 'pig' => q\a = load ':INPATH:/singlefile/studentnulltab10k' as (name, age:int, gpa:double); b = load ':INPATH:/singlefile/studentcolon10k' using PigStorage(':') as (name, age:int, gpa:double); @@ -1496,6 +1507,7 @@ e = filter d by (name matches '.*MIKE.*' store e into ':OUTPATH:';\, }, { + # Union + Groupby + Replicate join 'num' => 13, 'pig' => q\a = load ':INPATH:/singlefile/studenttab10k' as (name, age, gpa); b = load ':INPATH:/singlefile/studentcolon10k' using PigStorage(':') as (name, age, gpa); @@ -1505,7 +1517,8 @@ e = load ':INPATH:/singlefile/votertab10 f = join d by group, e by name using 'replicated'; store f into ':OUTPATH:';\, }, - { ## Secondary Key + { + # Group by with Secondary Key + Union 'num' => 14, 'pig' => q\a = load ':INPATH:/singlefile/studenttab10k' as (name:chararray, age, gpa); b = load ':INPATH:/singlefile/studentcolon10k' using PigStorage(':') as (name:chararray, age, gpa); @@ -1529,6 +1542,7 @@ store j into ':OUTPATH:';\, 'sortArgs' => ['-t', ' ', '-k', '1,1'], }, { + # Union + Cross 'num' => 15, 'pig' => q\a = load ':INPATH:/singlefile/studenttab10k' as (name, age, gpa:float); b = load ':INPATH:/singlefile/studentcolon10k' using PigStorage(':') as (name, age, gpa:float); @@ -1537,22 +1551,6 @@ d = cross a, c; e = union b, d; store e into ':OUTPATH:';\, }, - { ## Multiple splits - 'num' => 16, - 'pig' => q\a = load ':INPATH:/singlefile/studenttab10k' as (name, age, gpa:float); -b = load ':INPATH:/singlefile/studentcolon10k' using PigStorage(':') as (name, age, gpa:float); -c = filter a by gpa >= 4; -c1 = foreach c generate *; -c2 = foreach c generate *; -c3 = union c1, c2; -d = filter a by gpa < 4; -d1 = foreach d generate *; -d2 = foreach d generate *; -d3 = union d1, d2; -a1 = union c3, d3; -e = join a1 by name, b by name; -store e into ':OUTPATH:';\, - }, ] }, { Modified: pig/trunk/test/org/apache/pig/test/TestFRJoin.java URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/TestFRJoin.java?rev=1671973&r1=1671972&r2=1671973&view=diff ============================================================================== --- pig/trunk/test/org/apache/pig/test/TestFRJoin.java (original) +++ pig/trunk/test/org/apache/pig/test/TestFRJoin.java Tue Apr 7 23:21:01 2015 @@ -46,9 +46,8 @@ import org.apache.pig.impl.io.FileSpec; import org.apache.pig.impl.logicalLayer.schema.Schema; import org.apache.pig.impl.plan.OperatorKey; import org.apache.pig.test.utils.TestHelper; -import org.junit.After; import org.junit.AfterClass; -import org.junit.Before; +import org.junit.BeforeClass; import org.junit.Test; public class TestFRJoin { @@ -61,8 +60,8 @@ public class TestFRJoin { pigServer = new PigServer(cluster.getExecType(), cluster.getProperties()); } - @Before - public void setUp() throws Exception { + @BeforeClass + public static void oneTimeSetup() throws Exception { int LOOP_SIZE = 2; String[] input = new String[2 * LOOP_SIZE]; int k = 0; @@ -85,13 +84,9 @@ public class TestFRJoin { @AfterClass public static void oneTimeTearDown() throws Exception { - cluster.shutDown(); - } - - @After - public void tearDown() throws Exception { Util.deleteFile(cluster, INPUT_FILE); Util.deleteFile(cluster, INPUT_FILE2); + cluster.shutDown(); } public static class FRJoin extends EvalFunc { Modified: pig/trunk/test/org/apache/pig/test/TestMultiQueryLocal.java URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/TestMultiQueryLocal.java?rev=1671973&r1=1671972&r2=1671973&view=diff ============================================================================== --- pig/trunk/test/org/apache/pig/test/TestMultiQueryLocal.java (original) +++ pig/trunk/test/org/apache/pig/test/TestMultiQueryLocal.java Tue Apr 7 23:21:01 2015 @@ -27,8 +27,6 @@ import java.util.Collections; import java.util.Iterator; import java.util.Properties; -import junit.framework.Assert; - import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; @@ -58,6 +56,7 @@ import org.apache.pig.tools.pigscript.pa import org.apache.pig.tools.pigstats.JobStats; import org.apache.pig.tools.pigstats.PigStats; import org.junit.After; +import org.junit.Assert; import org.junit.Assume; import org.junit.Before; import org.junit.Test; @@ -84,7 +83,7 @@ public class TestMultiQueryLocal { } @Test - public void testMultiQueryWithTwoStores() { + public void testMultiQueryWithTwoStores() throws Exception { System.out.println("===== test multi-query with 2 stores ====="); @@ -106,32 +105,23 @@ public class TestMultiQueryLocal { Assert.assertTrue(executePlan(pp)); - } catch (Exception e) { - e.printStackTrace(); - Assert.fail(); } finally { deleteOutputFiles(); } } @Test - public void testEmptyExecute() { + public void testEmptyExecute() throws Exception { System.out.println("=== test empty execute ==="); - try { - myPig.setBatchOn(); - myPig.executeBatch(); - myPig.executeBatch(); - myPig.discardBatch(); - } - catch (Exception e) { - e.printStackTrace(); - Assert.fail(); - } + myPig.setBatchOn(); + myPig.executeBatch(); + myPig.executeBatch(); + myPig.discardBatch(); } @Test - public void testMultiQueryWithTwoStores2() { + public void testMultiQueryWithTwoStores2() throws Exception { System.out.println("===== test multi-query with 2 stores (2) ====="); @@ -147,16 +137,13 @@ public class TestMultiQueryLocal { myPig.executeBatch(); - } catch (Exception e) { - e.printStackTrace(); - Assert.fail(); } finally { deleteOutputFiles(); } } @Test - public void testMultiQueryWithTwoStores2Execs() { + public void testMultiQueryWithTwoStores2Execs() throws Exception { System.out.println("===== test multi-query with 2 stores (2) ====="); @@ -175,16 +162,13 @@ public class TestMultiQueryLocal { myPig.executeBatch(); myPig.discardBatch(); - } catch (Exception e) { - e.printStackTrace(); - Assert.fail(); } finally { deleteOutputFiles(); } } @Test - public void testMultiQueryWithThreeStores() { + public void testMultiQueryWithThreeStores() throws Exception { System.out.println("===== test multi-query with 3 stores ====="); @@ -206,16 +190,13 @@ public class TestMultiQueryLocal { Assert.assertTrue(executePlan(pp)); - } catch (Exception e) { - e.printStackTrace(); - Assert.fail(); } finally { deleteOutputFiles(); } } @Test - public void testMultiQueryWithThreeStores2() { + public void testMultiQueryWithThreeStores2() throws Exception { System.out.println("===== test multi-query with 3 stores (2) ====="); @@ -234,16 +215,13 @@ public class TestMultiQueryLocal { myPig.executeBatch(); myPig.discardBatch(); - } catch (Exception e) { - e.printStackTrace(); - Assert.fail(); } finally { deleteOutputFiles(); } } @Test - public void testMultiQueryWithTwoLoads() { + public void testMultiQueryWithTwoLoads() throws Exception { System.out.println("===== test multi-query with two loads ====="); @@ -268,16 +246,13 @@ public class TestMultiQueryLocal { Assert.assertTrue(executePlan(pp)); - } catch (Exception e) { - e.printStackTrace(); - Assert.fail(); } finally { deleteOutputFiles(); } } @Test - public void testMultiQueryWithTwoLoads2() { + public void testMultiQueryWithTwoLoads2() throws Exception { System.out.println("===== test multi-query with two loads (2) ====="); @@ -298,58 +273,45 @@ public class TestMultiQueryLocal { myPig.executeBatch(); myPig.discardBatch(); - } catch (Exception e) { - e.printStackTrace(); - Assert.fail(); } finally { deleteOutputFiles(); } } @Test - public void testMultiQueryWithNoStore() { + public void testMultiQueryWithNoStore() throws Exception { System.out.println("===== test multi-query with no store ====="); - try { - myPig.setBatchOn(); + myPig.setBatchOn(); - myPig.registerQuery("a = load 'test/org/apache/pig/test/data/passwd' " + - "using PigStorage(':') as (uname:chararray, passwd:chararray, uid:int,gid:int);"); - myPig.registerQuery("b = filter a by uid > 5;"); - myPig.registerQuery("group b by gid;"); + myPig.registerQuery("a = load 'test/org/apache/pig/test/data/passwd' " + + "using PigStorage(':') as (uname:chararray, passwd:chararray, uid:int,gid:int);"); + myPig.registerQuery("b = filter a by uid > 5;"); + myPig.registerQuery("group b by gid;"); - LogicalPlan lp = checkLogicalPlan(0, 0, 0); + LogicalPlan lp = checkLogicalPlan(0, 0, 0); - // XXX Physical plan has one less node in the local case - PhysicalPlan pp = checkPhysicalPlan(lp, 0, 0, 0); + // XXX Physical plan has one less node in the local case + checkPhysicalPlan(lp, 0, 0, 0); - } catch (Exception e) { - e.printStackTrace(); - Assert.fail(); - } } @Test - public void testMultiQueryWithNoStore2() { + public void testMultiQueryWithNoStore2() throws Exception { System.out.println("===== test multi-query with no store (2) ====="); - try { - myPig.setBatchOn(); + myPig.setBatchOn(); - myPig.registerQuery("a = load 'test/org/apache/pig/test/data/passwd' " + - "using PigStorage(':') as (uname:chararray, passwd:chararray, uid:int,gid:int);"); - myPig.registerQuery("b = filter a by uid > 5;"); - myPig.registerQuery("group b by gid;"); + myPig.registerQuery("a = load 'test/org/apache/pig/test/data/passwd' " + + "using PigStorage(':') as (uname:chararray, passwd:chararray, uid:int,gid:int);"); + myPig.registerQuery("b = filter a by uid > 5;"); + myPig.registerQuery("group b by gid;"); - myPig.executeBatch(); - myPig.discardBatch(); + myPig.executeBatch(); + myPig.discardBatch(); - } catch (Exception e) { - e.printStackTrace(); - Assert.fail(); - } } public static class PigStorageWithConfig extends PigStorage { @@ -428,43 +390,38 @@ public class TestMultiQueryLocal { // See PIG-2912 @Test - public void testMultiStoreWithConfig() { + public void testMultiStoreWithConfig() throws Exception { System.out.println("===== test multi-query with competing config ====="); - try { - myPig.setBatchOn(); + myPig.setBatchOn(); - myPig.registerQuery("a = load 'test/org/apache/pig/test/data/passwd' " + - "using PigStorage(':') as (uname:chararray, passwd:chararray, uid:int,gid:int);"); - myPig.registerQuery("b = filter a by uid < 5;"); - myPig.registerQuery("c = filter a by uid > 5;"); - myPig.registerQuery("store b into '" + TMP_DIR + "/Pig-TestMultiQueryLocal1' using " + PigStorageWithConfig.class.getName() + "('test.key1', 'a');"); - myPig.registerQuery("store c into '" + TMP_DIR + "/Pig-TestMultiQueryLocal2' using " + PigStorageWithConfig.class.getName() + "('test.key2', 'b');"); + myPig.registerQuery("a = load 'test/org/apache/pig/test/data/passwd' " + + "using PigStorage(':') as (uname:chararray, passwd:chararray, uid:int,gid:int);"); + myPig.registerQuery("b = filter a by uid < 5;"); + myPig.registerQuery("c = filter a by uid > 5;"); + myPig.registerQuery("store b into '" + TMP_DIR + "/Pig-TestMultiQueryLocal1' using " + PigStorageWithConfig.class.getName() + "('test.key1', 'a');"); + myPig.registerQuery("store c into '" + TMP_DIR + "/Pig-TestMultiQueryLocal2' using " + PigStorageWithConfig.class.getName() + "('test.key2', 'b');"); - myPig.executeBatch(); - myPig.discardBatch(); - FileSystem fs = FileSystem.getLocal(new Configuration()); - BufferedReader reader = new BufferedReader(new InputStreamReader - (fs.open(Util.getFirstPartFile(new Path(TMP_DIR + "/Pig-TestMultiQueryLocal1"))))); - String line; - while ((line = reader.readLine())!=null) { - Assert.assertTrue(line.endsWith("a")); - } - reader = new BufferedReader(new InputStreamReader - (fs.open(Util.getFirstPartFile(new Path(TMP_DIR + "/Pig-TestMultiQueryLocal2"))))); - while ((line = reader.readLine())!=null) { - Assert.assertTrue(line.endsWith("b")); - } - - } catch (Exception e) { - e.printStackTrace(); - Assert.fail(); + myPig.executeBatch(); + myPig.discardBatch(); + FileSystem fs = FileSystem.getLocal(new Configuration()); + BufferedReader reader = new BufferedReader(new InputStreamReader + (fs.open(Util.getFirstPartFile(new Path("file:///" + TMP_DIR + "/Pig-TestMultiQueryLocal1"))))); + String line; + while ((line = reader.readLine())!=null) { + Assert.assertTrue(line.endsWith("a")); + } + reader = new BufferedReader(new InputStreamReader + (fs.open(Util.getFirstPartFile(new Path("file:///" + TMP_DIR + "/Pig-TestMultiQueryLocal2"))))); + while ((line = reader.readLine())!=null) { + Assert.assertTrue(line.endsWith("b")); } + } @Test - public void testMultiQueryWithExplain() { + public void testMultiQueryWithExplain() throws Exception { System.out.println("===== test multi-query with explain ====="); @@ -479,16 +436,13 @@ public class TestMultiQueryLocal { parser.setInteractive(false); parser.parseStopOnError(); - } catch (Exception e) { - e.printStackTrace(); - Assert.fail(); } finally { deleteOutputFiles(); } } @Test - public void testMultiQueryWithDump() { + public void testMultiQueryWithDump() throws Exception { System.out.println("===== test multi-query with dump ====="); @@ -503,16 +457,13 @@ public class TestMultiQueryLocal { parser.setInteractive(false); parser.parseStopOnError(); - } catch (Exception e) { - e.printStackTrace(); - Assert.fail(); } finally { deleteOutputFiles(); } } @Test - public void testMultiQueryWithDescribe() { + public void testMultiQueryWithDescribe() throws Exception { System.out.println("===== test multi-query with describe ====="); @@ -527,9 +478,6 @@ public class TestMultiQueryLocal { parser.setInteractive(false); parser.parseStopOnError(); - } catch (Exception e) { - e.printStackTrace(); - Assert.fail(); } finally { deleteOutputFiles(); } @@ -553,9 +501,6 @@ public class TestMultiQueryLocal { myPig.getPigContext().getProperties().setProperty("pig.usenewlogicalplan", "true"); parser.parseStopOnError(); - } catch (Exception e) { - e.printStackTrace(); - Assert.fail(); } finally { deleteOutputFiles(); myPig.getPigContext().getProperties().setProperty("pig.usenewlogicalplan", "false"); @@ -563,7 +508,7 @@ public class TestMultiQueryLocal { } @Test - public void testStoreOrder() { + public void testStoreOrder() throws Exception { System.out.println("===== multi-query store order ====="); try { @@ -582,7 +527,7 @@ public class TestMultiQueryLocal { myPig.registerQuery("store c into '" + TMP_DIR + "/Pig-TestMultiQueryLocal5';"); LogicalPlan lp = checkLogicalPlan(1, 3, 12); - PhysicalPlan pp = checkPhysicalPlan(lp, 1, 3, 15); + checkPhysicalPlan(lp, 1, 3, 15); myPig.executeBatch(); myPig.discardBatch(); @@ -593,10 +538,6 @@ public class TestMultiQueryLocal { Assert.assertTrue(new File(TMP_DIR + "/Pig-TestMultiQueryLocal4").exists()); Assert.assertTrue(new File(TMP_DIR + "/Pig-TestMultiQueryLocal5").exists()); - - } catch (Exception e) { - e.printStackTrace(); - Assert.fail(); } finally { deleteOutputFiles(); } @@ -656,7 +597,7 @@ public class TestMultiQueryLocal { int expectedLeaves, int expectedSize) throws IOException { lp.optimize(myPig.getPigContext()); - System.out.println("===== check physical plan ====="); + System.out.println("===== check physical plan ====="); PhysicalPlan pp = ((HExecutionEngine)myPig.getPigContext().getExecutionEngine()).compile( lp, null); Added: pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Cross-1.gld URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Cross-1.gld?rev=1671973&view=auto ============================================================================== --- pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Cross-1.gld (added) +++ pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Cross-1.gld Tue Apr 7 23:21:01 2015 @@ -0,0 +1,83 @@ +#-------------------------------------------------- +# There are 1 DAGs in the session +#-------------------------------------------------- +#-------------------------------------------------- +# TEZ DAG plan: pig-0_scope-0 +#-------------------------------------------------- +Tez vertex scope-42 -> Tez vertex scope-44, +Tez vertex scope-43 -> Tez vertex scope-44, +Tez vertex scope-44 + +Tez vertex scope-42 +# Plan on vertex +c: Local Rearrange[tuple]{tuple}(false) - scope-20 -> scope-44 +| | +| Project[int][0] - scope-21 +| | +| Project[int][1] - scope-22 +| +|---c: New For Each(true,true)[tuple] - scope-19 + | | + | POUserFunc(org.apache.pig.impl.builtin.GFCross)[bag] - scope-17 + | | + | |---Constant(2) - scope-15 + | | + | |---Constant(0) - scope-16 + | | + | Project[tuple][*] - scope-18 + | + |---a: New For Each(false,false)[bag] - scope-7 + | | + | Cast[int] - scope-2 + | | + | |---Project[bytearray][0] - scope-1 + | | + | Cast[int] - scope-5 + | | + | |---Project[bytearray][1] - scope-4 + | + |---a: Load(file:///tmp/input1:org.apache.pig.builtin.PigStorage) - scope-0 +Tez vertex scope-43 +# Plan on vertex +c: Local Rearrange[tuple]{tuple}(false) - scope-28 -> scope-44 +| | +| Project[int][0] - scope-29 +| | +| Project[int][1] - scope-30 +| +|---c: New For Each(true,true)[tuple] - scope-27 + | | + | POUserFunc(org.apache.pig.impl.builtin.GFCross)[bag] - scope-25 + | | + | |---Constant(2) - scope-23 + | | + | |---Constant(1) - scope-24 + | | + | Project[tuple][*] - scope-26 + | + |---b: New For Each(false)[bag] - scope-12 + | | + | Cast[int] - scope-10 + | | + | |---Project[bytearray][0] - scope-9 + | + |---b: Load(file:///tmp/input2:org.apache.pig.builtin.PigStorage) - scope-8 +Tez vertex scope-44 +# Plan on vertex +d: Store(file:///tmp/output:org.apache.pig.builtin.PigStorage) - scope-41 +| +|---d: New For Each(false,false,false)[bag] - scope-40 + | | + | Project[int][0] - scope-34 + | | + | Project[int][1] - scope-36 + | | + | Project[int][2] - scope-38 + | + |---c: New For Each(true,true)[tuple] - scope-33 + | | + | Project[bag][1] - scope-31 + | | + | Project[bag][2] - scope-32 + | + |---Package(Packager)[tuple]{tuple} - scope-14 Added: pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Cross-2.gld URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Cross-2.gld?rev=1671973&view=auto ============================================================================== --- pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Cross-2.gld (added) +++ pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Cross-2.gld Tue Apr 7 23:21:01 2015 @@ -0,0 +1,83 @@ +#-------------------------------------------------- +# There are 1 DAGs in the session +#-------------------------------------------------- +#-------------------------------------------------- +# TEZ DAG plan: pig-0_scope-0 +#-------------------------------------------------- +Tez vertex scope-43 -> Tez vertex scope-49, +Tez vertex scope-49 + +Tez vertex scope-43 +# Plan on vertex +a: Split - scope-50 +| | +| d: Local Rearrange[tuple]{tuple}(false) - scope-28 -> scope-49 +| | | +| | Project[int][0] - scope-29 +| | | +| | Project[int][1] - scope-30 +| | +| |---d: New For Each(true,true)[tuple] - scope-27 +| | | +| | POUserFunc(org.apache.pig.impl.builtin.GFCross)[bag] - scope-25 +| | | +| | |---Constant(2) - scope-23 +| | | +| | |---Constant(0) - scope-24 +| | | +| | Project[tuple][*] - scope-26 +| | +| |---b: Filter[bag] - scope-11 +| | | +| | Less Than[boolean] - scope-14 +| | | +| | |---Project[int][0] - scope-12 +| | | +| | |---Constant(5) - scope-13 +| | +| d: Local Rearrange[tuple]{tuple}(false) - scope-36 -> scope-49 +| | | +| | Project[int][0] - scope-37 +| | | +| | Project[int][1] - scope-38 +| | +| |---d: New For Each(true,true)[tuple] - scope-35 +| | | +| | POUserFunc(org.apache.pig.impl.builtin.GFCross)[bag] - scope-33 +| | | +| | |---Constant(2) - scope-31 +| | | +| | |---Constant(1) - scope-32 +| | | +| | Project[tuple][*] - scope-34 +| | +| |---c: Filter[bag] - scope-17 +| | | +| | Equal To[boolean] - scope-20 +| | | +| | |---Project[int][0] - scope-18 +| | | +| | |---Constant(10) - scope-19 +| +|---a: New For Each(false,false)[bag] - scope-7 + | | + | Cast[int] - scope-2 + | | + | |---Project[bytearray][0] - scope-1 + | | + | Cast[int] - scope-5 + | | + | |---Project[bytearray][1] - scope-4 + | + |---a: Load(file:///tmp/input1:org.apache.pig.builtin.PigStorage) - scope-0 +Tez vertex scope-49 +# Plan on vertex +d: Store(file:///tmp/output:org.apache.pig.builtin.PigStorage) - scope-42 +| +|---d: New For Each(true,true)[tuple] - scope-41 + | | + | Project[bag][1] - scope-39 + | | + | Project[bag][2] - scope-40 + | + |---Package(Packager)[tuple]{tuple} - scope-22 Modified: pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-MQ-2-OPTOFF.gld URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-MQ-2-OPTOFF.gld?rev=1671973&r1=1671972&r2=1671973&view=diff ============================================================================== --- pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-MQ-2-OPTOFF.gld (original) +++ pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-MQ-2-OPTOFF.gld Tue Apr 7 23:21:01 2015 @@ -294,12 +294,12 @@ f1: Store(file:///tmp/output/f1:org.apac |---POValueInputTez - scope-383 <- scope-378 Tez vertex scope-384 # Plan on vertex -f2: Store(file:///tmp/output/f2:org.apache.pig.builtin.PigStorage) - scope-413 +f2: Store(file:///tmp/output/f2:org.apache.pig.builtin.PigStorage) - scope-413 -> scope-316 | |---POValueInputTez - scope-385 <- scope-341 Tez vertex scope-386 # Plan on vertex -f2: Store(file:///tmp/output/f2:org.apache.pig.builtin.PigStorage) - scope-414 +f2: Store(file:///tmp/output/f2:org.apache.pig.builtin.PigStorage) - scope-414 -> scope-316 | |---POValueInputTez - scope-387 <- scope-378 Tez vertex group scope-412 <- [scope-384, scope-386] -> null Modified: pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-MQ-2.gld URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-MQ-2.gld?rev=1671973&r1=1671972&r2=1671973&view=diff ============================================================================== --- pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-MQ-2.gld (original) +++ pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-MQ-2.gld Tue Apr 7 23:21:01 2015 @@ -4,9 +4,8 @@ #-------------------------------------------------- # TEZ DAG plan: pig-0_scope-0 #-------------------------------------------------- -Tez vertex scope-106 -> Tez vertex scope-113,Tez vertex scope-119,Tez vertex scope-126,Tez vertex scope-148,Tez vertex scope-158,Tez vertex scope-167,Tez vertex group scope-208, +Tez vertex scope-106 -> Tez vertex scope-113,Tez vertex scope-126,Tez vertex scope-129,Tez vertex scope-148,Tez vertex scope-158,Tez vertex scope-167,Tez vertex group scope-208, Tez vertex scope-113 -Tez vertex scope-119 -> Tez vertex scope-126,Tez vertex scope-129, Tez vertex scope-126 Tez vertex scope-129 Tez vertex scope-148 -> Tez vertex scope-158, @@ -77,11 +76,41 @@ Tez vertex scope-106 | | | | | |---Constant(5) - scope-11 | | +| c: Split - scope-205 +| | | +| | c1: Local Rearrange[tuple]{int}(false) - scope-47 -> scope-126 +| | | | +| | | Project[int][0] - scope-48 +| | | +| | c2: Local Rearrange[tuple]{int}(false) - scope-193 -> scope-129 +| | | | +| | | Project[int][0] - scope-195 +| | | +| | |---c3: New For Each(false,false)[bag] - scope-181 +| | | | +| | | Project[int][0] - scope-182 +| | | | +| | | POUserFunc(org.apache.pig.builtin.AlgebraicMathBase$Initial)[tuple] - scope-183 +| | | | +| | | |---Project[bag][0] - scope-184 +| | | | +| | | |---Project[bag][1] - scope-185 +| | | +| | |---Pre Combiner Local Rearrange[tuple]{Unknown} - scope-196 +| | +| |---c: Filter[bag] - scope-34 +| | | +| | Less Than or Equal[boolean] - scope-37 +| | | +| | |---Project[int][0] - scope-35 +| | | +| | |---Constant(10) - scope-36 +| | | d1: Split - scope-206 | | | | | d1: Store(file:///tmp/output/d1:org.apache.pig.builtin.PigStorage) - scope-80 | | | -| | f2: Store(file:///tmp/output/f2:org.apache.pig.builtin.PigStorage) - scope-209 +| | f2: Store(file:///tmp/output/f2:org.apache.pig.builtin.PigStorage) - scope-209 -> scope-105 | | | |---d1: Filter[bag] - scope-73 | | | @@ -98,8 +127,6 @@ Tez vertex scope-106 | | |---Project[int][0] - scope-70 | | | | | |---Constant(10) - scope-71 -| | -| POValueOutputTez - scope-107 -> [scope-119] | |---a: New For Each(false,false)[bag] - scope-7 | | @@ -131,39 +158,6 @@ b1: Split - scope-201 | | |---Project[bag][1] - scope-28 | |---b1: Package(Packager)[tuple]{int} - scope-17 -Tez vertex scope-119 -# Plan on vertex -c: Split - scope-205 -| | -| c1: Local Rearrange[tuple]{int}(false) - scope-47 -> scope-126 -| | | -| | Project[int][0] - scope-48 -| | -| c2: Local Rearrange[tuple]{int}(false) - scope-193 -> scope-129 -| | | -| | Project[int][0] - scope-195 -| | -| |---c3: New For Each(false,false)[bag] - scope-181 -| | | -| | Project[int][0] - scope-182 -| | | -| | POUserFunc(org.apache.pig.builtin.AlgebraicMathBase$Initial)[tuple] - scope-183 -| | | -| | |---Project[bag][0] - scope-184 -| | | -| | |---Project[bag][1] - scope-185 -| | -| |---Pre Combiner Local Rearrange[tuple]{Unknown} - scope-196 -| -|---c: Filter[bag] - scope-34 - | | - | Less Than or Equal[boolean] - scope-37 - | | - | |---Project[int][0] - scope-35 - | | - | |---Constant(10) - scope-36 - | - |---POValueInputTez - scope-120 <- scope-106 Tez vertex scope-126 # Plan on vertex c1: Store(file:///tmp/output/c1:org.apache.pig.builtin.PigStorage) - scope-54 @@ -176,7 +170,7 @@ c1: Store(file:///tmp/output/c1:org.apac | |---c1: Package(Packager)[tuple]{int} - scope-46 Tez vertex scope-129 -# Combine plan on edge +# Combine plan on edge c2: Local Rearrange[tuple]{int}(false) - scope-197 -> scope-129 | | | Project[int][0] - scope-199 @@ -239,7 +233,7 @@ f1: Split - scope-202 | | | f1: Store(file:///tmp/output/f1:org.apache.pig.builtin.PigStorage) - scope-99 | | -| f2: Store(file:///tmp/output/f2:org.apache.pig.builtin.PigStorage) - scope-210 +| f2: Store(file:///tmp/output/f2:org.apache.pig.builtin.PigStorage) - scope-210 -> scope-105 | |---f1: Limit - scope-169 | Added: pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-MQ-6-OPTOFF.gld URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-MQ-6-OPTOFF.gld?rev=1671973&view=auto ============================================================================== --- pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-MQ-6-OPTOFF.gld (added) +++ pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-MQ-6-OPTOFF.gld Tue Apr 7 23:21:01 2015 @@ -0,0 +1,125 @@ +#-------------------------------------------------- +# There are 1 DAGs in the session +#-------------------------------------------------- +#-------------------------------------------------- +# TEZ DAG plan: pig-0_scope-0 +#-------------------------------------------------- +Tez vertex scope-48 -> Tez vertex scope-50,Tez vertex scope-53, +Tez vertex scope-53 -> Tez vertex scope-56,Tez vertex scope-58, +Tez vertex scope-50 -> Tez vertex scope-52, +Tez vertex scope-52 -> Tez vertex scope-56,Tez vertex scope-58, +Tez vertex scope-56 +Tez vertex scope-58 + +Tez vertex scope-48 +# Plan on vertex +POValueOutputTez - scope-49 -> [scope-50, scope-53] +| +|---a: New For Each(false,false)[bag] - scope-7 + | | + | Cast[int] - scope-2 + | | + | |---Project[bytearray][0] - scope-1 + | | + | Cast[int] - scope-5 + | | + | |---Project[bytearray][1] - scope-4 + | + |---a: Load(file:///tmp/input:org.apache.pig.builtin.PigStorage) - scope-0 +Tez vertex scope-53 +# Plan on vertex +POValueOutputTez - scope-55 -> [scope-56, scope-58] +| +|---POValueInputTez - scope-54 <- scope-48 +Tez vertex scope-50 +# Plan on vertex +b: Local Rearrange[tuple]{int}(false) - scope-72 -> scope-52 +| | +| Project[int][0] - scope-74 +| +|---c: New For Each(false,false)[bag] - scope-61 + | | + | Project[int][0] - scope-62 + | | + | POUserFunc(org.apache.pig.builtin.COUNT$Initial)[tuple] - scope-63 + | | + | |---Project[bag][1] - scope-64 + | + |---Pre Combiner Local Rearrange[tuple]{Unknown} - scope-75 + | + |---POValueInputTez - scope-51 <- scope-48 +Tez vertex scope-52 +# Combine plan on edge +b: Local Rearrange[tuple]{int}(false) - scope-76 -> scope-52 +| | +| Project[int][0] - scope-78 +| +|---c: New For Each(false,false)[bag] - scope-65 + | | + | Project[int][0] - scope-66 + | | + | POUserFunc(org.apache.pig.builtin.COUNT$Intermediate)[tuple] - scope-67 + | | + | |---Project[bag][1] - scope-68 + | + |---b: Package(CombinerPackager)[tuple]{int} - scope-71 +# Plan on vertex +POValueOutputTez - scope-60 -> [scope-56, scope-58] +| +|---c: New For Each(false,false)[bag] - scope-23 + | | + | Project[int][0] - scope-18 + | | + | POUserFunc(org.apache.pig.builtin.COUNT$Final)[long] - scope-21 + | | + | |---Project[bag][1] - scope-69 + | + |---b: Package(CombinerPackager)[tuple]{int} - scope-15 +Tez vertex scope-56 +# Plan on vertex +d: Store(file:///tmp/output1:org.apache.pig.builtin.PigStorage) - scope-35 +| +|---d: Filter[bag] - scope-25 + | | + | Less Than[boolean] - scope-34 + | | + | |---Multiply[long] - scope-31 + | | | + | | |---Cast[long] - scope-27 + | | | | + | | | |---Constant(2) - scope-26 + | | | + | | |---POUserFunc(org.apache.pig.backend.hadoop.executionengine.tez.plan.udf.ReadScalarsTez)[long] - scope-30 + | | | + | | |---Constant(1) - scope-28 + | | + | |---Cast[long] - scope-33 + | | + | |---Project[int][1] - scope-32 + | + |---POValueInputTez - scope-57 <- scope-53 +Tez vertex scope-58 +# Plan on vertex +e: Store(file:///tmp/output2:org.apache.pig.builtin.PigStorage) - scope-47 +| +|---e: Filter[bag] - scope-36 + | | + | Not[boolean] - scope-46 + | | + | |---Less Than[boolean] - scope-45 + | | + | |---Multiply[long] - scope-42 + | | | + | | |---Cast[long] - scope-38 + | | | | + | | | |---Constant(2) - scope-37 + | | | + | | |---POUserFunc(org.apache.pig.backend.hadoop.executionengine.tez.plan.udf.ReadScalarsTez)[long] - scope-41 + | | | + | | |---Constant(1) - scope-39 + | | + | |---Cast[long] - scope-44 + | | + | |---Project[int][1] - scope-43 + | + |---POValueInputTez - scope-59 <- scope-53 Added: pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-MQ-6.gld URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-MQ-6.gld?rev=1671973&view=auto ============================================================================== --- pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-MQ-6.gld (added) +++ pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-MQ-6.gld Tue Apr 7 23:21:01 2015 @@ -0,0 +1,121 @@ +#-------------------------------------------------- +# There are 1 DAGs in the session +#-------------------------------------------------- +#-------------------------------------------------- +# TEZ DAG plan: pig-0_scope-0 +#-------------------------------------------------- +Tez vertex scope-48 -> Tez vertex scope-52,Tez vertex scope-53, +Tez vertex scope-52 -> Tez vertex scope-53,Tez vertex scope-58, +Tez vertex scope-53 -> Tez vertex scope-58, +Tez vertex scope-58 + +Tez vertex scope-48 +# Plan on vertex +a: Split - scope-80 +| | +| b: Local Rearrange[tuple]{int}(false) - scope-72 -> scope-52 +| | | +| | Project[int][0] - scope-74 +| | +| |---c: New For Each(false,false)[bag] - scope-61 +| | | +| | Project[int][0] - scope-62 +| | | +| | POUserFunc(org.apache.pig.builtin.COUNT$Initial)[tuple] - scope-63 +| | | +| | |---Project[bag][1] - scope-64 +| | +| |---Pre Combiner Local Rearrange[tuple]{Unknown} - scope-75 +| | +| POValueOutputTez - scope-49 -> [scope-53] +| +|---a: New For Each(false,false)[bag] - scope-7 + | | + | Cast[int] - scope-2 + | | + | |---Project[bytearray][0] - scope-1 + | | + | Cast[int] - scope-5 + | | + | |---Project[bytearray][1] - scope-4 + | + |---a: Load(file:///tmp/input:org.apache.pig.builtin.PigStorage) - scope-0 +Tez vertex scope-52 +# Combine plan on edge +b: Local Rearrange[tuple]{int}(false) - scope-76 -> scope-52 +| | +| Project[int][0] - scope-78 +| +|---c: New For Each(false,false)[bag] - scope-65 + | | + | Project[int][0] - scope-66 + | | + | POUserFunc(org.apache.pig.builtin.COUNT$Intermediate)[tuple] - scope-67 + | | + | |---Project[bag][1] - scope-68 + | + |---b: Package(CombinerPackager)[tuple]{int} - scope-71 +# Plan on vertex +POValueOutputTez - scope-60 -> [scope-53, scope-58] +| +|---c: New For Each(false,false)[bag] - scope-23 + | | + | Project[int][0] - scope-18 + | | + | POUserFunc(org.apache.pig.builtin.COUNT$Final)[long] - scope-21 + | | + | |---Project[bag][1] - scope-69 + | + |---b: Package(CombinerPackager)[tuple]{int} - scope-15 +Tez vertex scope-53 +# Plan on vertex +1-1: Split - scope-79 +| | +| d: Store(file:///tmp/output1:org.apache.pig.builtin.PigStorage) - scope-35 +| | +| |---d: Filter[bag] - scope-25 +| | | +| | Less Than[boolean] - scope-34 +| | | +| | |---Multiply[long] - scope-31 +| | | | +| | | |---Cast[long] - scope-27 +| | | | | +| | | | |---Constant(2) - scope-26 +| | | | +| | | |---POUserFunc(org.apache.pig.backend.hadoop.executionengine.tez.plan.udf.ReadScalarsTez)[long] - scope-30 +| | | | +| | | |---Constant(1) - scope-28 +| | | +| | |---Cast[long] - scope-33 +| | | +| | |---Project[int][1] - scope-32 +| | +| POValueOutputTez - scope-55 -> [scope-58] +| +|---POValueInputTez - scope-54 <- scope-48 +Tez vertex scope-58 +# Plan on vertex +e: Store(file:///tmp/output2:org.apache.pig.builtin.PigStorage) - scope-47 +| +|---e: Filter[bag] - scope-36 + | | + | Not[boolean] - scope-46 + | | + | |---Less Than[boolean] - scope-45 + | | + | |---Multiply[long] - scope-42 + | | | + | | |---Cast[long] - scope-38 + | | | | + | | | |---Constant(2) - scope-37 + | | | + | | |---POUserFunc(org.apache.pig.backend.hadoop.executionengine.tez.plan.udf.ReadScalarsTez)[long] - scope-41 + | | | + | | |---Constant(1) - scope-39 + | | + | |---Cast[long] - scope-44 + | | + | |---Project[int][1] - scope-43 + | + |---POValueInputTez - scope-59 <- scope-53 Added: pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-MQ-7-OPTOFF.gld URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-MQ-7-OPTOFF.gld?rev=1671973&view=auto ============================================================================== --- pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-MQ-7-OPTOFF.gld (added) +++ pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-MQ-7-OPTOFF.gld Tue Apr 7 23:21:01 2015 @@ -0,0 +1,82 @@ +#-------------------------------------------------- +# There are 1 DAGs in the session +#-------------------------------------------------- +#-------------------------------------------------- +# TEZ DAG plan: pig-0_scope-0 +#-------------------------------------------------- +Tez vertex scope-44 -> Tez vertex scope-46,Tez vertex scope-52, +Tez vertex scope-48 -> Tez vertex scope-50,Tez vertex scope-54, +Tez vertex scope-50 -> Tez vertex scope-46, +Tez vertex scope-46 -> Tez vertex group scope-60, +Tez vertex scope-54 -> Tez vertex scope-52, +Tez vertex scope-52 -> Tez vertex group scope-60, +Tez vertex group scope-60 + +Tez vertex scope-44 +# Plan on vertex +POValueOutputTez - scope-45 -> [scope-46, scope-52] +| +|---a: New For Each(false,false)[bag] - scope-7 + | | + | Cast[int] - scope-2 + | | + | |---Project[bytearray][0] - scope-1 + | | + | Cast[int] - scope-5 + | | + | |---Project[bytearray][1] - scope-4 + | + |---a: Load(file:///tmp/input:org.apache.pig.builtin.PigStorage) - scope-0 +Tez vertex scope-48 +# Plan on vertex +POValueOutputTez - scope-49 -> [scope-50, scope-54] +| +|---b: New For Each(false,false)[bag] - scope-18 + | | + | Cast[int] - scope-13 + | | + | |---Project[bytearray][0] - scope-12 + | | + | Cast[int] - scope-16 + | | + | |---Project[bytearray][1] - scope-15 + | + |---b: Load(file:///tmp/input:org.apache.pig.builtin.PigStorage) - scope-11 +Tez vertex scope-50 +# Plan on vertex +Local Rearrange[tuple]{int}(false) - scope-27 -> scope-46 +| | +| Project[int][0] - scope-23 +| +|---POValueInputTez - scope-51 <- scope-48 +Tez vertex scope-46 +# Plan on vertex +e: Store(file:///tmp/output:org.apache.pig.builtin.PigStorage) - scope-61 -> scope-43 +| +|---c: FRJoin[tuple] - scope-24 <- scope-50 + | | + | Project[int][0] - scope-22 + | | + | Project[int][0] - scope-23 + | + |---POValueInputTez - scope-47 <- scope-44 +Tez vertex scope-54 +# Plan on vertex +Local Rearrange[tuple]{int}(false) - scope-39 -> scope-52 +| | +| Project[int][1] - scope-35 +| +|---POValueInputTez - scope-55 <- scope-48 +Tez vertex scope-52 +# Plan on vertex +e: Store(file:///tmp/output:org.apache.pig.builtin.PigStorage) - scope-62 -> scope-43 +| +|---d: FRJoin[tuple] - scope-36 <- scope-54 + | | + | Project[int][1] - scope-34 + | | + | Project[int][1] - scope-35 + | + |---POValueInputTez - scope-53 <- scope-44 +Tez vertex group scope-60 <- [scope-46, scope-52] -> null +# No plan on vertex group Added: pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-MQ-7.gld URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-MQ-7.gld?rev=1671973&view=auto ============================================================================== --- pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-MQ-7.gld (added) +++ pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-MQ-7.gld Tue Apr 7 23:21:01 2015 @@ -0,0 +1,72 @@ +#-------------------------------------------------- +# There are 1 DAGs in the session +#-------------------------------------------------- +#-------------------------------------------------- +# TEZ DAG plan: pig-0_scope-0 +#-------------------------------------------------- +Tez vertex scope-48 -> Tez vertex scope-44,Tez vertex scope-52, +Tez vertex scope-44 -> Tez vertex scope-52,Tez vertex group scope-62, +Tez vertex scope-52 -> Tez vertex group scope-62, +Tez vertex group scope-62 + +Tez vertex scope-48 +# Plan on vertex +b: Split - scope-61 +| | +| Local Rearrange[tuple]{int}(false) - scope-27 -> scope-44 +| | | +| | Project[int][0] - scope-23 +| | +| Local Rearrange[tuple]{int}(false) - scope-39 -> scope-52 +| | | +| | Project[int][1] - scope-35 +| +|---b: New For Each(false,false)[bag] - scope-18 + | | + | Cast[int] - scope-13 + | | + | |---Project[bytearray][0] - scope-12 + | | + | Cast[int] - scope-16 + | | + | |---Project[bytearray][1] - scope-15 + | + |---b: Load(file:///tmp/input:org.apache.pig.builtin.PigStorage) - scope-11 +Tez vertex scope-44 +# Plan on vertex +a: Split - scope-60 +| | +| e: Store(file:///tmp/output:org.apache.pig.builtin.PigStorage) - scope-63 -> scope-43 +| | +| |---c: FRJoin[tuple] - scope-24 <- scope-48 +| | | +| | Project[int][0] - scope-22 +| | | +| | Project[int][0] - scope-23 +| | +| POValueOutputTez - scope-45 -> [scope-52] +| +|---a: New For Each(false,false)[bag] - scope-7 + | | + | Cast[int] - scope-2 + | | + | |---Project[bytearray][0] - scope-1 + | | + | Cast[int] - scope-5 + | | + | |---Project[bytearray][1] - scope-4 + | + |---a: Load(file:///tmp/input:org.apache.pig.builtin.PigStorage) - scope-0 +Tez vertex scope-52 +# Plan on vertex +e: Store(file:///tmp/output:org.apache.pig.builtin.PigStorage) - scope-64 -> scope-43 +| +|---d: FRJoin[tuple] - scope-36 <- scope-48 + | | + | Project[int][1] - scope-34 + | | + | Project[int][1] - scope-35 + | + |---POValueInputTez - scope-53 <- scope-44 +Tez vertex group scope-62 <- [scope-44, scope-52] -> null +# No plan on vertex group Added: pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-SelfJoin-1.gld URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-SelfJoin-1.gld?rev=1671973&view=auto ============================================================================== --- pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-SelfJoin-1.gld (added) +++ pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-SelfJoin-1.gld Tue Apr 7 23:21:01 2015 @@ -0,0 +1,73 @@ +#-------------------------------------------------- +# There are 1 DAGs in the session +#-------------------------------------------------- +#-------------------------------------------------- +# TEZ DAG plan: pig-0_scope-0 +#-------------------------------------------------- +Tez vertex scope-43 -> Tez vertex scope-51, +Tez vertex scope-51 + +Tez vertex scope-43 +# Plan on vertex +a: Split - scope-52 +| | +| e: Local Rearrange[tuple]{int}(false) - scope-32 -> scope-51 +| | | +| | Project[int][0] - scope-33 +| | +| |---b: Filter[bag] - scope-11 +| | | +| | Less Than[boolean] - scope-14 +| | | +| | |---Project[int][0] - scope-12 +| | | +| | |---Constant(5) - scope-13 +| | +| e: Local Rearrange[tuple]{int}(false) - scope-34 -> scope-51 +| | | +| | Project[int][0] - scope-35 +| | +| |---c: Filter[bag] - scope-17 +| | | +| | Equal To[boolean] - scope-20 +| | | +| | |---Project[int][0] - scope-18 +| | | +| | |---Constant(10) - scope-19 +| | +| e: Local Rearrange[tuple]{int}(false) - scope-36 -> scope-51 +| | | +| | Project[int][0] - scope-37 +| | +| |---d: Filter[bag] - scope-23 +| | | +| | Greater Than[boolean] - scope-26 +| | | +| | |---Project[int][0] - scope-24 +| | | +| | |---Constant(10) - scope-25 +| +|---a: New For Each(false,false)[bag] - scope-7 + | | + | Cast[int] - scope-2 + | | + | |---Project[bytearray][0] - scope-1 + | | + | Cast[int] - scope-5 + | | + | |---Project[bytearray][1] - scope-4 + | + |---a: Load(file:///tmp/input1:org.apache.pig.builtin.PigStorage) - scope-0 +Tez vertex scope-51 +# Plan on vertex +e: Store(file:///tmp/output:org.apache.pig.builtin.PigStorage) - scope-42 +| +|---e: New For Each(true,true,true)[tuple] - scope-41 + | | + | Project[bag][1] - scope-38 + | | + | Project[bag][2] - scope-39 + | | + | Project[bag][3] - scope-40 + | + |---e: Package(Packager)[tuple]{int} - scope-31 Added: pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-SelfJoin-2.gld URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-SelfJoin-2.gld?rev=1671973&view=auto ============================================================================== --- pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-SelfJoin-2.gld (added) +++ pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-SelfJoin-2.gld Tue Apr 7 23:21:01 2015 @@ -0,0 +1,103 @@ +#-------------------------------------------------- +# There are 1 DAGs in the session +#-------------------------------------------------- +#-------------------------------------------------- +# TEZ DAG plan: pig-0_scope-0 +#-------------------------------------------------- +Tez vertex scope-25 -> Tez vertex scope-29,Tez vertex scope-38,Tez vertex scope-48, +Tez vertex scope-38 -> Tez vertex scope-29,Tez vertex scope-48, +Tez vertex scope-48 -> Tez vertex scope-52, +Tez vertex scope-29 -> Tez vertex scope-52, +Tez vertex scope-52 + +Tez vertex scope-25 +# Plan on vertex +a: Split - scope-58 +| | +| Local Rearrange[tuple]{tuple}(false) - scope-33 -> scope-38 +| | | +| | Constant(DummyVal) - scope-32 +| | +| |---New For Each(true,true)[tuple] - scope-37 +| | | +| | Project[int][0] - scope-21 +| | | +| | POUserFunc(org.apache.pig.impl.builtin.GetMemNumRows)[tuple] - scope-36 +| | | +| | |---Project[tuple][*] - scope-35 +| | +| |---PoissonSample - scope-34 +| | +| |---Local Rearrange[tuple]{int}(false) - scope-31 -> scope-48 +| | | +| | Project[int][0] - scope-21 +| | +| |---b: Filter[bag] - scope-11 +| | | +| | Less Than[boolean] - scope-14 +| | | +| | |---Project[int][0] - scope-12 +| | | +| | |---Constant(5) - scope-13 +| | +| POValueOutputTez - scope-26 -> [scope-29] +| +|---a: New For Each(false,false)[bag] - scope-7 + | | + | Cast[int] - scope-2 + | | + | |---Project[bytearray][0] - scope-1 + | | + | Cast[int] - scope-5 + | | + | |---Project[bytearray][1] - scope-4 + | + |---a: Load(file:///tmp/input1:org.apache.pig.builtin.PigStorage) - scope-0 +Tez vertex scope-38 +# Plan on vertex +POValueOutputTez - scope-47 -> [scope-29, scope-48] +| +|---New For Each(false)[tuple] - scope-46 + | | + | POUserFunc(org.apache.pig.backend.hadoop.executionengine.tez.plan.udf.PartitionSkewedKeysTez)[tuple] - scope-45 + | | + | |---Project[tuple][*] - scope-44 + | + |---New For Each(false,false)[tuple] - scope-43 + | | + | Constant(-1) - scope-42 + | | + | Project[bag][1] - scope-40 + | + |---Package(Packager)[tuple]{bytearray} - scope-39 +Tez vertex scope-48 +# Plan on vertex +POIdentityInOutTez - scope-49 <- scope-25 -> scope-52 +| | +| Project[int][0] - scope-21 +Tez vertex scope-29 +# Plan on vertex +Partition Rearrange[tuple]{int}(false) - scope-50 -> scope-52 +| | +| Project[int][0] - scope-22 +| +|---c: Filter[bag] - scope-17 + | | + | Equal To[boolean] - scope-20 + | | + | |---Project[int][0] - scope-18 + | | + | |---Constant(10) - scope-19 + | + |---POValueInputTez - scope-30 <- scope-25 +Tez vertex scope-52 +# Plan on vertex +d: Store(file:///tmp/output:org.apache.pig.builtin.PigStorage) - scope-24 +| +|---New For Each(true,true)[tuple] - scope-56 + | | + | Project[bag][1] - scope-54 + | | + | Project[bag][2] - scope-55 + | + |---Package(Packager)[tuple]{int} - scope-53 Added: pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-SelfJoin-3.gld URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-SelfJoin-3.gld?rev=1671973&view=auto ============================================================================== --- pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-SelfJoin-3.gld (added) +++ pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-SelfJoin-3.gld Tue Apr 7 23:21:01 2015 @@ -0,0 +1,77 @@ +#-------------------------------------------------- +# There are 1 DAGs in the session +#-------------------------------------------------- +#-------------------------------------------------- +# TEZ DAG plan: pig-0_scope-0 +#-------------------------------------------------- +Tez vertex scope-39 -> Tez vertex scope-41,Tez vertex scope-43,Tez vertex scope-45, +Tez vertex scope-43 -> Tez vertex scope-41, +Tez vertex scope-45 -> Tez vertex scope-41, +Tez vertex scope-41 + +Tez vertex scope-39 +# Plan on vertex +POValueOutputTez - scope-40 -> [scope-41, scope-43, scope-45] +| +|---a: New For Each(false,false)[bag] - scope-7 + | | + | Cast[int] - scope-2 + | | + | |---Project[bytearray][0] - scope-1 + | | + | Cast[int] - scope-5 + | | + | |---Project[bytearray][1] - scope-4 + | + |---a: Load(file:///tmp/input1:org.apache.pig.builtin.PigStorage) - scope-0 +Tez vertex scope-43 +# Plan on vertex +Local Rearrange[tuple]{int}(false) - scope-33 -> scope-41 +| | +| Project[int][0] - scope-28 +| +|---c: Filter[bag] - scope-17 + | | + | Equal To[boolean] - scope-20 + | | + | |---Project[int][0] - scope-18 + | | + | |---Constant(10) - scope-19 + | + |---POValueInputTez - scope-44 <- scope-39 +Tez vertex scope-45 +# Plan on vertex +Local Rearrange[tuple]{int}(false) - scope-35 -> scope-41 +| | +| Project[int][0] - scope-29 +| +|---d: Filter[bag] - scope-23 + | | + | Greater Than[boolean] - scope-26 + | | + | |---Project[int][0] - scope-24 + | | + | |---Constant(10) - scope-25 + | + |---POValueInputTez - scope-46 <- scope-39 +Tez vertex scope-41 +# Plan on vertex +e: Store(file:///tmp/output:org.apache.pig.builtin.PigStorage) - scope-38 +| +|---e: FRJoin[tuple] - scope-30 <- scope-43,scope-45 + | | + | Project[int][0] - scope-27 + | | + | Project[int][0] - scope-28 + | | + | Project[int][0] - scope-29 + | + |---b: Filter[bag] - scope-11 + | | + | Less Than[boolean] - scope-14 + | | + | |---Project[int][0] - scope-12 + | | + | |---Constant(5) - scope-13 + | + |---POValueInputTez - scope-42 <- scope-39 Modified: pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-1.gld URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-1.gld?rev=1671973&r1=1671972&r2=1671973&view=diff ============================================================================== --- pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-1.gld (original) +++ pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-1.gld Tue Apr 7 23:21:01 2015 @@ -10,7 +10,7 @@ Tez vertex group scope-24 Tez vertex scope-18 # Plan on vertex -c: Store(file:///tmp/output:org.apache.pig.builtin.PigStorage) - scope-25 +c: Store(file:///tmp/output:org.apache.pig.builtin.PigStorage) - scope-25 -> scope-17 | |---a: New For Each(false,false)[bag] - scope-7 | | @@ -25,7 +25,7 @@ c: Store(file:///tmp/output:org.apache.p |---a: Load(file:///tmp/input:org.apache.pig.builtin.PigStorage) - scope-0 Tez vertex scope-19 # Plan on vertex -c: Store(file:///tmp/output:org.apache.pig.builtin.PigStorage) - scope-26 +c: Store(file:///tmp/output:org.apache.pig.builtin.PigStorage) - scope-26 -> scope-17 | |---c: New For Each(false,false)[bag] - scope-15 | | Added: pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-11-OPTOFF.gld URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-11-OPTOFF.gld?rev=1671973&view=auto ============================================================================== --- pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-11-OPTOFF.gld (added) +++ pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-11-OPTOFF.gld Tue Apr 7 23:21:01 2015 @@ -0,0 +1,67 @@ +#-------------------------------------------------- +# There are 1 DAGs in the session +#-------------------------------------------------- +#-------------------------------------------------- +# TEZ DAG plan: pig-0_scope-0 +#-------------------------------------------------- +Tez vertex scope-27 -> Tez vertex scope-29, +Tez vertex scope-28 -> Tez vertex scope-29, +Tez vertex scope-29 -> Tez vertex scope-34, +Tez vertex scope-33 -> Tez vertex scope-34, +Tez vertex scope-34 + +Tez vertex scope-27 +# Plan on vertex +POValueOutputTez - scope-31 -> [scope-29] +| +|---a: New For Each(false,false)[bag] - scope-7 + | | + | Cast[int] - scope-2 + | | + | |---Project[bytearray][0] - scope-1 + | | + | Cast[chararray] - scope-5 + | | + | |---Project[bytearray][1] - scope-4 + | + |---a: Load(file:///tmp/input:org.apache.pig.builtin.PigStorage) - scope-0 +Tez vertex scope-28 +# Plan on vertex +POValueOutputTez - scope-32 -> [scope-29] +| +|---c: New For Each(false,false)[bag] - scope-15 + | | + | Cast[int] - scope-10 + | | + | |---Project[bytearray][1] - scope-9 + | | + | Cast[chararray] - scope-13 + | | + | |---Project[bytearray][0] - scope-12 + | + |---b: Load(file:///tmp/input:org.apache.pig.builtin.PigStorage) - scope-8 +Tez vertex scope-29 +# Plan on vertex +POValueOutputTez - scope-36 -> [scope-34] +| +|---POShuffledValueInputTez - scope-30 <- [scope-27, scope-28] +Tez vertex scope-33 +# Plan on vertex +POValueOutputTez - scope-37 -> [scope-34] +| +|---d: New For Each(false,false)[bag] - scope-24 + | | + | Cast[int] - scope-19 + | | + | |---Project[bytearray][0] - scope-18 + | | + | Cast[chararray] - scope-22 + | | + | |---Project[bytearray][1] - scope-21 + | + |---d: Load(file:///tmp/input1:org.apache.pig.builtin.PigStorage) - scope-17 +Tez vertex scope-34 +# Plan on vertex +e: Store(file:///tmp/output:org.apache.pig.builtin.PigStorage) - scope-26 +| +|---POShuffledValueInputTez - scope-35 <- [scope-29, scope-33] Added: pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-11.gld URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-11.gld?rev=1671973&view=auto ============================================================================== --- pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-11.gld (added) +++ pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-11.gld Tue Apr 7 23:21:01 2015 @@ -0,0 +1,58 @@ +#-------------------------------------------------- +# There are 1 DAGs in the session +#-------------------------------------------------- +#-------------------------------------------------- +# TEZ DAG plan: pig-0_scope-0 +#-------------------------------------------------- +Tez vertex scope-33 -> Tez vertex group scope-38, +Tez vertex scope-27 -> Tez vertex group scope-38, +Tez vertex scope-28 -> Tez vertex group scope-38, +Tez vertex group scope-38 + +Tez vertex scope-33 +# Plan on vertex +e: Store(file:///tmp/output:org.apache.pig.builtin.PigStorage) - scope-40 -> scope-26 +| +|---d: New For Each(false,false)[bag] - scope-24 + | | + | Cast[int] - scope-19 + | | + | |---Project[bytearray][0] - scope-18 + | | + | Cast[chararray] - scope-22 + | | + | |---Project[bytearray][1] - scope-21 + | + |---d: Load(file:///tmp/input1:org.apache.pig.builtin.PigStorage) - scope-17 +Tez vertex scope-27 +# Plan on vertex +e: Store(file:///tmp/output:org.apache.pig.builtin.PigStorage) - scope-41 -> scope-26 +| +|---a: New For Each(false,false)[bag] - scope-7 + | | + | Cast[int] - scope-2 + | | + | |---Project[bytearray][0] - scope-1 + | | + | Cast[chararray] - scope-5 + | | + | |---Project[bytearray][1] - scope-4 + | + |---a: Load(file:///tmp/input:org.apache.pig.builtin.PigStorage) - scope-0 +Tez vertex scope-28 +# Plan on vertex +e: Store(file:///tmp/output:org.apache.pig.builtin.PigStorage) - scope-42 -> scope-26 +| +|---c: New For Each(false,false)[bag] - scope-15 + | | + | Cast[int] - scope-10 + | | + | |---Project[bytearray][1] - scope-9 + | | + | Cast[chararray] - scope-13 + | | + | |---Project[bytearray][0] - scope-12 + | + |---b: Load(file:///tmp/input:org.apache.pig.builtin.PigStorage) - scope-8 +Tez vertex group scope-38 <- [scope-29, scope-33, scope-27, scope-28] -> null +# No plan on vertex group Added: pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-12-OPTOFF.gld URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-12-OPTOFF.gld?rev=1671973&view=auto ============================================================================== --- pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-12-OPTOFF.gld (added) +++ pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-12-OPTOFF.gld Tue Apr 7 23:21:01 2015 @@ -0,0 +1,143 @@ +#-------------------------------------------------- +# There are 1 DAGs in the session +#-------------------------------------------------- +#-------------------------------------------------- +# TEZ DAG plan: pig-0_scope-0 +#-------------------------------------------------- +Tez vertex scope-79 -> Tez vertex scope-86,Tez vertex scope-88,Tez vertex scope-97,Tez vertex scope-99, +Tez vertex scope-86 -> Tez vertex scope-88, +Tez vertex scope-88 -> Tez vertex scope-103, +Tez vertex scope-97 -> Tez vertex scope-99, +Tez vertex scope-99 -> Tez vertex scope-103, +Tez vertex scope-103 -> Tez vertex scope-108, +Tez vertex scope-107 -> Tez vertex scope-108, +Tez vertex scope-108 + +Tez vertex scope-79 +# Plan on vertex +a: Split - scope-111 +| | +| b: Split - scope-109 +| | | +| | POValueOutputTez - scope-90 -> [scope-88] +| | | +| | |---b1: New For Each(false,false)[bag] - scope-22 +| | | | +| | | Project[int][0] - scope-18 +| | | | +| | | Project[chararray][1] - scope-20 +| | | +| | POValueOutputTez - scope-83 -> [scope-86] +| | +| |---b: Filter[bag] - scope-11 +| | | +| | Equal To[boolean] - scope-14 +| | | +| | |---Project[int][0] - scope-12 +| | | +| | |---Constant(2) - scope-13 +| | +| c: Split - scope-110 +| | | +| | POValueOutputTez - scope-101 -> [scope-99] +| | | +| | |---c1: New For Each(false,false)[bag] - scope-44 +| | | | +| | | Project[chararray][1] - scope-40 +| | | | +| | | Project[int][0] - scope-42 +| | | +| | POValueOutputTez - scope-94 -> [scope-97] +| | +| |---c: Filter[bag] - scope-33 +| | | +| | Equal To[boolean] - scope-36 +| | | +| | |---Project[int][0] - scope-34 +| | | +| | |---Constant(3) - scope-35 +| +|---a: New For Each(false,false)[bag] - scope-7 + | | + | Cast[int] - scope-2 + | | + | |---Project[bytearray][0] - scope-1 + | | + | Cast[chararray] - scope-5 + | | + | |---Project[bytearray][1] - scope-4 + | + |---a: Load(file:///tmp/input:org.apache.pig.builtin.PigStorage) - scope-0 +Tez vertex scope-86 +# Plan on vertex +POValueOutputTez - scope-91 -> [scope-88] +| +|---b2: New For Each(false,false)[bag] - scope-29 + | | + | Project[int][0] - scope-25 + | | + | Project[chararray][1] - scope-27 + | + |---POValueInputTez - scope-87 <- scope-79 +Tez vertex scope-88 +# Plan on vertex +POValueOutputTez - scope-105 -> [scope-103] +| +|---POShuffledValueInputTez - scope-89 <- [scope-79, scope-86] +Tez vertex scope-97 +# Plan on vertex +POValueOutputTez - scope-102 -> [scope-99] +| +|---c2: New For Each(false,false)[bag] - scope-51 + | | + | Project[chararray][1] - scope-47 + | | + | Project[int][0] - scope-49 + | + |---POValueInputTez - scope-98 <- scope-79 +Tez vertex scope-99 +# Plan on vertex +POValueOutputTez - scope-106 -> [scope-103] +| +|---a1: New For Each(false,false)[bag] - scope-57 + | | + | Project[int][1] - scope-53 + | | + | Project[chararray][0] - scope-55 + | + |---POShuffledValueInputTez - scope-100 <- [scope-79, scope-97] +Tez vertex scope-103 +# Plan on vertex +e: Local Rearrange[tuple]{int}(false) - scope-71 -> scope-108 +| | +| Project[int][0] - scope-72 +| +|---POShuffledValueInputTez - scope-104 <- [scope-88, scope-99] +Tez vertex scope-107 +# Plan on vertex +e: Local Rearrange[tuple]{int}(false) - scope-73 -> scope-108 +| | +| Project[int][0] - scope-74 +| +|---d: New For Each(false,false)[bag] - scope-66 + | | + | Cast[int] - scope-61 + | | + | |---Project[bytearray][0] - scope-60 + | | + | Cast[chararray] - scope-64 + | | + | |---Project[bytearray][1] - scope-63 + | + |---d: Load(file:///tmp/input1:org.apache.pig.builtin.PigStorage) - scope-59 +Tez vertex scope-108 +# Plan on vertex +e: Store(file:///tmp/output:org.apache.pig.builtin.PigStorage) - scope-78 +| +|---e: New For Each(true,true)[tuple] - scope-77 + | | + | Project[bag][1] - scope-75 + | | + | Project[bag][2] - scope-76 + | + |---e: Package(Packager)[tuple]{int} - scope-70 Added: pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-12.gld URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-12.gld?rev=1671973&view=auto ============================================================================== --- pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-12.gld (added) +++ pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-12.gld Tue Apr 7 23:21:01 2015 @@ -0,0 +1,125 @@ +#-------------------------------------------------- +# There are 1 DAGs in the session +#-------------------------------------------------- +#-------------------------------------------------- +# TEZ DAG plan: pig-0_scope-0 +#-------------------------------------------------- +Tez vertex scope-107 -> Tez vertex scope-108, +Tez vertex scope-79 -> Tez vertex scope-108, +Tez vertex scope-108 + +Tez vertex scope-107 +# Plan on vertex +e: Local Rearrange[tuple]{int}(false) - scope-73 -> scope-108 +| | +| Project[int][0] - scope-74 +| +|---d: New For Each(false,false)[bag] - scope-66 + | | + | Cast[int] - scope-61 + | | + | |---Project[bytearray][0] - scope-60 + | | + | Cast[chararray] - scope-64 + | | + | |---Project[bytearray][1] - scope-63 + | + |---d: Load(file:///tmp/input1:org.apache.pig.builtin.PigStorage) - scope-59 +Tez vertex scope-79 +# Plan on vertex +a: Split - scope-111 +| | +| b: Split - scope-109 +| | | +| | e: Local Rearrange[tuple]{int}(false) - scope-117 -> scope-108 +| | | | +| | | Project[int][0] - scope-118 +| | | +| | |---b1: New For Each(false,false)[bag] - scope-22 +| | | | +| | | Project[int][0] - scope-18 +| | | | +| | | Project[chararray][1] - scope-20 +| | | +| | e: Local Rearrange[tuple]{int}(false) - scope-119 -> scope-108 +| | | | +| | | Project[int][0] - scope-120 +| | | +| | |---b2: New For Each(false,false)[bag] - scope-29 +| | | | +| | | Project[int][0] - scope-25 +| | | | +| | | Project[chararray][1] - scope-27 +| | +| |---b: Filter[bag] - scope-11 +| | | +| | Equal To[boolean] - scope-14 +| | | +| | |---Project[int][0] - scope-12 +| | | +| | |---Constant(2) - scope-13 +| | +| c: Split - scope-110 +| | | +| | e: Local Rearrange[tuple]{int}(false) - scope-124 -> scope-108 +| | | | +| | | Project[int][0] - scope-125 +| | | +| | |---a1: New For Each(false,false)[bag] - scope-123 +| | | | +| | | Project[int][1] - scope-121 +| | | | +| | | Project[chararray][0] - scope-122 +| | | +| | |---c1: New For Each(false,false)[bag] - scope-44 +| | | | +| | | Project[chararray][1] - scope-40 +| | | | +| | | Project[int][0] - scope-42 +| | | +| | e: Local Rearrange[tuple]{int}(false) - scope-129 -> scope-108 +| | | | +| | | Project[int][0] - scope-130 +| | | +| | |---a1: New For Each(false,false)[bag] - scope-128 +| | | | +| | | Project[int][1] - scope-126 +| | | | +| | | Project[chararray][0] - scope-127 +| | | +| | |---c2: New For Each(false,false)[bag] - scope-51 +| | | | +| | | Project[chararray][1] - scope-47 +| | | | +| | | Project[int][0] - scope-49 +| | +| |---c: Filter[bag] - scope-33 +| | | +| | Equal To[boolean] - scope-36 +| | | +| | |---Project[int][0] - scope-34 +| | | +| | |---Constant(3) - scope-35 +| +|---a: New For Each(false,false)[bag] - scope-7 + | | + | Cast[int] - scope-2 + | | + | |---Project[bytearray][0] - scope-1 + | | + | Cast[chararray] - scope-5 + | | + | |---Project[bytearray][1] - scope-4 + | + |---a: Load(file:///tmp/input:org.apache.pig.builtin.PigStorage) - scope-0 +Tez vertex scope-108 +# Plan on vertex +e: Store(file:///tmp/output:org.apache.pig.builtin.PigStorage) - scope-78 +| +|---e: New For Each(true,true)[tuple] - scope-77 + | | + | Project[bag][1] - scope-75 + | | + | Project[bag][2] - scope-76 + | + |---e: Package(Packager)[tuple]{int} - scope-70 Added: pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-13-OPTOFF.gld URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-13-OPTOFF.gld?rev=1671973&view=auto ============================================================================== --- pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-13-OPTOFF.gld (added) +++ pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-13-OPTOFF.gld Tue Apr 7 23:21:01 2015 @@ -0,0 +1,71 @@ +#-------------------------------------------------- +# There are 1 DAGs in the session +#-------------------------------------------------- +#-------------------------------------------------- +# TEZ DAG plan: pig-0_scope-0 +#-------------------------------------------------- +Tez vertex scope-35 -> Tez vertex scope-39,Tez vertex scope-41, +Tez vertex scope-39 -> Tez vertex scope-41, +Tez vertex scope-45 -> Tez vertex scope-41, +Tez vertex scope-41 + +Tez vertex scope-35 +# Plan on vertex +a: Split - scope-46 +| | +| POValueOutputTez - scope-43 -> [scope-41] +| | +| POValueOutputTez - scope-36 -> [scope-39] +| +|---a: New For Each(false,false)[bag] - scope-7 + | | + | Cast[int] - scope-2 + | | + | |---Project[bytearray][0] - scope-1 + | | + | Cast[chararray] - scope-5 + | | + | |---Project[bytearray][1] - scope-4 + | + |---a: Load(file:///tmp/input:org.apache.pig.builtin.PigStorage) - scope-0 +Tez vertex scope-39 +# Plan on vertex +POValueOutputTez - scope-44 -> [scope-41] +| +|---b: Filter[bag] - scope-13 + | | + | Equal To[boolean] - scope-16 + | | + | |---Project[int][0] - scope-14 + | | + | |---Constant(2) - scope-15 + | + |---POValueInputTez - scope-40 <- scope-35 +Tez vertex scope-45 +# Plan on vertex +Local Rearrange[tuple]{int}(false) - scope-31 -> scope-41 +| | +| Project[int][0] - scope-27 +| +|---d: New For Each(false,false)[bag] - scope-25 + | | + | Cast[int] - scope-20 + | | + | |---Project[bytearray][0] - scope-19 + | | + | Cast[chararray] - scope-23 + | | + | |---Project[bytearray][1] - scope-22 + | + |---d: Load(file:///tmp/input1:org.apache.pig.builtin.PigStorage) - scope-18 +Tez vertex scope-41 +# Plan on vertex +e: Store(file:///tmp/output:org.apache.pig.builtin.PigStorage) - scope-34 +| +|---e: FRJoin[tuple] - scope-28 <- scope-45 + | | + | Project[int][0] - scope-26 + | | + | Project[int][0] - scope-27 + | + |---POShuffledValueInputTez - scope-42 <- [scope-35, scope-39] Added: pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-13.gld URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-13.gld?rev=1671973&view=auto ============================================================================== --- pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-13.gld (added) +++ pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-13.gld Tue Apr 7 23:21:01 2015 @@ -0,0 +1,65 @@ +#-------------------------------------------------- +# There are 1 DAGs in the session +#-------------------------------------------------- +#-------------------------------------------------- +# TEZ DAG plan: pig-0_scope-0 +#-------------------------------------------------- +Tez vertex scope-45 -> Tez vertex scope-35, +Tez vertex scope-35 + +Tez vertex scope-45 +# Plan on vertex +Local Rearrange[tuple]{int}(false) - scope-31 -> scope-35 +| | +| Project[int][0] - scope-27 +| +|---d: New For Each(false,false)[bag] - scope-25 + | | + | Cast[int] - scope-20 + | | + | |---Project[bytearray][0] - scope-19 + | | + | Cast[chararray] - scope-23 + | | + | |---Project[bytearray][1] - scope-22 + | + |---d: Load(file:///tmp/input1:org.apache.pig.builtin.PigStorage) - scope-18 +Tez vertex scope-35 +# Plan on vertex +a: Split - scope-46 +| | +| e: Store(file:///tmp/output:org.apache.pig.builtin.PigStorage) - scope-48 -> scope-34 +| | +| |---e: FRJoin[tuple] - scope-47 <- scope-45 +| | | +| | Project[int][0] - scope-26 +| | | +| | Project[int][0] - scope-27 +| | +| e: Store(file:///tmp/output:org.apache.pig.builtin.PigStorage) - scope-50 -> scope-34 +| | +| |---e: FRJoin[tuple] - scope-49 <- scope-45 +| | | +| | Project[int][0] - scope-26 +| | | +| | Project[int][0] - scope-27 +| | +| |---b: Filter[bag] - scope-13 +| | | +| | Equal To[boolean] - scope-16 +| | | +| | |---Project[int][0] - scope-14 +| | | +| | |---Constant(2) - scope-15 +| +|---a: New For Each(false,false)[bag] - scope-7 + | | + | Cast[int] - scope-2 + | | + | |---Project[bytearray][0] - scope-1 + | | + | Cast[chararray] - scope-5 + | | + | |---Project[bytearray][1] - scope-4 + | + |---a: Load(file:///tmp/input:org.apache.pig.builtin.PigStorage) - scope-0 Added: pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-14-OPTOFF.gld URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-14-OPTOFF.gld?rev=1671973&view=auto ============================================================================== --- pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-14-OPTOFF.gld (added) +++ pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-14-OPTOFF.gld Tue Apr 7 23:21:01 2015 @@ -0,0 +1,71 @@ +#-------------------------------------------------- +# There are 1 DAGs in the session +#-------------------------------------------------- +#-------------------------------------------------- +# TEZ DAG plan: pig-0_scope-0 +#-------------------------------------------------- +Tez vertex scope-36 -> Tez vertex scope-40,Tez vertex scope-42, +Tez vertex scope-40 -> Tez vertex scope-42, +Tez vertex scope-42 -> Tez vertex scope-35, +Tez vertex scope-35 + +Tez vertex scope-36 +# Plan on vertex +a: Split - scope-46 +| | +| POValueOutputTez - scope-44 -> [scope-42] +| | +| POValueOutputTez - scope-37 -> [scope-40] +| +|---a: New For Each(false,false)[bag] - scope-15 + | | + | Cast[int] - scope-10 + | | + | |---Project[bytearray][0] - scope-9 + | | + | Cast[chararray] - scope-13 + | | + | |---Project[bytearray][1] - scope-12 + | + |---a: Load(file:///tmp/input:org.apache.pig.builtin.PigStorage) - scope-8 +Tez vertex scope-40 +# Plan on vertex +POValueOutputTez - scope-45 -> [scope-42] +| +|---b: Filter[bag] - scope-21 + | | + | Equal To[boolean] - scope-24 + | | + | |---Project[int][0] - scope-22 + | | + | |---Constant(2) - scope-23 + | + |---POValueInputTez - scope-41 <- scope-36 +Tez vertex scope-42 +# Plan on vertex +Local Rearrange[tuple]{int}(false) - scope-31 -> scope-35 +| | +| Project[int][0] - scope-27 +| +|---POShuffledValueInputTez - scope-43 <- [scope-36, scope-40] +Tez vertex scope-35 +# Plan on vertex +e: Store(file:///tmp/output:org.apache.pig.builtin.PigStorage) - scope-34 +| +|---e: FRJoin[tuple] - scope-28 <- scope-42 + | | + | Project[int][0] - scope-26 + | | + | Project[int][0] - scope-27 + | + |---d: New For Each(false,false)[bag] - scope-7 + | | + | Cast[int] - scope-2 + | | + | |---Project[bytearray][0] - scope-1 + | | + | Cast[chararray] - scope-5 + | | + | |---Project[bytearray][1] - scope-4 + | + |---d: Load(file:///tmp/input1:org.apache.pig.builtin.PigStorage) - scope-0 Added: pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-14.gld URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-14.gld?rev=1671973&view=auto ============================================================================== --- pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-14.gld (added) +++ pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-14.gld Tue Apr 7 23:21:01 2015 @@ -0,0 +1,61 @@ +#-------------------------------------------------- +# There are 1 DAGs in the session +#-------------------------------------------------- +#-------------------------------------------------- +# TEZ DAG plan: pig-0_scope-0 +#-------------------------------------------------- +Tez vertex scope-36 -> Tez vertex scope-35, +Tez vertex scope-35 + +Tez vertex scope-36 +# Plan on vertex +a: Split - scope-46 +| | +| Local Rearrange[tuple]{int}(false) - scope-47 -> scope-35 +| | | +| | Project[int][0] - scope-48 +| | +| Local Rearrange[tuple]{int}(false) - scope-49 -> scope-35 +| | | +| | Project[int][0] - scope-50 +| | +| |---b: Filter[bag] - scope-21 +| | | +| | Equal To[boolean] - scope-24 +| | | +| | |---Project[int][0] - scope-22 +| | | +| | |---Constant(2) - scope-23 +| +|---a: New For Each(false,false)[bag] - scope-15 + | | + | Cast[int] - scope-10 + | | + | |---Project[bytearray][0] - scope-9 + | | + | Cast[chararray] - scope-13 + | | + | |---Project[bytearray][1] - scope-12 + | + |---a: Load(file:///tmp/input:org.apache.pig.builtin.PigStorage) - scope-8 +Tez vertex scope-35 +# Plan on vertex +e: Store(file:///tmp/output:org.apache.pig.builtin.PigStorage) - scope-34 +| +|---e: FRJoin[tuple] - scope-28 <- scope-36 + | | + | Project[int][0] - scope-26 + | | + | Project[int][0] - scope-27 + | + |---d: New For Each(false,false)[bag] - scope-7 + | | + | Cast[int] - scope-2 + | | + | |---Project[bytearray][0] - scope-1 + | | + | Cast[chararray] - scope-5 + | | + | |---Project[bytearray][1] - scope-4 + | + |---d: Load(file:///tmp/input1:org.apache.pig.builtin.PigStorage) - scope-0 Added: pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-15-OPTOFF.gld URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-15-OPTOFF.gld?rev=1671973&view=auto ============================================================================== --- pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-15-OPTOFF.gld (added) +++ pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-15-OPTOFF.gld Tue Apr 7 23:21:01 2015 @@ -0,0 +1,117 @@ +#-------------------------------------------------- +# There are 1 DAGs in the session +#-------------------------------------------------- +#-------------------------------------------------- +# TEZ DAG plan: pig-0_scope-0 +#-------------------------------------------------- +Tez vertex scope-30 -> Tez vertex scope-34,Tez vertex scope-36, +Tez vertex scope-34 -> Tez vertex scope-36, +Tez vertex scope-36 -> Tez vertex scope-48,Tez vertex scope-58, +Tez vertex scope-48 -> Tez vertex scope-40,Tez vertex scope-58, +Tez vertex scope-58 -> Tez vertex scope-62, +Tez vertex scope-40 -> Tez vertex scope-62, +Tez vertex scope-62 + +Tez vertex scope-30 +# Plan on vertex +a: Split - scope-68 +| | +| POValueOutputTez - scope-38 -> [scope-36] +| | +| POValueOutputTez - scope-31 -> [scope-34] +| +|---a: New For Each(false,false)[bag] - scope-7 + | | + | Cast[int] - scope-2 + | | + | |---Project[bytearray][0] - scope-1 + | | + | Cast[chararray] - scope-5 + | | + | |---Project[bytearray][1] - scope-4 + | + |---a: Load(file:///tmp/input:org.apache.pig.builtin.PigStorage) - scope-0 +Tez vertex scope-34 +# Plan on vertex +POValueOutputTez - scope-39 -> [scope-36] +| +|---b: Filter[bag] - scope-13 + | | + | Equal To[boolean] - scope-16 + | | + | |---Project[int][0] - scope-14 + | | + | |---Constant(2) - scope-15 + | + |---POValueInputTez - scope-35 <- scope-30 +Tez vertex scope-36 +# Plan on vertex +Local Rearrange[tuple]{tuple}(false) - scope-43 -> scope-48 +| | +| Constant(DummyVal) - scope-42 +| +|---New For Each(true,true)[tuple] - scope-47 + | | + | Project[int][0] - scope-26 + | | + | POUserFunc(org.apache.pig.impl.builtin.GetMemNumRows)[tuple] - scope-46 + | | + | |---Project[tuple][*] - scope-45 + | + |---PoissonSample - scope-44 + | + |---Local Rearrange[tuple]{int}(false) - scope-41 -> scope-58 + | | + | Project[int][0] - scope-26 + | + |---POShuffledValueInputTez - scope-37 <- [scope-30, scope-34] +Tez vertex scope-48 +# Plan on vertex +POValueOutputTez - scope-57 -> [scope-40, scope-58] +| +|---New For Each(false)[tuple] - scope-56 + | | + | POUserFunc(org.apache.pig.backend.hadoop.executionengine.tez.plan.udf.PartitionSkewedKeysTez)[tuple] - scope-55 + | | + | |---Project[tuple][*] - scope-54 + | + |---New For Each(false,false)[tuple] - scope-53 + | | + | Constant(-1) - scope-52 + | | + | Project[bag][1] - scope-50 + | + |---Package(Packager)[tuple]{bytearray} - scope-49 +Tez vertex scope-58 +# Plan on vertex +POIdentityInOutTez - scope-59 <- scope-36 -> scope-62 +| | +| Project[int][0] - scope-26 +Tez vertex scope-40 +# Plan on vertex +Partition Rearrange[tuple]{int}(false) - scope-60 -> scope-62 +| | +| Project[int][0] - scope-27 +| +|---d: New For Each(false,false)[bag] - scope-25 + | | + | Cast[int] - scope-20 + | | + | |---Project[bytearray][0] - scope-19 + | | + | Cast[chararray] - scope-23 + | | + | |---Project[bytearray][1] - scope-22 + | + |---d: Load(file:///tmp/input1:org.apache.pig.builtin.PigStorage) - scope-18 +Tez vertex scope-62 +# Plan on vertex +e: Store(file:///tmp/output:org.apache.pig.builtin.PigStorage) - scope-29 +| +|---New For Each(true,true)[tuple] - scope-66 + | | + | Project[bag][1] - scope-64 + | | + | Project[bag][2] - scope-65 + | + |---Package(Packager)[tuple]{int} - scope-63