pig-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From da...@apache.org
Subject svn commit: r1603243 [2/2] - in /pig/trunk: ./ src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/ src/org/apache/pig/backend/hadoop/executionengine/tez/ src/org/apache/pig/backend/hadoop/executionengine/tez/optimizers/...
Date Tue, 17 Jun 2014 18:15:27 GMT
Modified: pig/trunk/test/org/apache/pig/test/data/GoldenFiles/TEZC16.gld
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/data/GoldenFiles/TEZC16.gld?rev=1603243&r1=1603242&r2=1603243&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/data/GoldenFiles/TEZC16.gld (original)
+++ pig/trunk/test/org/apache/pig/test/data/GoldenFiles/TEZC16.gld Tue Jun 17 18:15:26 2014
@@ -2,26 +2,30 @@
 # There are 1 DAGs in the session
 #--------------------------------------------------
 #--------------------------------------------------
-# TEZ DAG plan: scope-35
+# TEZ DAG plan: scope-37
 #--------------------------------------------------
-Tez vertex scope-11	->	Tez vertex scope-28,Tez vertex scope-18,
-Tez vertex scope-18	->	Tez vertex scope-28,
-Tez vertex scope-28	->	Tez vertex scope-30,
-Tez vertex scope-30
+Tez vertex scope-11	->	Tez vertex scope-30,Tez vertex scope-20,
+Tez vertex scope-20	->	Tez vertex scope-30,
+Tez vertex scope-30	->	Tez vertex scope-32,
+Tez vertex scope-32
 
 Tez vertex scope-11
 # Plan on vertex
-Local Rearrange[tuple]{tuple}(false) - scope-14	->	 scope-18
+Local Rearrange[tuple]{tuple}(false) - scope-14	->	 scope-20
 |   |
 |   Constant(DummyVal) - scope-13
 |
-|---ReservoirSample - scope-17
+|---New For Each(true,true)[tuple] - scope-19
+    |   |
+    |   Project[int][0] - scope-8
+    |   |
+    |   POUserFunc(org.apache.pig.impl.builtin.GetMemNumRows)[tuple] - scope-18
+    |   |
+    |   |---Project[tuple][*] - scope-17
     |
-    |---New For Each(false)[tuple] - scope-16
-        |   |
-        |   Project[int][0] - scope-15
+    |---ReservoirSample - scope-16
         |
-        |---b: Local Rearrange[tuple]{int}(false) - scope-12	->	 scope-28
+        |---b: Local Rearrange[tuple]{int}(false) - scope-12	->	 scope-30
             |   |
             |   Project[int][0] - scope-8
             |
@@ -36,34 +40,34 @@ Local Rearrange[tuple]{tuple}(false) - s
                 |   |---Project[bytearray][1] - scope-4
                 |
                 |---a: Load(file:///tmp/input:PigStorage(',')) - scope-0
-Tez vertex scope-18
+Tez vertex scope-20
 # Plan on vertex
-POValueOutputTez - scope-27	->	 [scope-28]
+POValueOutputTez - scope-29	->	 [scope-30]
 |
-|---New For Each(false)[tuple] - scope-26
+|---New For Each(false)[tuple] - scope-28
     |   |
-    |   POUserFunc(org.apache.pig.impl.builtin.FindQuantiles)[tuple] - scope-25
+    |   POUserFunc(org.apache.pig.backend.hadoop.executionengine.tez.FindQuantilesTez)[tuple] - scope-27
     |   |
-    |   |---Project[tuple][*] - scope-24
+    |   |---Project[tuple][*] - scope-26
     |
-    |---New For Each(false,false)[tuple] - scope-23
+    |---New For Each(false,false)[tuple] - scope-25
         |   |
-        |   Constant(1) - scope-22
+        |   Constant(-1) - scope-24
         |   |
-        |   Project[bag][1] - scope-20
+        |   Project[bag][1] - scope-22
         |
-        |---Package(Packager)[tuple]{bytearray} - scope-19
-Tez vertex scope-28
+        |---Package(Packager)[tuple]{bytearray} - scope-21
+Tez vertex scope-30
 # Plan on vertex
-POIdentityInOutTez - scope-29	<-	 scope-11	->	 scope-30
+POIdentityInOutTez - scope-31	<-	 scope-11	->	 scope-32
 |   |
 |   Project[int][0] - scope-8
-Tez vertex scope-30
+Tez vertex scope-32
 # Plan on vertex
 b: Store(file:///tmp/output:org.apache.pig.builtin.PigStorage) - scope-10
 |
-|---New For Each(true)[tuple] - scope-33
+|---New For Each(true)[tuple] - scope-35
     |   |
-    |   Project[bag][1] - scope-32
+    |   Project[bag][1] - scope-34
     |
-    |---Package(LitePackager)[tuple]{int} - scope-31
+    |---Package(LitePackager)[tuple]{int} - scope-33

Modified: pig/trunk/test/org/apache/pig/test/data/GoldenFiles/TEZC17.gld
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/data/GoldenFiles/TEZC17.gld?rev=1603243&r1=1603242&r2=1603243&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/data/GoldenFiles/TEZC17.gld (original)
+++ pig/trunk/test/org/apache/pig/test/data/GoldenFiles/TEZC17.gld Tue Jun 17 18:15:26 2014
@@ -47,13 +47,13 @@ POValueOutputTez - scope-45	->	 [scope-4
 |
 |---New For Each(false)[tuple] - scope-44
     |   |
-    |   POUserFunc(org.apache.pig.impl.builtin.PartitionSkewedKeys)[tuple] - scope-43
+    |   POUserFunc(org.apache.pig.backend.hadoop.executionengine.tez.PartitionSkewedKeysTez)[tuple] - scope-43
     |   |
     |   |---Project[tuple][*] - scope-42
     |
     |---New For Each(false,false)[tuple] - scope-41
         |   |
-        |   Constant(1) - scope-40
+        |   Constant(-1) - scope-40
         |   |
         |   Project[bag][1] - scope-38
         |

Modified: pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-MQ-2-OPTOFF.gld
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-MQ-2-OPTOFF.gld?rev=1603243&r1=1603242&r2=1603243&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-MQ-2-OPTOFF.gld (original)
+++ pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-MQ-2-OPTOFF.gld Tue Jun 17 18:15:26 2014
@@ -2,307 +2,311 @@
 # There are 1 DAGs in the session
 #--------------------------------------------------
 #--------------------------------------------------
-# TEZ DAG plan: scope-418
+# TEZ DAG plan: scope-422
 #--------------------------------------------------
-Tez vertex scope-319	->	Tez vertex scope-321,Tez vertex scope-332,Tez vertex scope-343,
-Tez vertex scope-332	->	Tez vertex scope-335,Tez vertex scope-340,
-Tez vertex scope-335	->	Tez vertex scope-339,
-Tez vertex scope-321	->	Tez vertex scope-324,Tez vertex scope-337,Tez vertex scope-348,
-Tez vertex scope-337	->	Tez vertex scope-339,
-Tez vertex scope-339
-Tez vertex scope-348	->	Tez vertex scope-351,Tez vertex scope-375,
-Tez vertex scope-375	->	Tez vertex scope-379,
-Tez vertex scope-379	->	Tez vertex scope-385,Tez vertex scope-389,
-Tez vertex scope-385
-Tez vertex scope-343	->	Tez vertex scope-346,Tez vertex scope-387,
-Tez vertex scope-387	->	Tez vertex scope-415,
-Tez vertex scope-389	->	Tez vertex scope-415,
-Tez vertex scope-415
-Tez vertex scope-346
-Tez vertex scope-340	->	Tez vertex scope-342,
-Tez vertex scope-342
-Tez vertex scope-324	->	Tez vertex scope-326,
-Tez vertex scope-326	->	Tez vertex scope-328,Tez vertex scope-330,
-Tez vertex scope-328
-Tez vertex scope-351	->	Tez vertex scope-369,Tez vertex scope-359,
-Tez vertex scope-359	->	Tez vertex scope-369,
-Tez vertex scope-369	->	Tez vertex scope-371,
-Tez vertex scope-371
+Tez vertex scope-321	->	Tez vertex scope-323,Tez vertex scope-334,Tez vertex scope-345,
+Tez vertex scope-323	->	Tez vertex scope-326,Tez vertex scope-339,Tez vertex scope-350,
+Tez vertex scope-350	->	Tez vertex scope-353,Tez vertex scope-379,
+Tez vertex scope-379	->	Tez vertex scope-383,
+Tez vertex scope-383	->	Tez vertex scope-389,Tez vertex scope-393,
+Tez vertex scope-389
+Tez vertex scope-334	->	Tez vertex scope-337,Tez vertex scope-342,
+Tez vertex scope-337	->	Tez vertex scope-341,
+Tez vertex scope-339	->	Tez vertex scope-341,
+Tez vertex scope-341
+Tez vertex scope-345	->	Tez vertex scope-348,Tez vertex scope-391,
+Tez vertex scope-348
+Tez vertex scope-342	->	Tez vertex scope-344,
+Tez vertex scope-344
+Tez vertex scope-353	->	Tez vertex scope-373,Tez vertex scope-363,
+Tez vertex scope-363	->	Tez vertex scope-373,
+Tez vertex scope-373	->	Tez vertex scope-375,
+Tez vertex scope-375
+Tez vertex scope-326	->	Tez vertex scope-328,
+Tez vertex scope-328	->	Tez vertex scope-330,Tez vertex scope-332,
 Tez vertex scope-330
+Tez vertex scope-391	->	Tez vertex scope-419,
+Tez vertex scope-393	->	Tez vertex scope-419,
+Tez vertex scope-419
+Tez vertex scope-332
 
-Tez vertex scope-319
+Tez vertex scope-321
 # Plan on vertex
-POValueOutputTez - scope-320	->	 [scope-321, scope-332, scope-343]
+POValueOutputTez - scope-322	->	 [scope-323, scope-345, scope-334]
 |
-|---a: New For Each(false,false)[bag] - scope-220
+|---a: New For Each(false,false)[bag] - scope-222
     |   |
-    |   Cast[int] - scope-215
+    |   Cast[int] - scope-217
     |   |
-    |   |---Project[bytearray][0] - scope-214
+    |   |---Project[bytearray][0] - scope-216
     |   |
-    |   Cast[int] - scope-218
+    |   Cast[int] - scope-220
     |   |
-    |   |---Project[bytearray][1] - scope-217
+    |   |---Project[bytearray][1] - scope-219
     |
-    |---a: Load(file:///tmp/input:org.apache.pig.builtin.PigStorage) - scope-213
-Tez vertex scope-332
+    |---a: Load(file:///tmp/input:org.apache.pig.builtin.PigStorage) - scope-215
+Tez vertex scope-323
 # Plan on vertex
-POValueOutputTez - scope-334	->	 [scope-340, scope-335]
+POValueOutputTez - scope-325	->	 [scope-339, scope-326, scope-350]
 |
-|---c: Filter[bag] - scope-247
+|---b: Filter[bag] - scope-224
     |   |
-    |   Less Than or Equal[boolean] - scope-250
+    |   Less Than or Equal[boolean] - scope-227
     |   |
-    |   |---Project[int][0] - scope-248
+    |   |---Project[int][0] - scope-225
     |   |
-    |   |---Constant(10) - scope-249
+    |   |---Constant(5) - scope-226
     |
-    |---POValueInputTez - scope-333	<-	 scope-319
-Tez vertex scope-335
+    |---POValueInputTez - scope-324	<-	 scope-321
+Tez vertex scope-350
+# Plan on vertex
+POValueOutputTez - scope-352	->	 [scope-379, scope-353]
+|
+|---POValueInputTez - scope-351	<-	 scope-323
+Tez vertex scope-379
 # Plan on vertex
-c1: Local Rearrange[tuple]{int}(false) - scope-260	->	 scope-339
+f1: Local Rearrange[tuple]{tuple}(false) - scope-382	->	 scope-383
 |   |
-|   Project[int][0] - scope-261
+|   Project[tuple][*] - scope-381
 |
-|---POValueInputTez - scope-336	<-	 scope-332
-Tez vertex scope-321
+|---f1: Limit - scope-310
+    |
+    |---f: Filter[bag] - scope-306
+        |   |
+        |   Greater Than or Equal[boolean] - scope-309
+        |   |
+        |   |---Project[int][0] - scope-307
+        |   |
+        |   |---Constant(3) - scope-308
+        |
+        |---POValueInputTez - scope-380	<-	 scope-350
+Tez vertex scope-383
 # Plan on vertex
-POValueOutputTez - scope-323	->	 [scope-348, scope-337, scope-324]
+POValueOutputTez - scope-388	->	 [scope-393, scope-389]
 |
-|---b: Filter[bag] - scope-222
-    |   |
-    |   Less Than or Equal[boolean] - scope-225
-    |   |
-    |   |---Project[int][0] - scope-223
-    |   |
-    |   |---Constant(5) - scope-224
+|---f1: Limit - scope-387
     |
-    |---POValueInputTez - scope-322	<-	 scope-319
-Tez vertex scope-337
+    |---f1: New For Each(true)[bag] - scope-386
+        |   |
+        |   Project[tuple][1] - scope-385
+        |
+        |---f1: Package(Packager)[tuple]{tuple} - scope-384
+Tez vertex scope-389
 # Plan on vertex
-c1: Local Rearrange[tuple]{int}(false) - scope-262	->	 scope-339
-|   |
-|   Project[int][0] - scope-263
+f1: Store(file:///tmp/output/f1:org.apache.pig.builtin.PigStorage) - scope-314
 |
-|---POValueInputTez - scope-338	<-	 scope-321
-Tez vertex scope-339
+|---POValueInputTez - scope-390	<-	 scope-383
+Tez vertex scope-334
 # Plan on vertex
-c1: Store(file:///tmp/output/c1:org.apache.pig.builtin.PigStorage) - scope-267
+POValueOutputTez - scope-336	->	 [scope-337, scope-342]
 |
-|---c1: New For Each(true,true)[tuple] - scope-266
+|---c: Filter[bag] - scope-249
     |   |
-    |   Project[bag][1] - scope-264
+    |   Less Than or Equal[boolean] - scope-252
     |   |
-    |   Project[bag][2] - scope-265
+    |   |---Project[int][0] - scope-250
+    |   |
+    |   |---Constant(10) - scope-251
     |
-    |---c1: Package(Packager)[tuple]{int} - scope-259
-Tez vertex scope-348
+    |---POValueInputTez - scope-335	<-	 scope-321
+Tez vertex scope-337
 # Plan on vertex
-POValueOutputTez - scope-350	->	 [scope-375, scope-351]
+c1: Local Rearrange[tuple]{int}(false) - scope-262	->	 scope-341
+|   |
+|   Project[int][0] - scope-263
 |
-|---POValueInputTez - scope-349	<-	 scope-321
-Tez vertex scope-375
+|---POValueInputTez - scope-338	<-	 scope-334
+Tez vertex scope-339
 # Plan on vertex
-f1: Local Rearrange[tuple]{tuple}(false) - scope-378	->	 scope-379
+c1: Local Rearrange[tuple]{int}(false) - scope-264	->	 scope-341
 |   |
-|   Project[tuple][*] - scope-377
+|   Project[int][0] - scope-265
 |
-|---f1: Limit - scope-308
-    |
-    |---f: Filter[bag] - scope-304
-        |   |
-        |   Greater Than or Equal[boolean] - scope-307
-        |   |
-        |   |---Project[int][0] - scope-305
-        |   |
-        |   |---Constant(3) - scope-306
-        |
-        |---POValueInputTez - scope-376	<-	 scope-348
-Tez vertex scope-379
+|---POValueInputTez - scope-340	<-	 scope-323
+Tez vertex scope-341
 # Plan on vertex
-POValueOutputTez - scope-384	->	 [scope-389, scope-385]
+c1: Store(file:///tmp/output/c1:org.apache.pig.builtin.PigStorage) - scope-269
 |
-|---f1: Limit - scope-383
+|---c1: New For Each(true,true)[tuple] - scope-268
+    |   |
+    |   Project[bag][1] - scope-266
+    |   |
+    |   Project[bag][2] - scope-267
     |
-    |---f1: New For Each(true)[bag] - scope-382
-        |   |
-        |   Project[tuple][1] - scope-381
-        |
-        |---f1: Package(Packager)[tuple]{tuple} - scope-380
-Tez vertex scope-385
-# Plan on vertex
-f1: Store(file:///tmp/output/f1:org.apache.pig.builtin.PigStorage) - scope-312
-|
-|---POValueInputTez - scope-386	<-	 scope-379
-Tez vertex scope-343
+    |---c1: Package(Packager)[tuple]{int} - scope-261
+Tez vertex scope-345
 # Plan on vertex
-POValueOutputTez - scope-345	->	 [scope-387, scope-346]
+POValueOutputTez - scope-347	->	 [scope-391, scope-348]
 |
-|---d1: Filter[bag] - scope-286
+|---d1: Filter[bag] - scope-288
     |   |
-    |   Equal To[boolean] - scope-289
+    |   Equal To[boolean] - scope-291
     |   |
-    |   |---Project[int][0] - scope-287
+    |   |---Project[int][0] - scope-289
     |   |
-    |   |---Constant(5) - scope-288
+    |   |---Constant(5) - scope-290
     |
-    |---d: Filter[bag] - scope-282
+    |---d: Filter[bag] - scope-284
         |   |
-        |   Greater Than[boolean] - scope-285
+        |   Greater Than[boolean] - scope-287
         |   |
-        |   |---Project[int][0] - scope-283
+        |   |---Project[int][0] - scope-285
         |   |
-        |   |---Constant(10) - scope-284
+        |   |---Constant(10) - scope-286
         |
-        |---POValueInputTez - scope-344	<-	 scope-319
-Tez vertex scope-387
-# Plan on vertex
-f2: Store(file:///tmp/output/f2:org.apache.pig.builtin.PigStorage) - scope-416
-|
-|---POValueInputTez - scope-388	<-	 scope-343
-Tez vertex scope-389
-# Plan on vertex
-f2: Store(file:///tmp/output/f2:org.apache.pig.builtin.PigStorage) - scope-417
-|
-|---POValueInputTez - scope-390	<-	 scope-379
-Tez vertex group scope-415	<-	 [scope-387, scope-389]	->	 null
-# No plan on vertex group
-Tez vertex scope-346
+        |---POValueInputTez - scope-346	<-	 scope-321
+Tez vertex scope-348
 # Plan on vertex
-d1: Store(file:///tmp/output/d1:org.apache.pig.builtin.PigStorage) - scope-293
+d1: Store(file:///tmp/output/d1:org.apache.pig.builtin.PigStorage) - scope-295
 |
-|---POValueInputTez - scope-347	<-	 scope-343
-Tez vertex scope-340
+|---POValueInputTez - scope-349	<-	 scope-345
+Tez vertex scope-342
 # Plan on vertex
-c2: Local Rearrange[tuple]{int}(false) - scope-407	->	 scope-342
+c2: Local Rearrange[tuple]{int}(false) - scope-411	->	 scope-344
 |   |
-|   Project[int][0] - scope-409
+|   Project[int][0] - scope-413
 |
-|---c3: New For Each(false,false)[bag] - scope-395
+|---c3: New For Each(false,false)[bag] - scope-399
     |   |
-    |   Project[int][0] - scope-396
+    |   Project[int][0] - scope-400
     |   |
-    |   POUserFunc(org.apache.pig.builtin.AlgebraicMathBase$Initial)[tuple] - scope-397
+    |   POUserFunc(org.apache.pig.builtin.AlgebraicMathBase$Initial)[tuple] - scope-401
     |   |
-    |   |---Project[bag][0] - scope-398
+    |   |---Project[bag][0] - scope-402
     |       |
-    |       |---Project[bag][1] - scope-399
+    |       |---Project[bag][1] - scope-403
     |
-    |---Pre Combiner Local Rearrange[tuple]{Unknown} - scope-410
+    |---Pre Combiner Local Rearrange[tuple]{Unknown} - scope-414
         |
-        |---POValueInputTez - scope-341	<-	 scope-332
-Tez vertex scope-342
-# Combine plan on edge <scope-340>
-c2: Local Rearrange[tuple]{int}(false) - scope-411	->	 scope-342
+        |---POValueInputTez - scope-343	<-	 scope-334
+Tez vertex scope-344
+# Combine plan on edge <scope-342>
+c2: Local Rearrange[tuple]{int}(false) - scope-415	->	 scope-344
 |   |
-|   Project[int][0] - scope-413
+|   Project[int][0] - scope-417
 |
-|---c3: New For Each(false,false)[bag] - scope-400
+|---c3: New For Each(false,false)[bag] - scope-404
     |   |
-    |   Project[int][0] - scope-401
+    |   Project[int][0] - scope-405
     |   |
-    |   POUserFunc(org.apache.pig.builtin.LongSum$Intermediate)[tuple] - scope-402
+    |   POUserFunc(org.apache.pig.builtin.LongSum$Intermediate)[tuple] - scope-406
     |   |
-    |   |---Project[bag][1] - scope-403
+    |   |---Project[bag][1] - scope-407
     |
-    |---c2: Package(CombinerPackager)[tuple]{int} - scope-406
+    |---c2: Package(CombinerPackager)[tuple]{int} - scope-410
 # Plan on vertex
-c3: Store(file:///tmp/output/c1:org.apache.pig.builtin.PigStorage) - scope-281
+c3: Store(file:///tmp/output/c1:org.apache.pig.builtin.PigStorage) - scope-283
 |
-|---c3: New For Each(false,false)[bag] - scope-280
+|---c3: New For Each(false,false)[bag] - scope-282
     |   |
-    |   Project[int][0] - scope-274
+    |   Project[int][0] - scope-276
     |   |
-    |   POUserFunc(org.apache.pig.builtin.LongSum$Final)[long] - scope-278
+    |   POUserFunc(org.apache.pig.builtin.LongSum$Final)[long] - scope-280
     |   |
-    |   |---Project[bag][1] - scope-404
+    |   |---Project[bag][1] - scope-408
     |
-    |---c2: Package(CombinerPackager)[tuple]{int} - scope-271
-Tez vertex scope-324
+    |---c2: Package(CombinerPackager)[tuple]{int} - scope-273
+Tez vertex scope-353
 # Plan on vertex
-b1: Local Rearrange[tuple]{int}(false) - scope-231	->	 scope-326
+Local Rearrange[tuple]{tuple}(false) - scope-357	->	 scope-363
 |   |
-|   Project[int][0] - scope-232
-|
-|---POValueInputTez - scope-325	<-	 scope-321
-Tez vertex scope-326
-# Plan on vertex
-POValueOutputTez - scope-327	->	 [scope-330, scope-328]
+|   Constant(DummyVal) - scope-356
 |
-|---b1: Package(Packager)[tuple]{int} - scope-230
-Tez vertex scope-328
-# Plan on vertex
-b1: Store(file:///tmp/output/b1:org.apache.pig.builtin.PigStorage) - scope-236
-|
-|---POValueInputTez - scope-329	<-	 scope-326
-Tez vertex scope-351
-# Plan on vertex
-Local Rearrange[tuple]{tuple}(false) - scope-355	->	 scope-359
-|   |
-|   Constant(DummyVal) - scope-354
-|
-|---ReservoirSample - scope-358
+|---New For Each(true,true)[tuple] - scope-362
+    |   |
+    |   Project[int][0] - scope-303
+    |   |
+    |   POUserFunc(org.apache.pig.impl.builtin.GetMemNumRows)[tuple] - scope-361
+    |   |
+    |   |---Project[tuple][*] - scope-360
     |
-    |---New For Each(false)[tuple] - scope-357
-        |   |
-        |   Project[int][0] - scope-356
+    |---ReservoirSample - scope-359
         |
-        |---e1: Local Rearrange[tuple]{int}(false) - scope-353	->	 scope-369
+        |---e1: Local Rearrange[tuple]{int}(false) - scope-355	->	 scope-373
             |   |
-            |   Project[int][0] - scope-301
+            |   Project[int][0] - scope-303
             |
-            |---e: Filter[bag] - scope-297
+            |---e: Filter[bag] - scope-299
                 |   |
-                |   Less Than[boolean] - scope-300
+                |   Less Than[boolean] - scope-302
                 |   |
-                |   |---Project[int][0] - scope-298
+                |   |---Project[int][0] - scope-300
                 |   |
-                |   |---Constant(3) - scope-299
+                |   |---Constant(3) - scope-301
                 |
-                |---POValueInputTez - scope-352	<-	 scope-348
-Tez vertex scope-359
+                |---POValueInputTez - scope-354	<-	 scope-350
+Tez vertex scope-363
 # Plan on vertex
-POValueOutputTez - scope-368	->	 [scope-369]
+POValueOutputTez - scope-372	->	 [scope-373]
 |
-|---New For Each(false)[tuple] - scope-367
+|---New For Each(false)[tuple] - scope-371
     |   |
-    |   POUserFunc(org.apache.pig.impl.builtin.FindQuantiles)[tuple] - scope-366
+    |   POUserFunc(org.apache.pig.backend.hadoop.executionengine.tez.FindQuantilesTez)[tuple] - scope-370
     |   |
-    |   |---Project[tuple][*] - scope-365
+    |   |---Project[tuple][*] - scope-369
     |
-    |---New For Each(false,false)[tuple] - scope-364
+    |---New For Each(false,false)[tuple] - scope-368
         |   |
-        |   Constant(1) - scope-363
+        |   Constant(-1) - scope-367
         |   |
-        |   Project[bag][1] - scope-361
+        |   Project[bag][1] - scope-365
         |
-        |---Package(Packager)[tuple]{bytearray} - scope-360
-Tez vertex scope-369
+        |---Package(Packager)[tuple]{bytearray} - scope-364
+Tez vertex scope-373
 # Plan on vertex
-POIdentityInOutTez - scope-370	<-	 scope-351	->	 scope-371
+POIdentityInOutTez - scope-374	<-	 scope-353	->	 scope-375
 |   |
-|   Project[int][0] - scope-301
-Tez vertex scope-371
+|   Project[int][0] - scope-303
+Tez vertex scope-375
 # Plan on vertex
-e1: Store(file:///tmp/output/e1:org.apache.pig.builtin.PigStorage) - scope-303
+e1: Store(file:///tmp/output/e1:org.apache.pig.builtin.PigStorage) - scope-305
 |
-|---New For Each(true)[tuple] - scope-374
+|---New For Each(true)[tuple] - scope-378
     |   |
-    |   Project[bag][1] - scope-373
+    |   Project[bag][1] - scope-377
     |
-    |---Package(LitePackager)[tuple]{int} - scope-372
+    |---Package(LitePackager)[tuple]{int} - scope-376
+Tez vertex scope-326
+# Plan on vertex
+b1: Local Rearrange[tuple]{int}(false) - scope-233	->	 scope-328
+|   |
+|   Project[int][0] - scope-234
+|
+|---POValueInputTez - scope-327	<-	 scope-323
+Tez vertex scope-328
+# Plan on vertex
+POValueOutputTez - scope-329	->	 [scope-330, scope-332]
+|
+|---b1: Package(Packager)[tuple]{int} - scope-232
 Tez vertex scope-330
 # Plan on vertex
-b2: Store(file:///tmp/output/b2:org.apache.pig.builtin.PigStorage) - scope-246
+b1: Store(file:///tmp/output/b1:org.apache.pig.builtin.PigStorage) - scope-238
+|
+|---POValueInputTez - scope-331	<-	 scope-328
+Tez vertex scope-391
+# Plan on vertex
+f2: Store(file:///tmp/output/f2:org.apache.pig.builtin.PigStorage) - scope-420
+|
+|---POValueInputTez - scope-392	<-	 scope-345
+Tez vertex scope-393
+# Plan on vertex
+f2: Store(file:///tmp/output/f2:org.apache.pig.builtin.PigStorage) - scope-421
+|
+|---POValueInputTez - scope-394	<-	 scope-383
+Tez vertex group scope-419	<-	 [scope-391, scope-393]	->	 null
+# No plan on vertex group
+Tez vertex scope-332
+# Plan on vertex
+b2: Store(file:///tmp/output/b2:org.apache.pig.builtin.PigStorage) - scope-248
 |
-|---b2: New For Each(false,false)[bag] - scope-245
+|---b2: New For Each(false,false)[bag] - scope-247
     |   |
-    |   Project[int][0] - scope-239
+    |   Project[int][0] - scope-241
     |   |
-    |   POUserFunc(org.apache.pig.builtin.LongSum)[long] - scope-243
+    |   POUserFunc(org.apache.pig.builtin.LongSum)[long] - scope-245
     |   |
-    |   |---Project[bag][0] - scope-242
+    |   |---Project[bag][0] - scope-244
     |       |
-    |       |---Project[bag][1] - scope-241
+    |       |---Project[bag][1] - scope-243
     |
-    |---POValueInputTez - scope-331	<-	 scope-326
+    |---POValueInputTez - scope-333	<-	 scope-328

Modified: pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-MQ-2.gld
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-MQ-2.gld?rev=1603243&r1=1603242&r2=1603243&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-MQ-2.gld (original)
+++ pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-MQ-2.gld Tue Jun 17 18:15:26 2014
@@ -2,24 +2,24 @@
 # There are 1 DAGs in the session
 #--------------------------------------------------
 #--------------------------------------------------
-# TEZ DAG plan: scope-212
+# TEZ DAG plan: scope-214
 #--------------------------------------------------
-Tez vertex scope-106	->	Tez vertex scope-119,Tez vertex scope-113,Tez vertex scope-126,Tez vertex scope-156,Tez vertex scope-146,Tez vertex scope-166,Tez vertex scope-209,
-Tez vertex scope-166	->	Tez vertex scope-209,
-Tez vertex scope-209
-Tez vertex scope-113
+Tez vertex scope-106	->	Tez vertex scope-119,Tez vertex scope-113,Tez vertex scope-126,Tez vertex scope-158,Tez vertex scope-148,Tez vertex scope-168,Tez vertex scope-211,
+Tez vertex scope-148	->	Tez vertex scope-158,
+Tez vertex scope-158	->	Tez vertex scope-160,
+Tez vertex scope-160
 Tez vertex scope-119	->	Tez vertex scope-126,Tez vertex scope-129,
 Tez vertex scope-126
 Tez vertex scope-129
-Tez vertex scope-146	->	Tez vertex scope-156,
-Tez vertex scope-156	->	Tez vertex scope-158,
-Tez vertex scope-158
+Tez vertex scope-113
+Tez vertex scope-168	->	Tez vertex scope-211,
+Tez vertex scope-211
 
 Tez vertex scope-106
 # Plan on vertex
-1-1: Split - scope-208
+1-1: Split - scope-210
 |   |
-|   b: Split - scope-205
+|   b: Split - scope-207
 |   |   |
 |   |   b1: Local Rearrange[tuple]{int}(false) - scope-18	->	 scope-113
 |   |   |   |
@@ -29,19 +29,23 @@ Tez vertex scope-106
 |   |   |   |
 |   |   |   Project[int][0] - scope-50
 |   |   |
-|   |   1-2: Split - scope-204
+|   |   1-2: Split - scope-206
 |   |   |   |
-|   |   |   Local Rearrange[tuple]{tuple}(false) - scope-142	->	 scope-146
+|   |   |   Local Rearrange[tuple]{tuple}(false) - scope-142	->	 scope-148
 |   |   |   |   |
 |   |   |   |   Constant(DummyVal) - scope-141
 |   |   |   |
-|   |   |   |---ReservoirSample - scope-145
+|   |   |   |---New For Each(true,true)[tuple] - scope-147
+|   |   |       |   |
+|   |   |       |   Project[int][0] - scope-88
+|   |   |       |   |
+|   |   |       |   POUserFunc(org.apache.pig.impl.builtin.GetMemNumRows)[tuple] - scope-146
+|   |   |       |   |
+|   |   |       |   |---Project[tuple][*] - scope-145
 |   |   |       |
-|   |   |       |---New For Each(false)[tuple] - scope-144
-|   |   |           |   |
-|   |   |           |   Project[int][0] - scope-143
+|   |   |       |---ReservoirSample - scope-144
 |   |   |           |
-|   |   |           |---e1: Local Rearrange[tuple]{int}(false) - scope-140	->	 scope-156
+|   |   |           |---e1: Local Rearrange[tuple]{int}(false) - scope-140	->	 scope-158
 |   |   |               |   |
 |   |   |               |   Project[int][0] - scope-88
 |   |   |               |
@@ -53,9 +57,9 @@ Tez vertex scope-106
 |   |   |                   |   |
 |   |   |                   |   |---Constant(3) - scope-86
 |   |   |   |
-|   |   |   f1: Local Rearrange[tuple]{tuple}(false) - scope-165	->	 scope-166
+|   |   |   f1: Local Rearrange[tuple]{tuple}(false) - scope-167	->	 scope-168
 |   |   |   |   |
-|   |   |   |   Project[tuple][*] - scope-164
+|   |   |   |   Project[tuple][*] - scope-166
 |   |   |   |
 |   |   |   |---f1: Limit - scope-95
 |   |   |       |
@@ -75,11 +79,11 @@ Tez vertex scope-106
 |       |   |
 |       |   |---Constant(5) - scope-11
 |   |
-|   d1: Split - scope-207
+|   d1: Split - scope-209
 |   |   |
 |   |   d1: Store(file:///tmp/output/d1:org.apache.pig.builtin.PigStorage) - scope-80
 |   |   |
-|   |   f2: Store(file:///tmp/output/f2:org.apache.pig.builtin.PigStorage) - scope-210
+|   |   f2: Store(file:///tmp/output/f2:org.apache.pig.builtin.PigStorage) - scope-212
 |   |
 |   |---d1: Filter[bag] - scope-73
 |       |   |
@@ -110,65 +114,60 @@ Tez vertex scope-106
     |   |---Project[bytearray][1] - scope-4
     |
     |---a: Load(file:///tmp/input:org.apache.pig.builtin.PigStorage) - scope-0
-Tez vertex scope-166
+Tez vertex scope-148
 # Plan on vertex
-f1: Split - scope-203
-|   |
-|   f1: Store(file:///tmp/output/f1:org.apache.pig.builtin.PigStorage) - scope-99
-|   |
-|   f2: Store(file:///tmp/output/f2:org.apache.pig.builtin.PigStorage) - scope-211
+POValueOutputTez - scope-157	->	 [scope-158]
 |
-|---f1: Limit - scope-170
+|---New For Each(false)[tuple] - scope-156
+    |   |
+    |   POUserFunc(org.apache.pig.backend.hadoop.executionengine.tez.FindQuantilesTez)[tuple] - scope-155
+    |   |
+    |   |---Project[tuple][*] - scope-154
     |
-    |---f1: New For Each(true)[bag] - scope-169
+    |---New For Each(false,false)[tuple] - scope-153
+        |   |
+        |   Constant(-1) - scope-152
         |   |
-        |   Project[tuple][1] - scope-168
+        |   Project[bag][1] - scope-150
         |
-        |---f1: Package(Packager)[tuple]{tuple} - scope-167
-Tez vertex group scope-209	<-	 [scope-106, scope-166]	->	 null
-# No plan on vertex group
-Tez vertex scope-113
+        |---Package(Packager)[tuple]{bytearray} - scope-149
+Tez vertex scope-158
 # Plan on vertex
-b1: Split - scope-202
-|   |
-|   b1: Store(file:///tmp/output/b1:org.apache.pig.builtin.PigStorage) - scope-23
-|   |
-|   b2: Store(file:///tmp/output/b2:org.apache.pig.builtin.PigStorage) - scope-33
+POIdentityInOutTez - scope-159	<-	 scope-106	->	 scope-160
 |   |
-|   |---b2: New For Each(false,false)[bag] - scope-32
-|       |   |
-|       |   Project[int][0] - scope-26
-|       |   |
-|       |   POUserFunc(org.apache.pig.builtin.LongSum)[long] - scope-30
-|       |   |
-|       |   |---Project[bag][0] - scope-29
-|       |       |
-|       |       |---Project[bag][1] - scope-28
+|   Project[int][0] - scope-88
+Tez vertex scope-160
+# Plan on vertex
+e1: Store(file:///tmp/output/e1:org.apache.pig.builtin.PigStorage) - scope-90
 |
-|---b1: Package(Packager)[tuple]{int} - scope-17
+|---New For Each(true)[tuple] - scope-163
+    |   |
+    |   Project[bag][1] - scope-162
+    |
+    |---Package(LitePackager)[tuple]{int} - scope-161
 Tez vertex scope-119
 # Plan on vertex
-c: Split - scope-206
+c: Split - scope-208
 |   |
 |   c1: Local Rearrange[tuple]{int}(false) - scope-47	->	 scope-126
 |   |   |
 |   |   Project[int][0] - scope-48
 |   |
-|   c2: Local Rearrange[tuple]{int}(false) - scope-194	->	 scope-129
+|   c2: Local Rearrange[tuple]{int}(false) - scope-196	->	 scope-129
 |   |   |
-|   |   Project[int][0] - scope-196
+|   |   Project[int][0] - scope-198
 |   |
-|   |---c3: New For Each(false,false)[bag] - scope-182
+|   |---c3: New For Each(false,false)[bag] - scope-184
 |       |   |
-|       |   Project[int][0] - scope-183
+|       |   Project[int][0] - scope-185
 |       |   |
-|       |   POUserFunc(org.apache.pig.builtin.AlgebraicMathBase$Initial)[tuple] - scope-184
+|       |   POUserFunc(org.apache.pig.builtin.AlgebraicMathBase$Initial)[tuple] - scope-186
 |       |   |
-|       |   |---Project[bag][0] - scope-185
+|       |   |---Project[bag][0] - scope-187
 |       |       |
-|       |       |---Project[bag][1] - scope-186
+|       |       |---Project[bag][1] - scope-188
 |       |
-|       |---Pre Combiner Local Rearrange[tuple]{Unknown} - scope-197
+|       |---Pre Combiner Local Rearrange[tuple]{Unknown} - scope-199
 |
 |---c: Filter[bag] - scope-34
     |   |
@@ -192,19 +191,19 @@ c1: Store(file:///tmp/output/c1:org.apac
     |---c1: Package(Packager)[tuple]{int} - scope-46
 Tez vertex scope-129
 # Combine plan on edge <scope-119>
-c2: Local Rearrange[tuple]{int}(false) - scope-198	->	 scope-129
+c2: Local Rearrange[tuple]{int}(false) - scope-200	->	 scope-129
 |   |
-|   Project[int][0] - scope-200
+|   Project[int][0] - scope-202
 |
-|---c3: New For Each(false,false)[bag] - scope-187
+|---c3: New For Each(false,false)[bag] - scope-189
     |   |
-    |   Project[int][0] - scope-188
+    |   Project[int][0] - scope-190
     |   |
-    |   POUserFunc(org.apache.pig.builtin.LongSum$Intermediate)[tuple] - scope-189
+    |   POUserFunc(org.apache.pig.builtin.LongSum$Intermediate)[tuple] - scope-191
     |   |
-    |   |---Project[bag][1] - scope-190
+    |   |---Project[bag][1] - scope-192
     |
-    |---c2: Package(CombinerPackager)[tuple]{int} - scope-193
+    |---c2: Package(CombinerPackager)[tuple]{int} - scope-195
 # Plan on vertex
 c3: Store(file:///tmp/output/c1:org.apache.pig.builtin.PigStorage) - scope-68
 |
@@ -214,37 +213,42 @@ c3: Store(file:///tmp/output/c1:org.apac
     |   |
     |   POUserFunc(org.apache.pig.builtin.LongSum$Final)[long] - scope-65
     |   |
-    |   |---Project[bag][1] - scope-191
+    |   |---Project[bag][1] - scope-193
     |
     |---c2: Package(CombinerPackager)[tuple]{int} - scope-58
-Tez vertex scope-146
+Tez vertex scope-113
 # Plan on vertex
-POValueOutputTez - scope-155	->	 [scope-156]
+b1: Split - scope-204
+|   |
+|   b1: Store(file:///tmp/output/b1:org.apache.pig.builtin.PigStorage) - scope-23
+|   |
+|   b2: Store(file:///tmp/output/b2:org.apache.pig.builtin.PigStorage) - scope-33
+|   |
+|   |---b2: New For Each(false,false)[bag] - scope-32
+|       |   |
+|       |   Project[int][0] - scope-26
+|       |   |
+|       |   POUserFunc(org.apache.pig.builtin.LongSum)[long] - scope-30
+|       |   |
+|       |   |---Project[bag][0] - scope-29
+|       |       |
+|       |       |---Project[bag][1] - scope-28
 |
-|---New For Each(false)[tuple] - scope-154
-    |   |
-    |   POUserFunc(org.apache.pig.impl.builtin.FindQuantiles)[tuple] - scope-153
-    |   |
-    |   |---Project[tuple][*] - scope-152
-    |
-    |---New For Each(false,false)[tuple] - scope-151
-        |   |
-        |   Constant(1) - scope-150
-        |   |
-        |   Project[bag][1] - scope-148
-        |
-        |---Package(Packager)[tuple]{bytearray} - scope-147
-Tez vertex scope-156
+|---b1: Package(Packager)[tuple]{int} - scope-17
+Tez vertex scope-168
 # Plan on vertex
-POIdentityInOutTez - scope-157	<-	 scope-106	->	 scope-158
+f1: Split - scope-205
 |   |
-|   Project[int][0] - scope-88
-Tez vertex scope-158
-# Plan on vertex
-e1: Store(file:///tmp/output/e1:org.apache.pig.builtin.PigStorage) - scope-90
+|   f1: Store(file:///tmp/output/f1:org.apache.pig.builtin.PigStorage) - scope-99
+|   |
+|   f2: Store(file:///tmp/output/f2:org.apache.pig.builtin.PigStorage) - scope-213
 |
-|---New For Each(true)[tuple] - scope-161
-    |   |
-    |   Project[bag][1] - scope-160
+|---f1: Limit - scope-172
     |
-    |---Package(LitePackager)[tuple]{int} - scope-159
+    |---f1: New For Each(true)[bag] - scope-171
+        |   |
+        |   Project[tuple][1] - scope-170
+        |
+        |---f1: Package(Packager)[tuple]{tuple} - scope-169
+Tez vertex group scope-211	<-	 [scope-106, scope-168]	->	 null
+# No plan on vertex group

Modified: pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Rank-2.gld
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Rank-2.gld?rev=1603243&r1=1603242&r2=1603243&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Rank-2.gld (original)
+++ pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Rank-2.gld Tue Jun 17 18:15:26 2014
@@ -2,15 +2,15 @@
 # There are 1 DAGs in the session
 #--------------------------------------------------
 #--------------------------------------------------
-# TEZ DAG plan: scope-51
+# TEZ DAG plan: scope-53
 #--------------------------------------------------
 Tez vertex scope-23	->	Tez vertex scope-24,
-Tez vertex scope-24	->	Tez vertex scope-41,Tez vertex scope-31,
-Tez vertex scope-31	->	Tez vertex scope-41,
-Tez vertex scope-41	->	Tez vertex scope-43,
-Tez vertex scope-43	->	Tez vertex scope-49,Tez vertex scope-47,
-Tez vertex scope-47	->	Tez vertex scope-49,
-Tez vertex scope-49
+Tez vertex scope-24	->	Tez vertex scope-43,Tez vertex scope-33,
+Tez vertex scope-33	->	Tez vertex scope-43,
+Tez vertex scope-43	->	Tez vertex scope-45,
+Tez vertex scope-45	->	Tez vertex scope-51,Tez vertex scope-49,
+Tez vertex scope-49	->	Tez vertex scope-51,
+Tez vertex scope-51
 
 Tez vertex scope-23
 # Plan on vertex
@@ -31,17 +31,21 @@ b: Local Rearrange[tuple]{int}(false) - 
     |---a: Load(file:///tmp/input1:org.apache.pig.builtin.PigStorage) - scope-0
 Tez vertex scope-24
 # Plan on vertex
-Local Rearrange[tuple]{tuple}(false) - scope-27	->	 scope-31
+Local Rearrange[tuple]{tuple}(false) - scope-27	->	 scope-33
 |   |
 |   Constant(DummyVal) - scope-26
 |
-|---ReservoirSample - scope-30
+|---New For Each(true,true)[tuple] - scope-32
+    |   |
+    |   Project[int][0] - scope-15
+    |   |
+    |   POUserFunc(org.apache.pig.impl.builtin.GetMemNumRows)[tuple] - scope-31
+    |   |
+    |   |---Project[tuple][*] - scope-30
     |
-    |---New For Each(false)[tuple] - scope-29
-        |   |
-        |   Project[int][0] - scope-28
+    |---ReservoirSample - scope-29
         |
-        |---b: Local Rearrange[tuple]{int}(false) - scope-25	->	 scope-41
+        |---b: Local Rearrange[tuple]{int}(false) - scope-25	->	 scope-43
             |   |
             |   Project[int][0] - scope-15
             |
@@ -52,44 +56,44 @@ Local Rearrange[tuple]{tuple}(false) - s
                 |   Project[bag][1] - scope-13
                 |
                 |---b: Package(Packager)[tuple]{int} - scope-9
-Tez vertex scope-31
+Tez vertex scope-33
 # Plan on vertex
-POValueOutputTez - scope-40	->	 [scope-41]
+POValueOutputTez - scope-42	->	 [scope-43]
 |
-|---New For Each(false)[tuple] - scope-39
+|---New For Each(false)[tuple] - scope-41
     |   |
-    |   POUserFunc(org.apache.pig.impl.builtin.FindQuantiles)[tuple] - scope-38
+    |   POUserFunc(org.apache.pig.backend.hadoop.executionengine.tez.FindQuantilesTez)[tuple] - scope-40
     |   |
-    |   |---Project[tuple][*] - scope-37
+    |   |---Project[tuple][*] - scope-39
     |
-    |---New For Each(false,false)[tuple] - scope-36
+    |---New For Each(false,false)[tuple] - scope-38
         |   |
-        |   Constant(1) - scope-35
+        |   Constant(-1) - scope-37
         |   |
-        |   Project[bag][1] - scope-33
+        |   Project[bag][1] - scope-35
         |
-        |---Package(Packager)[tuple]{bytearray} - scope-32
-Tez vertex scope-41
+        |---Package(Packager)[tuple]{bytearray} - scope-34
+Tez vertex scope-43
 # Plan on vertex
-POIdentityInOutTez - scope-42	<-	 scope-24	->	 scope-43
+POIdentityInOutTez - scope-44	<-	 scope-24	->	 scope-45
 |   |
 |   Project[int][0] - scope-15
-Tez vertex scope-43
+Tez vertex scope-45
 # Plan on vertex
-POCounterTez - scope-17	->	 scope-49,scope-47
+POCounterTez - scope-17	->	 scope-51,scope-49
 |   |
 |   Project[int][0] - scope-15
 |
-|---New For Each(true)[tuple] - scope-46
+|---New For Each(true)[tuple] - scope-48
     |   |
-    |   Project[bag][1] - scope-45
+    |   Project[bag][1] - scope-47
     |
-    |---Package(LitePackager)[tuple]{int} - scope-44
-Tez vertex scope-47
-# Plan on vertex
-PORankStatsTez - scope-48	<-	 scope-43	->	 scope-49
+    |---Package(LitePackager)[tuple]{int} - scope-46
 Tez vertex scope-49
 # Plan on vertex
+PORankStatsTez - scope-50	<-	 scope-45	->	 scope-51
+Tez vertex scope-51
+# Plan on vertex
 b: Store(file:///tmp/output/d:org.apache.pig.builtin.PigStorage) - scope-22
 |
 |---New For Each(false,true)[tuple] - scope-21
@@ -98,6 +102,6 @@ b: Store(file:///tmp/output/d:org.apache
     |   |
     |   Project[bag][2] - scope-20
     |
-    |---PORankTez - scope-18	<-	 scope-43,scope-47
+    |---PORankTez - scope-18	<-	 scope-45,scope-49
         |   |
         |   Project[int][0] - scope-15

Modified: pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-6-OPTOFF.gld
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-6-OPTOFF.gld?rev=1603243&r1=1603242&r2=1603243&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-6-OPTOFF.gld (original)
+++ pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-6-OPTOFF.gld Tue Jun 17 18:15:26 2014
@@ -69,13 +69,13 @@ POValueOutputTez - scope-138	->	 [scope-
 |
 |---New For Each(false)[tuple] - scope-137
     |   |
-    |   POUserFunc(org.apache.pig.impl.builtin.PartitionSkewedKeys)[tuple] - scope-136
+    |   POUserFunc(org.apache.pig.backend.hadoop.executionengine.tez.PartitionSkewedKeysTez)[tuple] - scope-136
     |   |
     |   |---Project[tuple][*] - scope-135
     |
     |---New For Each(false,false)[tuple] - scope-134
         |   |
-        |   Constant(1) - scope-133
+        |   Constant(-1) - scope-133
         |   |
         |   Project[bag][1] - scope-131
         |

Modified: pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-6.gld
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-6.gld?rev=1603243&r1=1603242&r2=1603243&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-6.gld (original)
+++ pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-6.gld Tue Jun 17 18:15:26 2014
@@ -83,13 +83,13 @@ POValueOutputTez - scope-52	->	 [scope-5
 |
 |---New For Each(false)[tuple] - scope-51
     |   |
-    |   POUserFunc(org.apache.pig.impl.builtin.PartitionSkewedKeys)[tuple] - scope-50
+    |   POUserFunc(org.apache.pig.backend.hadoop.executionengine.tez.PartitionSkewedKeysTez)[tuple] - scope-50
     |   |
     |   |---Project[tuple][*] - scope-49
     |
     |---New For Each(false,false)[tuple] - scope-48
         |   |
-        |   Constant(1) - scope-47
+        |   Constant(-1) - scope-47
         |   |
         |   Project[bag][1] - scope-45
         |

Modified: pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-7-OPTOFF.gld
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-7-OPTOFF.gld?rev=1603243&r1=1603242&r2=1603243&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-7-OPTOFF.gld (original)
+++ pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-7-OPTOFF.gld Tue Jun 17 18:15:26 2014
@@ -2,90 +2,94 @@
 # There are 1 DAGs in the session
 #--------------------------------------------------
 #--------------------------------------------------
-# TEZ DAG plan: scope-117
+# TEZ DAG plan: scope-125
 #--------------------------------------------------
-Tez vertex scope-88	->	Tez vertex scope-90,
-Tez vertex scope-89	->	Tez vertex scope-90,
-Tez vertex scope-90	->	Tez vertex scope-110,Tez vertex scope-100,
-Tez vertex scope-100	->	Tez vertex scope-110,
-Tez vertex scope-110	->	Tez vertex scope-112,
-Tez vertex scope-112
+Tez vertex scope-94	->	Tez vertex scope-96,
+Tez vertex scope-95	->	Tez vertex scope-96,
+Tez vertex scope-96	->	Tez vertex scope-118,Tez vertex scope-108,
+Tez vertex scope-108	->	Tez vertex scope-118,
+Tez vertex scope-118	->	Tez vertex scope-120,
+Tez vertex scope-120
 
-Tez vertex scope-88
+Tez vertex scope-94
 # Plan on vertex
-POValueOutputTez - scope-92	->	 [scope-90]
+POValueOutputTez - scope-98	->	 [scope-96]
 |
-|---a: New For Each(false,false)[bag] - scope-75
+|---a: New For Each(false,false)[bag] - scope-81
     |   |
-    |   Cast[int] - scope-70
+    |   Cast[int] - scope-76
     |   |
-    |   |---Project[bytearray][0] - scope-69
+    |   |---Project[bytearray][0] - scope-75
     |   |
-    |   Cast[chararray] - scope-73
+    |   Cast[chararray] - scope-79
     |   |
-    |   |---Project[bytearray][1] - scope-72
+    |   |---Project[bytearray][1] - scope-78
     |
-    |---a: Load(file:///tmp/input:org.apache.pig.builtin.PigStorage) - scope-68
-Tez vertex scope-89
+    |---a: Load(file:///tmp/input:org.apache.pig.builtin.PigStorage) - scope-74
+Tez vertex scope-95
 # Plan on vertex
-POValueOutputTez - scope-93	->	 [scope-90]
+POValueOutputTez - scope-99	->	 [scope-96]
 |
-|---c: New For Each(false,false)[bag] - scope-83
+|---c: New For Each(false,false)[bag] - scope-89
     |   |
-    |   Cast[int] - scope-78
+    |   Cast[int] - scope-84
     |   |
-    |   |---Project[bytearray][1] - scope-77
+    |   |---Project[bytearray][1] - scope-83
     |   |
-    |   Cast[chararray] - scope-81
+    |   Cast[chararray] - scope-87
     |   |
-    |   |---Project[bytearray][0] - scope-80
+    |   |---Project[bytearray][0] - scope-86
     |
-    |---b: Load(file:///tmp/input:org.apache.pig.builtin.PigStorage) - scope-76
-Tez vertex scope-90
+    |---b: Load(file:///tmp/input:org.apache.pig.builtin.PigStorage) - scope-82
+Tez vertex scope-96
 # Plan on vertex
-Local Rearrange[tuple]{tuple}(false) - scope-96	->	 scope-100
+Local Rearrange[tuple]{tuple}(false) - scope-102	->	 scope-108
 |   |
-|   Constant(DummyVal) - scope-95
+|   Constant(DummyVal) - scope-101
 |
-|---ReservoirSample - scope-99
+|---New For Each(true,true)[tuple] - scope-107
+    |   |
+    |   Project[int][0] - scope-91
+    |   |
+    |   POUserFunc(org.apache.pig.impl.builtin.GetMemNumRows)[tuple] - scope-106
+    |   |
+    |   |---Project[tuple][*] - scope-105
     |
-    |---New For Each(false)[tuple] - scope-98
-        |   |
-        |   Project[int][0] - scope-97
+    |---ReservoirSample - scope-104
         |
-        |---d: Local Rearrange[tuple]{int}(false) - scope-94	->	 scope-110
+        |---d: Local Rearrange[tuple]{int}(false) - scope-100	->	 scope-118
             |   |
-            |   Project[int][0] - scope-85
+            |   Project[int][0] - scope-91
             |
-            |---POShuffledValueInputTez - scope-91	<-	 [scope-88, scope-89]
-Tez vertex scope-100
+            |---POShuffledValueInputTez - scope-97	<-	 [scope-94, scope-95]
+Tez vertex scope-108
 # Plan on vertex
-POValueOutputTez - scope-109	->	 [scope-110]
+POValueOutputTez - scope-117	->	 [scope-118]
 |
-|---New For Each(false)[tuple] - scope-108
+|---New For Each(false)[tuple] - scope-116
     |   |
-    |   POUserFunc(org.apache.pig.impl.builtin.FindQuantiles)[tuple] - scope-107
+    |   POUserFunc(org.apache.pig.backend.hadoop.executionengine.tez.FindQuantilesTez)[tuple] - scope-115
     |   |
-    |   |---Project[tuple][*] - scope-106
+    |   |---Project[tuple][*] - scope-114
     |
-    |---New For Each(false,false)[tuple] - scope-105
+    |---New For Each(false,false)[tuple] - scope-113
         |   |
-        |   Constant(1) - scope-104
+        |   Constant(-1) - scope-112
         |   |
-        |   Project[bag][1] - scope-102
+        |   Project[bag][1] - scope-110
         |
-        |---Package(Packager)[tuple]{bytearray} - scope-101
-Tez vertex scope-110
+        |---Package(Packager)[tuple]{bytearray} - scope-109
+Tez vertex scope-118
 # Plan on vertex
-POIdentityInOutTez - scope-111	<-	 scope-90	->	 scope-112
+POIdentityInOutTez - scope-119	<-	 scope-96	->	 scope-120
 |   |
-|   Project[int][0] - scope-85
-Tez vertex scope-112
+|   Project[int][0] - scope-91
+Tez vertex scope-120
 # Plan on vertex
-d: Store(file:///tmp/output:org.apache.pig.builtin.PigStorage) - scope-87
+d: Store(file:///tmp/output:org.apache.pig.builtin.PigStorage) - scope-93
 |
-|---New For Each(true)[tuple] - scope-115
+|---New For Each(true)[tuple] - scope-123
     |   |
-    |   Project[bag][1] - scope-114
+    |   Project[bag][1] - scope-122
     |
-    |---Package(LitePackager)[tuple]{int} - scope-113
+    |---Package(LitePackager)[tuple]{int} - scope-121

Modified: pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-7.gld
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-7.gld?rev=1603243&r1=1603242&r2=1603243&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-7.gld (original)
+++ pig/trunk/test/org/apache/pig/test/data/GoldenFiles/tez/TEZC-Union-7.gld Tue Jun 17 18:15:26 2014
@@ -2,29 +2,33 @@
 # There are 1 DAGs in the session
 #--------------------------------------------------
 #--------------------------------------------------
-# TEZ DAG plan: scope-67
+# TEZ DAG plan: scope-73
 #--------------------------------------------------
-Tez vertex scope-20	->	Tez vertex scope-49,Tez vertex scope-50,
-Tez vertex scope-21	->	Tez vertex scope-49,Tez vertex scope-50,
-Tez vertex scope-50	->	Tez vertex scope-32,
-Tez vertex scope-32	->	Tez vertex scope-42,
-Tez vertex scope-49	->	Tez vertex scope-42,
-Tez vertex scope-42	->	Tez vertex scope-44,
-Tez vertex scope-44
+Tez vertex scope-20	->	Tez vertex scope-51,Tez vertex scope-52,
+Tez vertex scope-21	->	Tez vertex scope-51,Tez vertex scope-52,
+Tez vertex scope-52	->	Tez vertex scope-34,
+Tez vertex scope-34	->	Tez vertex scope-44,
+Tez vertex scope-51	->	Tez vertex scope-44,
+Tez vertex scope-44	->	Tez vertex scope-46,
+Tez vertex scope-46
 
 Tez vertex scope-20
 # Plan on vertex
-Local Rearrange[tuple]{tuple}(false) - scope-54	->	 scope-32
+Local Rearrange[tuple]{tuple}(false) - scope-54	->	 scope-34
 |   |
 |   Constant(DummyVal) - scope-55
 |
-|---ReservoirSample - scope-53
+|---New For Each(true,true)[tuple] - scope-62
+    |   |
+    |   Project[int][0] - scope-59
+    |   |
+    |   POUserFunc(org.apache.pig.impl.builtin.GetMemNumRows)[tuple] - scope-61
+    |   |
+    |   |---Project[tuple][*] - scope-60
     |
-    |---New For Each(false)[tuple] - scope-52
-        |   |
-        |   Project[int][0] - scope-51
+    |---ReservoirSample - scope-53
         |
-        |---d: Local Rearrange[tuple]{int}(false) - scope-57	->	 scope-42
+        |---d: Local Rearrange[tuple]{int}(false) - scope-57	->	 scope-44
             |   |
             |   Project[int][0] - scope-58
             |
@@ -41,19 +45,23 @@ Local Rearrange[tuple]{tuple}(false) - s
                 |---a: Load(file:///tmp/input:org.apache.pig.builtin.PigStorage) - scope-0
 Tez vertex scope-21
 # Plan on vertex
-Local Rearrange[tuple]{tuple}(false) - scope-62	->	 scope-32
+Local Rearrange[tuple]{tuple}(false) - scope-64	->	 scope-34
 |   |
-|   Constant(DummyVal) - scope-63
+|   Constant(DummyVal) - scope-65
 |
-|---ReservoirSample - scope-61
+|---New For Each(true,true)[tuple] - scope-72
+    |   |
+    |   Project[int][0] - scope-69
+    |   |
+    |   POUserFunc(org.apache.pig.impl.builtin.GetMemNumRows)[tuple] - scope-71
+    |   |
+    |   |---Project[tuple][*] - scope-70
     |
-    |---New For Each(false)[tuple] - scope-60
-        |   |
-        |   Project[int][0] - scope-59
+    |---ReservoirSample - scope-63
         |
-        |---d: Local Rearrange[tuple]{int}(false) - scope-65	->	 scope-42
+        |---d: Local Rearrange[tuple]{int}(false) - scope-67	->	 scope-44
             |   |
-            |   Project[int][0] - scope-66
+            |   Project[int][0] - scope-68
             |
             |---c: New For Each(false,false)[bag] - scope-15
                 |   |
@@ -66,38 +74,38 @@ Local Rearrange[tuple]{tuple}(false) - s
                 |   |---Project[bytearray][0] - scope-12
                 |
                 |---b: Load(file:///tmp/input:org.apache.pig.builtin.PigStorage) - scope-8
-Tez vertex group scope-50	<-	 [scope-20, scope-21]	->	 scope-32
+Tez vertex group scope-52	<-	 [scope-20, scope-21]	->	 scope-34
 # No plan on vertex group
-Tez vertex scope-32
+Tez vertex scope-34
 # Plan on vertex
-POValueOutputTez - scope-41	->	 [scope-42]
+POValueOutputTez - scope-43	->	 [scope-44]
 |
-|---New For Each(false)[tuple] - scope-40
+|---New For Each(false)[tuple] - scope-42
     |   |
-    |   POUserFunc(org.apache.pig.impl.builtin.FindQuantiles)[tuple] - scope-39
+    |   POUserFunc(org.apache.pig.backend.hadoop.executionengine.tez.FindQuantilesTez)[tuple] - scope-41
     |   |
-    |   |---Project[tuple][*] - scope-38
+    |   |---Project[tuple][*] - scope-40
     |
-    |---New For Each(false,false)[tuple] - scope-37
+    |---New For Each(false,false)[tuple] - scope-39
         |   |
-        |   Constant(1) - scope-36
+        |   Constant(-1) - scope-38
         |   |
-        |   Project[bag][1] - scope-34
+        |   Project[bag][1] - scope-36
         |
-        |---Package(Packager)[tuple]{bytearray} - scope-33
-Tez vertex group scope-49	<-	 [scope-20, scope-21]	->	 scope-42
+        |---Package(Packager)[tuple]{bytearray} - scope-35
+Tez vertex group scope-51	<-	 [scope-20, scope-21]	->	 scope-44
 # No plan on vertex group
-Tez vertex scope-42
+Tez vertex scope-44
 # Plan on vertex
-POIdentityInOutTez - scope-43	<-	 scope-49	->	 scope-44
+POIdentityInOutTez - scope-45	<-	 scope-51	->	 scope-46
 |   |
 |   Project[int][0] - scope-17
-Tez vertex scope-44
+Tez vertex scope-46
 # Plan on vertex
 d: Store(file:///tmp/output:org.apache.pig.builtin.PigStorage) - scope-19
 |
-|---New For Each(true)[tuple] - scope-47
+|---New For Each(true)[tuple] - scope-49
     |   |
-    |   Project[bag][1] - scope-46
+    |   Project[bag][1] - scope-48
     |
-    |---Package(LitePackager)[tuple]{int} - scope-45
+    |---Package(LitePackager)[tuple]{int} - scope-47

Added: pig/trunk/test/org/apache/pig/tez/TestTezAutoParallelism.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/tez/TestTezAutoParallelism.java?rev=1603243&view=auto
==============================================================================
--- pig/trunk/test/org/apache/pig/tez/TestTezAutoParallelism.java (added)
+++ pig/trunk/test/org/apache/pig/tez/TestTezAutoParallelism.java Tue Jun 17 18:15:26 2014
@@ -0,0 +1,229 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.pig.tez;
+
+import static org.junit.Assert.assertEquals;
+import java.io.File;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.io.PrintWriter;
+import java.util.Properties;
+import java.util.Random;
+
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.PathFilter;
+import org.apache.pig.PigConfiguration;
+import org.apache.pig.PigServer;
+import org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.InputSizeReducerEstimator;
+import org.apache.pig.test.MiniGenericCluster;
+import org.apache.pig.test.Util;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+public class TestTezAutoParallelism {
+    private static final String INPUT_FILE1 = TestTezAutoParallelism.class.getName() + "_1";
+    private static final String INPUT_FILE2 = TestTezAutoParallelism.class.getName() + "_2";
+    private static final String INPUT_DIR = "build/test/data";
+
+    private static PigServer pigServer;
+    private static Properties properties;
+    private static MiniGenericCluster cluster;
+
+    @BeforeClass
+    public static void oneTimeSetUp() throws Exception {
+        cluster = MiniGenericCluster.buildCluster(MiniGenericCluster.EXECTYPE_TEZ);
+        properties = cluster.getProperties();
+        createFiles();
+    }
+
+    @AfterClass
+    public static void oneTimeTearDown() throws Exception {
+        deleteFiles();
+        cluster.shutDown();
+    }
+
+    @Before
+    public void setUp() throws Exception {
+        pigServer = new PigServer(MiniGenericCluster.EXECTYPE_TEZ, properties);
+    }
+
+    @After
+    public void tearDown() throws Exception {
+        pigServer.shutdown();
+        pigServer = null;
+    }
+
+    private static void createFiles() throws IOException {
+        new File(INPUT_DIR).mkdir();
+
+        PrintWriter w = new PrintWriter(new FileWriter(INPUT_DIR + "/" + INPUT_FILE1));
+
+        String boyNames[] = {"Noah", "Liam", "Jacob", "Mason", "William",
+                "Ethan", "Michael", "Alexander", "Jayden", "Daniel"};
+        String girlNames[] = {"Sophia", "Emma", "Olivia", "Isabella", "Ava",
+                "Mia", "Emily", "Abigail", "Madison", "Elizabeth"};
+
+        String names[] = new String[boyNames.length + girlNames.length];
+        for (int i=0;i<boyNames.length;i++) {
+            names[i] = boyNames[i];
+        }
+        for (int i=0;i<girlNames.length;i++) {
+            names[boyNames.length+i] = girlNames[i];
+        }
+
+        Random rand = new Random(1);
+        for (int i=0;i<1000;i++) {
+            w.println(names[rand.nextInt(names.length)] + "\t" + rand.nextInt(18));
+        }
+        w.close();
+        Util.copyFromLocalToCluster(cluster, INPUT_DIR + "/" + INPUT_FILE1, INPUT_FILE1);
+        
+        w = new PrintWriter(new FileWriter(INPUT_DIR + "/" + INPUT_FILE2));
+        for (String name : boyNames) {
+            w.println(name + "\t" + "M");
+        }
+        for (String name : girlNames) {
+            w.println(name + "\t" + "F");
+        }
+        w.close();
+        Util.copyFromLocalToCluster(cluster, INPUT_DIR + "/" + INPUT_FILE2, INPUT_FILE2);
+    }
+
+    private static void deleteFiles() {
+        Util.deleteDirectory(new File(INPUT_DIR));
+    }
+
+    @Test
+    public void testGroupBy() throws IOException{
+        // parallelism is 3 originally, reduce to 1
+        pigServer.getPigContext().getProperties().setProperty(PigConfiguration.PIG_NO_SPLIT_COMBINATION, "true");
+        pigServer.getPigContext().getProperties().setProperty("mapred.max.split.size", "3000");
+        pigServer.getPigContext().getProperties().setProperty(InputSizeReducerEstimator.BYTES_PER_REDUCER_PARAM, 
+                Long.toString(InputSizeReducerEstimator.DEFAULT_BYTES_PER_REDUCER));
+        pigServer.registerQuery("A = load '" + INPUT_FILE1 + "' as (name:chararray, age:int);");
+        pigServer.registerQuery("B = group A by name;");
+        pigServer.store("B", "output1");
+        FileSystem fs = cluster.getFileSystem();
+        FileStatus[] files = fs.listStatus(new Path("output1"), new PathFilter(){
+            public boolean accept(Path path) {
+                if (path.getName().startsWith("part")) {
+                    return true;
+                }
+                return false;
+            }
+        });
+        assertEquals(files.length, 1);
+    }
+
+    @Test
+    public void testOrderbyDecreaseParallelism() throws IOException{
+        // order by parallelism is 3 originally, reduce to 1
+        pigServer.getPigContext().getProperties().setProperty(PigConfiguration.PIG_NO_SPLIT_COMBINATION, "true");
+        pigServer.getPigContext().getProperties().setProperty("mapred.max.split.size", "3000");
+        pigServer.getPigContext().getProperties().setProperty(InputSizeReducerEstimator.BYTES_PER_REDUCER_PARAM, 
+                Long.toString(InputSizeReducerEstimator.DEFAULT_BYTES_PER_REDUCER));
+        pigServer.registerQuery("A = load '" + INPUT_FILE1 + "' as (name:chararray, age:int);");
+        pigServer.registerQuery("B = group A by name parallel 3;");
+        pigServer.registerQuery("C = foreach B generate group as name, AVG(A.age) as age;");
+        pigServer.registerQuery("D = order C by age;");
+        pigServer.store("D", "output2");
+        FileSystem fs = cluster.getFileSystem();
+        FileStatus[] files = fs.listStatus(new Path("output2"), new PathFilter(){
+            public boolean accept(Path path) {
+                if (path.getName().startsWith("part")) {
+                    return true;
+                }
+                return false;
+            }
+        });
+        assertEquals(files.length, 1);
+    }
+
+    @Test
+    public void testOrderbyIncreaseParallelism() throws IOException{
+        // order by parallelism is 3 originally, increase to 4
+        pigServer.getPigContext().getProperties().setProperty(PigConfiguration.PIG_NO_SPLIT_COMBINATION, "true");
+        pigServer.getPigContext().getProperties().setProperty("mapred.max.split.size", "3000");
+        pigServer.getPigContext().getProperties().setProperty(InputSizeReducerEstimator.BYTES_PER_REDUCER_PARAM, "1000");
+        pigServer.registerQuery("A = load '" + INPUT_FILE1 + "' as (name:chararray, age:int);");
+        pigServer.registerQuery("B = group A by name parallel 3;");
+        pigServer.registerQuery("C = foreach B generate group as name, AVG(A.age) as age;");
+        pigServer.registerQuery("D = order C by age;");
+        pigServer.store("D", "output3");
+        FileSystem fs = cluster.getFileSystem();
+        FileStatus[] files = fs.listStatus(new Path("output3"), new PathFilter(){
+            public boolean accept(Path path) {
+                if (path.getName().startsWith("part")) {
+                    return true;
+                }
+                return false;
+            }
+        });
+        assertEquals(files.length, 4);
+    }
+
+    @Test
+    public void testSkewedJoinDecreaseParallelism() throws IOException{
+        // skewed join parallelism is 4 originally, reduce to 1
+        pigServer.getPigContext().getProperties().setProperty(PigConfiguration.PIG_NO_SPLIT_COMBINATION, "true");
+        pigServer.getPigContext().getProperties().setProperty("mapred.max.split.size", "3000");
+        pigServer.getPigContext().getProperties().setProperty(InputSizeReducerEstimator.BYTES_PER_REDUCER_PARAM, 
+                Long.toString(InputSizeReducerEstimator.DEFAULT_BYTES_PER_REDUCER));
+        pigServer.registerQuery("A = load '" + INPUT_FILE1 + "' as (name:chararray, age:int);");
+        pigServer.registerQuery("B = load '" + INPUT_FILE2 + "' as (name:chararray, gender:chararray);");
+        pigServer.registerQuery("C = join A by name, B by name using 'skewed';");
+        pigServer.store("C", "output4");
+        FileSystem fs = cluster.getFileSystem();
+        FileStatus[] files = fs.listStatus(new Path("output4"), new PathFilter(){
+            public boolean accept(Path path) {
+                if (path.getName().startsWith("part")) {
+                    return true;
+                }
+                return false;
+            }
+        });
+        assertEquals(files.length, 1);
+    }
+
+    @Test
+    public void testSkewedJoinIncreaseParallelism() throws IOException{
+        // skewed join parallelism is 3 originally, increase to 5
+        pigServer.getPigContext().getProperties().setProperty(PigConfiguration.PIG_NO_SPLIT_COMBINATION, "true");
+        pigServer.getPigContext().getProperties().setProperty("mapred.max.split.size", "3000");
+        pigServer.getPigContext().getProperties().setProperty(InputSizeReducerEstimator.BYTES_PER_REDUCER_PARAM, "80000");
+        pigServer.registerQuery("A = load '" + INPUT_FILE1 + "' as (name:chararray, age:int);");
+        pigServer.registerQuery("B = load '" + INPUT_FILE2 + "' as (name:chararray, gender:chararray);");
+        pigServer.registerQuery("C = join A by name, B by name using 'skewed';");
+        pigServer.store("C", "output5");
+        FileSystem fs = cluster.getFileSystem();
+        FileStatus[] files = fs.listStatus(new Path("output5"), new PathFilter(){
+            public boolean accept(Path path) {
+                if (path.getName().startsWith("part")) {
+                    return true;
+                }
+                return false;
+            }
+        });
+        assertEquals(files.length, 5);
+    }
+}

Modified: pig/trunk/test/org/apache/pig/tez/TestTezJobControlCompiler.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/tez/TestTezJobControlCompiler.java?rev=1603243&r1=1603242&r2=1603243&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/tez/TestTezJobControlCompiler.java (original)
+++ pig/trunk/test/org/apache/pig/tez/TestTezJobControlCompiler.java Tue Jun 17 18:15:26 2014
@@ -21,21 +21,33 @@ import static org.junit.Assert.assertEqu
 import static org.junit.Assert.assertNotNull;
 import static org.junit.Assert.assertTrue;
 
+import java.io.File;
+import java.io.IOException;
 import java.net.URI;
+import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Properties;
 
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.mapreduce.InputFormat;
+import org.apache.hadoop.mapreduce.InputSplit;
+import org.apache.hadoop.mapreduce.JobContext;
+import org.apache.hadoop.mapreduce.lib.input.FileSplit;
+import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;
 import org.apache.hadoop.yarn.api.records.LocalResource;
+import org.apache.pig.PigConfiguration;
 import org.apache.pig.PigServer;
 import org.apache.pig.backend.executionengine.ExecException;
 import org.apache.pig.backend.hadoop.executionengine.physicalLayer.plans.PhysicalPlan;
+import org.apache.pig.backend.hadoop.executionengine.tez.MultiQueryOptimizerTez;
 import org.apache.pig.backend.hadoop.executionengine.tez.TezCompiler;
 import org.apache.pig.backend.hadoop.executionengine.tez.TezJobControlCompiler;
 import org.apache.pig.backend.hadoop.executionengine.tez.TezLocalExecType;
 import org.apache.pig.backend.hadoop.executionengine.tez.TezOperPlan;
 import org.apache.pig.backend.hadoop.executionengine.tez.TezOperator;
+import org.apache.pig.builtin.PigStorage;
 import org.apache.pig.impl.PigContext;
 import org.apache.pig.impl.util.Pair;
 import org.apache.pig.test.Util;
@@ -56,7 +68,12 @@ import org.junit.runner.RunWith;
 @TestOrder({
     "testRun1",
     "testRun2",
-    "testRun3"
+    "testRun3",
+    "testTezParallelismEstimatorOrderBy",
+    "testTezParallelismEstimatorFilterFlatten",
+    "testTezParallelismEstimatorHashJoin",
+    "testTezParallelismEstimatorSplitBranch",
+    "testTezParallelismDefaultParallelism"
 })
 public class TestTezJobControlCompiler {
     private static PigContext pc;
@@ -66,7 +83,6 @@ public class TestTezJobControlCompiler {
 
     @BeforeClass
     public static void setUpBeforeClass() throws Exception {
-        pc = new PigContext(new TezLocalExecType(), new Properties());
         input1 = Util.createTempFileDelOnExit("input1", "txt").toURI();
         input2 = Util.createTempFileDelOnExit("input2", "txt").toURI();
     }
@@ -77,6 +93,7 @@ public class TestTezJobControlCompiler {
 
     @Before
     public void setUp() throws ExecException {
+        pc = new PigContext(new TezLocalExecType(), new Properties());
         pigServer = new PigServer(pc);
     }
 
@@ -167,11 +184,112 @@ public class TestTezJobControlCompiler {
         assertTrue(leaf.getInputVertices().contains(root1));
     }
 
+    static public class ArbitarySplitsInputformat extends TextInputFormat {
+        @Override
+        public List<InputSplit> getSplits(JobContext job) throws IOException {
+            String inputDir = job.getConfiguration().get(INPUT_DIR, "");
+            String numSplitString = inputDir.substring(inputDir.lastIndexOf(File.separator)+1);
+            int numSplit = Integer.parseInt(numSplitString);
+            List<InputSplit> splits = new ArrayList<InputSplit>();
+            for (int i=0;i<numSplit;i++) {
+                splits.add(new FileSplit(new Path("dummy"), 0, 0, null));
+            }
+            return splits;
+        }
+    }
+
+    static public class ArbitarySplitsLoader extends PigStorage {
+        public ArbitarySplitsLoader() {}
+
+        @Override
+        public InputFormat getInputFormat() {
+            return new ArbitarySplitsInputformat();
+        }
+    }
+
+    @Test
+    public void testTezParallelismEstimatorOrderBy() throws Exception{
+        pc.getProperties().setProperty(PigConfiguration.PIG_NO_SPLIT_COMBINATION, "true");
+        String query = "a = load '2' using " + ArbitarySplitsLoader.class.getName()
+                + "() as (name:chararray, age:int, gpa:double);"
+                + "b = group a by name parallel 3;"
+                + "c = foreach b generate group as name, AVG(a.age) as age;"
+                + "d = order c by age;"
+                + "store d into 'output';";
+        Pair<TezOperPlan, DAG> compiledPlan = compile(query);
+        TezOperator sortOper = compiledPlan.first.getLeaves().get(0);
+        Vertex sortVertex = compiledPlan.second.getVertex(sortOper.getOperatorKey().toString());
+        assertEquals(sortVertex.getParallelism(), -1);
+    }
+
+    @Test
+    public void testTezParallelismEstimatorFilterFlatten() throws Exception{
+        pc.getProperties().setProperty(PigConfiguration.PIG_NO_SPLIT_COMBINATION, "true");
+        String query = "a = load '10' using " + ArbitarySplitsLoader.class.getName()
+                + "() as (name:chararray, age:int, gpa:double);"
+                + "b = filter a by age>20;"
+                + "c = group b by name;"
+                + "d = foreach c generate group, flatten(b.gpa);"
+                + "e = group d by group;"
+                + "store e into 'output';";
+        Pair<TezOperPlan, DAG> compiledPlan = compile(query);
+        TezOperator leafOper = compiledPlan.first.getLeaves().get(0);
+        Vertex leafVertex = compiledPlan.second.getVertex(leafOper.getOperatorKey().toString());
+        assertEquals(leafVertex.getParallelism(), 70);
+    }
+
+    @Test
+    public void testTezParallelismEstimatorHashJoin() throws Exception{
+        pc.getProperties().setProperty(PigConfiguration.PIG_NO_SPLIT_COMBINATION, "true");
+        String query = "a = load '10' using " + ArbitarySplitsLoader.class.getName()
+                + "() as (name:chararray, age:int, gpa:double);"
+                + "b = load '5' using " + ArbitarySplitsLoader.class.getName()
+                + "() as (name:chararray, course:chararray);"
+                + "c = join a by name, b by name;"
+                + "store c into 'output';";
+        Pair<TezOperPlan, DAG> compiledPlan = compile(query);
+        TezOperator leafOper = compiledPlan.first.getLeaves().get(0);
+        Vertex leafVertex = compiledPlan.second.getVertex(leafOper.getOperatorKey().toString());
+        assertEquals(leafVertex.getParallelism(), 15);
+    }
+    
+    @Test
+    public void testTezParallelismEstimatorSplitBranch() throws Exception{
+        pc.getProperties().setProperty(PigConfiguration.PIG_NO_SPLIT_COMBINATION, "true");
+        String query = "a = load '10' using " + ArbitarySplitsLoader.class.getName()
+                + "() as (name:chararray, age:int, gpa:double);"
+                + "b = filter a by age>20;"
+                + "c = filter a by age>50;"
+                + "store b into 'o1';"
+                + "d = group c by name;"
+                + "store d into 'o2';";
+        Pair<TezOperPlan, DAG> compiledPlan = compile(query);
+        TezOperator leafOper = compiledPlan.first.getLeaves().get(0);
+        Vertex leafVertex = compiledPlan.second.getVertex(leafOper.getOperatorKey().toString());
+        assertEquals(leafVertex.getParallelism(), 7);
+    }
+    
+    @Test
+    public void testTezParallelismDefaultParallelism() throws Exception{
+        pc.getProperties().setProperty(PigConfiguration.PIG_NO_SPLIT_COMBINATION, "true");
+        pc.defaultParallel = 5;
+        String query = "a = load '10' using " + ArbitarySplitsLoader.class.getName()
+                + "() as (name:chararray, age:int, gpa:double);"
+                + "b = group a by name;"
+                + "store b into 'output';";
+        Pair<TezOperPlan, DAG> compiledPlan = compile(query);
+        TezOperator leafOper = compiledPlan.first.getLeaves().get(0);
+        Vertex leafVertex = compiledPlan.second.getVertex(leafOper.getOperatorKey().toString());
+        assertEquals(leafVertex.getParallelism(), 5);
+    }
+
     private Pair<TezOperPlan, DAG> compile(String query) throws Exception {
         PhysicalPlan pp = Util.buildPp(pigServer, query);
         TezCompiler comp = new TezCompiler(pp, pc);
         TezOperPlan tezPlan = comp.compile();
         TezJobControlCompiler jobComp = new TezJobControlCompiler(pc, new Configuration());
+        MultiQueryOptimizerTez mqOptimizer = new MultiQueryOptimizerTez(tezPlan);
+        mqOptimizer.visit();
         DAG dag = jobComp.buildDAG(tezPlan, new HashMap<String, LocalResource>());
         return new Pair<TezOperPlan, DAG>(tezPlan, dag);
     }

Modified: pig/trunk/test/tez-tests
URL: http://svn.apache.org/viewvc/pig/trunk/test/tez-tests?rev=1603243&r1=1603242&r2=1603243&view=diff
==============================================================================
--- pig/trunk/test/tez-tests (original)
+++ pig/trunk/test/tez-tests Tue Jun 17 18:15:26 2014
@@ -46,4 +46,5 @@
 **/TestSecondarySortTez.java
 **/TestTezCompiler.java
 **/TestTezJobControlCompiler.java
-**/TestTezLauncher.java
\ No newline at end of file
+**/TestTezLauncher.java
+**/TestTezAutoParallelism.java



Mime
View raw message