hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From xu...@apache.org
Subject svn commit: r1642997 [25/42] - in /hive/branches/spark: itests/src/test/resources/ ql/src/java/org/apache/hadoop/hive/ql/optimizer/spark/ ql/src/test/results/clientpositive/spark/
Date Mon, 15 Dec 2014 17:11:42 GMT
Modified: hive/branches/spark/ql/src/test/results/clientpositive/spark/ppd_join.q.out
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/results/clientpositive/spark/ppd_join.q.out?rev=1642997&r1=1642996&r2=1642997&view=diff
==============================================================================
--- hive/branches/spark/ql/src/test/results/clientpositive/spark/ppd_join.q.out (original)
+++ hive/branches/spark/ql/src/test/results/clientpositive/spark/ppd_join.q.out Tue Dec  2 19:57:10 2014
@@ -21,13 +21,14 @@ ON src1.c1 = src2.c3 AND src1.c1 < '400'
 WHERE src1.c1 > '20' and (src1.c2 < 'val_50' or src1.c1 > '2') and (src2.c3 > '50' or src1.c1 < '50') and (src2.c3 <> '4')
 POSTHOOK: type: QUERY
 STAGE DEPENDENCIES:
-  Stage-2 is a root stage
-  Stage-1 depends on stages: Stage-2
+  Stage-1 is a root stage
   Stage-0 depends on stages: Stage-1
 
 STAGE PLANS:
-  Stage: Stage-2
+  Stage: Stage-1
     Spark
+      Edges:
+        Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1), Map 3 (PARTITION-LEVEL SORT, 1)
 #### A masked pattern was here ####
       Vertices:
         Map 1 
@@ -45,21 +46,13 @@ STAGE PLANS:
                       Filter Operator
                         predicate: (_col0 < '400') (type: boolean)
                         Statistics: Num rows: 4 Data size: 42 Basic stats: COMPLETE Column stats: NONE
-                        Spark HashTable Sink Operator
-                          condition expressions:
-                            0 {_col1}
-                            1 {_col0} {_col1}
-                          keys:
-                            0 _col0 (type: string)
-                            1 _col0 (type: string)
-            Local Work:
-              Map Reduce Local Work
-
-  Stage: Stage-1
-    Spark
-#### A masked pattern was here ####
-      Vertices:
-        Map 2 
+                        Reduce Output Operator
+                          key expressions: _col0 (type: string)
+                          sort order: +
+                          Map-reduce partition columns: _col0 (type: string)
+                          Statistics: Num rows: 4 Data size: 42 Basic stats: COMPLETE Column stats: NONE
+                          value expressions: _col1 (type: string)
+        Map 3 
             Map Operator Tree:
                 TableScan
                   alias: src
@@ -74,35 +67,36 @@ STAGE PLANS:
                       Filter Operator
                         predicate: (_col0 < '400') (type: boolean)
                         Statistics: Num rows: 6 Data size: 63 Basic stats: COMPLETE Column stats: NONE
-                        Map Join Operator
-                          condition map:
-                               Inner Join 0 to 1
-                          condition expressions:
-                            0 {_col0} {_col1}
-                            1 {_col0} {_col1}
-                          keys:
-                            0 _col0 (type: string)
-                            1 _col0 (type: string)
-                          outputColumnNames: _col0, _col1, _col2, _col3
-                          input vertices:
-                            0 Map 1
-                          Statistics: Num rows: 6 Data size: 69 Basic stats: COMPLETE Column stats: NONE
-                          Filter Operator
-                            predicate: ((((_col0 > '20') and ((_col1 < 'val_50') or (_col0 > '2'))) and ((_col2 > '50') or (_col0 < '50'))) and (_col2 <> '4')) (type: boolean)
-                            Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
-                            Select Operator
-                              expressions: _col0 (type: string), _col3 (type: string)
-                              outputColumnNames: _col0, _col1
-                              Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
-                              File Output Operator
-                                compressed: false
-                                Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
-                                table:
-                                    input format: org.apache.hadoop.mapred.TextInputFormat
-                                    output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
-                                    serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
-            Local Work:
-              Map Reduce Local Work
+                        Reduce Output Operator
+                          key expressions: _col0 (type: string)
+                          sort order: +
+                          Map-reduce partition columns: _col0 (type: string)
+                          Statistics: Num rows: 6 Data size: 63 Basic stats: COMPLETE Column stats: NONE
+                          value expressions: _col1 (type: string)
+        Reducer 2 
+            Reduce Operator Tree:
+              Join Operator
+                condition map:
+                     Inner Join 0 to 1
+                condition expressions:
+                  0 {KEY.reducesinkkey0} {VALUE._col0}
+                  1 {KEY.reducesinkkey0} {VALUE._col0}
+                outputColumnNames: _col0, _col1, _col2, _col3
+                Statistics: Num rows: 6 Data size: 69 Basic stats: COMPLETE Column stats: NONE
+                Filter Operator
+                  predicate: ((((_col0 > '20') and ((_col1 < 'val_50') or (_col0 > '2'))) and ((_col2 > '50') or (_col0 < '50'))) and (_col2 <> '4')) (type: boolean)
+                  Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
+                  Select Operator
+                    expressions: _col0 (type: string), _col3 (type: string)
+                    outputColumnNames: _col0, _col1
+                    Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
+                    File Output Operator
+                      compressed: false
+                      Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
+                      table:
+                          input format: org.apache.hadoop.mapred.TextInputFormat
+                          output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+                          serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
 
   Stage: Stage-0
     Fetch Operator
@@ -567,13 +561,14 @@ ON src1.c1 = src2.c3 AND src1.c1 < '400'
 WHERE src1.c1 > '20' and (src1.c2 < 'val_50' or src1.c1 > '2') and (src2.c3 > '50' or src1.c1 < '50') and (src2.c3 <> '4')
 POSTHOOK: type: QUERY
 STAGE DEPENDENCIES:
-  Stage-2 is a root stage
-  Stage-1 depends on stages: Stage-2
+  Stage-1 is a root stage
   Stage-0 depends on stages: Stage-1
 
 STAGE PLANS:
-  Stage: Stage-2
+  Stage: Stage-1
     Spark
+      Edges:
+        Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3), Map 3 (PARTITION-LEVEL SORT, 3)
 #### A masked pattern was here ####
       Vertices:
         Map 1 
@@ -588,21 +583,13 @@ STAGE PLANS:
                       expressions: key (type: string), value (type: string)
                       outputColumnNames: _col0, _col1
                       Statistics: Num rows: 12 Data size: 127 Basic stats: COMPLETE Column stats: NONE
-                      Spark HashTable Sink Operator
-                        condition expressions:
-                          0 {_col1}
-                          1 {_col0} {_col1}
-                        keys:
-                          0 _col0 (type: string)
-                          1 _col0 (type: string)
-            Local Work:
-              Map Reduce Local Work
-
-  Stage: Stage-1
-    Spark
-#### A masked pattern was here ####
-      Vertices:
-        Map 2 
+                      Reduce Output Operator
+                        key expressions: _col0 (type: string)
+                        sort order: +
+                        Map-reduce partition columns: _col0 (type: string)
+                        Statistics: Num rows: 12 Data size: 127 Basic stats: COMPLETE Column stats: NONE
+                        value expressions: _col1 (type: string)
+        Map 3 
             Map Operator Tree:
                 TableScan
                   alias: src
@@ -614,35 +601,36 @@ STAGE PLANS:
                       expressions: key (type: string), value (type: string)
                       outputColumnNames: _col0, _col1
                       Statistics: Num rows: 18 Data size: 191 Basic stats: COMPLETE Column stats: NONE
-                      Map Join Operator
-                        condition map:
-                             Inner Join 0 to 1
-                        condition expressions:
-                          0 {_col0} {_col1}
-                          1 {_col0} {_col1}
-                        keys:
-                          0 _col0 (type: string)
-                          1 _col0 (type: string)
-                        outputColumnNames: _col0, _col1, _col2, _col3
-                        input vertices:
-                          0 Map 1
-                        Statistics: Num rows: 19 Data size: 210 Basic stats: COMPLETE Column stats: NONE
-                        Filter Operator
-                          predicate: ((((_col0 > '20') and ((_col1 < 'val_50') or (_col0 > '2'))) and ((_col2 > '50') or (_col0 < '50'))) and (_col2 <> '4')) (type: boolean)
-                          Statistics: Num rows: 2 Data size: 22 Basic stats: COMPLETE Column stats: NONE
-                          Select Operator
-                            expressions: _col0 (type: string), _col3 (type: string)
-                            outputColumnNames: _col0, _col1
-                            Statistics: Num rows: 2 Data size: 22 Basic stats: COMPLETE Column stats: NONE
-                            File Output Operator
-                              compressed: false
-                              Statistics: Num rows: 2 Data size: 22 Basic stats: COMPLETE Column stats: NONE
-                              table:
-                                  input format: org.apache.hadoop.mapred.TextInputFormat
-                                  output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
-                                  serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
-            Local Work:
-              Map Reduce Local Work
+                      Reduce Output Operator
+                        key expressions: _col0 (type: string)
+                        sort order: +
+                        Map-reduce partition columns: _col0 (type: string)
+                        Statistics: Num rows: 18 Data size: 191 Basic stats: COMPLETE Column stats: NONE
+                        value expressions: _col1 (type: string)
+        Reducer 2 
+            Reduce Operator Tree:
+              Join Operator
+                condition map:
+                     Inner Join 0 to 1
+                condition expressions:
+                  0 {KEY.reducesinkkey0} {VALUE._col0}
+                  1 {KEY.reducesinkkey0} {VALUE._col0}
+                outputColumnNames: _col0, _col1, _col2, _col3
+                Statistics: Num rows: 19 Data size: 210 Basic stats: COMPLETE Column stats: NONE
+                Filter Operator
+                  predicate: ((((_col0 > '20') and ((_col1 < 'val_50') or (_col0 > '2'))) and ((_col2 > '50') or (_col0 < '50'))) and (_col2 <> '4')) (type: boolean)
+                  Statistics: Num rows: 2 Data size: 22 Basic stats: COMPLETE Column stats: NONE
+                  Select Operator
+                    expressions: _col0 (type: string), _col3 (type: string)
+                    outputColumnNames: _col0, _col1
+                    Statistics: Num rows: 2 Data size: 22 Basic stats: COMPLETE Column stats: NONE
+                    File Output Operator
+                      compressed: false
+                      Statistics: Num rows: 2 Data size: 22 Basic stats: COMPLETE Column stats: NONE
+                      table:
+                          input format: org.apache.hadoop.mapred.TextInputFormat
+                          output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+                          serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
 
   Stage: Stage-0
     Fetch Operator

Modified: hive/branches/spark/ql/src/test/results/clientpositive/spark/ppd_join2.q.out
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/results/clientpositive/spark/ppd_join2.q.out?rev=1642997&r1=1642996&r2=1642997&view=diff
==============================================================================
--- hive/branches/spark/ql/src/test/results/clientpositive/spark/ppd_join2.q.out (original)
+++ hive/branches/spark/ql/src/test/results/clientpositive/spark/ppd_join2.q.out Tue Dec  2 19:57:10 2014
@@ -27,14 +27,15 @@ ON src1.c2 = src3.c6
 WHERE src1.c1 <> '311' and (src1.c2 <> 'val_50' or src1.c1 > '1') and (src2.c3 <> '10' or src1.c1 <> '10') and (src2.c3 <> '14') and (sqrt(src3.c5) <> 13)
 POSTHOOK: type: QUERY
 STAGE DEPENDENCIES:
-  Stage-3 is a root stage
-  Stage-2 depends on stages: Stage-3
-  Stage-1 depends on stages: Stage-2
+  Stage-1 is a root stage
   Stage-0 depends on stages: Stage-1
 
 STAGE PLANS:
-  Stage: Stage-3
+  Stage: Stage-1
     Spark
+      Edges:
+        Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1), Map 4 (PARTITION-LEVEL SORT, 1)
+        Reducer 3 <- Map 5 (PARTITION-LEVEL SORT, 1), Reducer 2 (PARTITION-LEVEL SORT, 1)
 #### A masked pattern was here ####
       Vertices:
         Map 1 
@@ -52,21 +53,13 @@ STAGE PLANS:
                       Filter Operator
                         predicate: (_col0 < '400') (type: boolean)
                         Statistics: Num rows: 36 Data size: 382 Basic stats: COMPLETE Column stats: NONE
-                        Spark HashTable Sink Operator
-                          condition expressions:
-                            0 {_col1}
-                            1 {_col0} {_col1}
-                          keys:
-                            0 _col0 (type: string)
-                            1 _col0 (type: string)
-            Local Work:
-              Map Reduce Local Work
-
-  Stage: Stage-2
-    Spark
-#### A masked pattern was here ####
-      Vertices:
-        Map 2 
+                        Reduce Output Operator
+                          key expressions: _col0 (type: string)
+                          sort order: +
+                          Map-reduce partition columns: _col0 (type: string)
+                          Statistics: Num rows: 36 Data size: 382 Basic stats: COMPLETE Column stats: NONE
+                          value expressions: _col1 (type: string)
+        Map 4 
             Map Operator Tree:
                 TableScan
                   alias: src
@@ -81,37 +74,13 @@ STAGE PLANS:
                       Filter Operator
                         predicate: (_col0 < '400') (type: boolean)
                         Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE
-                        Map Join Operator
-                          condition map:
-                               Inner Join 0 to 1
-                          condition expressions:
-                            0 {_col0} {_col1}
-                            1 {_col0} {_col1}
-                          keys:
-                            0 _col0 (type: string)
-                            1 _col0 (type: string)
-                          outputColumnNames: _col0, _col1, _col2, _col3
-                          input vertices:
-                            0 Map 1
-                          Statistics: Num rows: 60 Data size: 642 Basic stats: COMPLETE Column stats: NONE
-                          Filter Operator
-                            predicate: _col1 is not null (type: boolean)
-                            Statistics: Num rows: 30 Data size: 321 Basic stats: COMPLETE Column stats: NONE
-                            Spark HashTable Sink Operator
-                              condition expressions:
-                                0 {_col0} {_col2} {_col3}
-                                1 {_col0}
-                              keys:
-                                0 _col1 (type: string)
-                                1 _col1 (type: string)
-            Local Work:
-              Map Reduce Local Work
-
-  Stage: Stage-1
-    Spark
-#### A masked pattern was here ####
-      Vertices:
-        Map 3 
+                        Reduce Output Operator
+                          key expressions: _col0 (type: string)
+                          sort order: +
+                          Map-reduce partition columns: _col0 (type: string)
+                          Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE
+                          value expressions: _col1 (type: string)
+        Map 5 
             Map Operator Tree:
                 TableScan
                   alias: src
@@ -126,35 +95,55 @@ STAGE PLANS:
                       Filter Operator
                         predicate: _col1 is not null (type: boolean)
                         Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE
-                        Map Join Operator
-                          condition map:
-                               Inner Join 0 to 1
-                          condition expressions:
-                            0 {_col0} {_col1} {_col2} {_col3}
-                            1 {_col0}
-                          keys:
-                            0 _col1 (type: string)
-                            1 _col1 (type: string)
-                          outputColumnNames: _col0, _col1, _col2, _col3, _col4
-                          input vertices:
-                            0 Map 2
-                          Statistics: Num rows: 137 Data size: 1460 Basic stats: COMPLETE Column stats: NONE
-                          Filter Operator
-                            predicate: (((((_col0 <> '311') and ((_col1 <> 'val_50') or (_col0 > '1'))) and ((_col2 <> '10') or (_col0 <> '10'))) and (_col2 <> '14')) and (sqrt(_col4) <> 13)) (type: boolean)
-                            Statistics: Num rows: 137 Data size: 1460 Basic stats: COMPLETE Column stats: NONE
-                            Select Operator
-                              expressions: _col0 (type: string), _col3 (type: string)
-                              outputColumnNames: _col0, _col1
-                              Statistics: Num rows: 137 Data size: 1460 Basic stats: COMPLETE Column stats: NONE
-                              File Output Operator
-                                compressed: false
-                                Statistics: Num rows: 137 Data size: 1460 Basic stats: COMPLETE Column stats: NONE
-                                table:
-                                    input format: org.apache.hadoop.mapred.TextInputFormat
-                                    output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
-                                    serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
-            Local Work:
-              Map Reduce Local Work
+                        Reduce Output Operator
+                          key expressions: _col1 (type: string)
+                          sort order: +
+                          Map-reduce partition columns: _col1 (type: string)
+                          Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE
+                          value expressions: _col0 (type: string)
+        Reducer 2 
+            Reduce Operator Tree:
+              Join Operator
+                condition map:
+                     Inner Join 0 to 1
+                condition expressions:
+                  0 {KEY.reducesinkkey0} {VALUE._col0}
+                  1 {KEY.reducesinkkey0} {VALUE._col0}
+                outputColumnNames: _col0, _col1, _col2, _col3
+                Statistics: Num rows: 60 Data size: 642 Basic stats: COMPLETE Column stats: NONE
+                Filter Operator
+                  predicate: _col1 is not null (type: boolean)
+                  Statistics: Num rows: 30 Data size: 321 Basic stats: COMPLETE Column stats: NONE
+                  Reduce Output Operator
+                    key expressions: _col1 (type: string)
+                    sort order: +
+                    Map-reduce partition columns: _col1 (type: string)
+                    Statistics: Num rows: 30 Data size: 321 Basic stats: COMPLETE Column stats: NONE
+                    value expressions: _col0 (type: string), _col2 (type: string), _col3 (type: string)
+        Reducer 3 
+            Reduce Operator Tree:
+              Join Operator
+                condition map:
+                     Inner Join 0 to 1
+                condition expressions:
+                  0 {VALUE._col0} {KEY.reducesinkkey0} {VALUE._col1} {VALUE._col2}
+                  1 {VALUE._col0}
+                outputColumnNames: _col0, _col1, _col2, _col3, _col4
+                Statistics: Num rows: 137 Data size: 1460 Basic stats: COMPLETE Column stats: NONE
+                Filter Operator
+                  predicate: (((((_col0 <> '311') and ((_col1 <> 'val_50') or (_col0 > '1'))) and ((_col2 <> '10') or (_col0 <> '10'))) and (_col2 <> '14')) and (sqrt(_col4) <> 13)) (type: boolean)
+                  Statistics: Num rows: 137 Data size: 1460 Basic stats: COMPLETE Column stats: NONE
+                  Select Operator
+                    expressions: _col0 (type: string), _col3 (type: string)
+                    outputColumnNames: _col0, _col1
+                    Statistics: Num rows: 137 Data size: 1460 Basic stats: COMPLETE Column stats: NONE
+                    File Output Operator
+                      compressed: false
+                      Statistics: Num rows: 137 Data size: 1460 Basic stats: COMPLETE Column stats: NONE
+                      table:
+                          input format: org.apache.hadoop.mapred.TextInputFormat
+                          output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+                          serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
 
   Stage: Stage-0
     Fetch Operator
@@ -1731,14 +1720,15 @@ ON src1.c2 = src3.c6
 WHERE src1.c1 <> '311' and (src1.c2 <> 'val_50' or src1.c1 > '1') and (src2.c3 <> '10' or src1.c1 <> '10') and (src2.c3 <> '14') and (sqrt(src3.c5) <> 13)
 POSTHOOK: type: QUERY
 STAGE DEPENDENCIES:
-  Stage-3 is a root stage
-  Stage-2 depends on stages: Stage-3
-  Stage-1 depends on stages: Stage-2
+  Stage-1 is a root stage
   Stage-0 depends on stages: Stage-1
 
 STAGE PLANS:
-  Stage: Stage-3
+  Stage: Stage-1
     Spark
+      Edges:
+        Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3), Map 4 (PARTITION-LEVEL SORT, 3)
+        Reducer 3 <- Map 5 (PARTITION-LEVEL SORT, 3), Reducer 2 (PARTITION-LEVEL SORT, 3)
 #### A masked pattern was here ####
       Vertices:
         Map 1 
@@ -1753,21 +1743,13 @@ STAGE PLANS:
                       expressions: key (type: string), value (type: string)
                       outputColumnNames: _col0, _col1
                       Statistics: Num rows: 110 Data size: 1168 Basic stats: COMPLETE Column stats: NONE
-                      Spark HashTable Sink Operator
-                        condition expressions:
-                          0 {_col1}
-                          1 {_col0} {_col1}
-                        keys:
-                          0 _col0 (type: string)
-                          1 _col0 (type: string)
-            Local Work:
-              Map Reduce Local Work
-
-  Stage: Stage-2
-    Spark
-#### A masked pattern was here ####
-      Vertices:
-        Map 2 
+                      Reduce Output Operator
+                        key expressions: _col0 (type: string)
+                        sort order: +
+                        Map-reduce partition columns: _col0 (type: string)
+                        Statistics: Num rows: 110 Data size: 1168 Basic stats: COMPLETE Column stats: NONE
+                        value expressions: _col1 (type: string)
+        Map 4 
             Map Operator Tree:
                 TableScan
                   alias: src
@@ -1779,34 +1761,13 @@ STAGE PLANS:
                       expressions: key (type: string), value (type: string)
                       outputColumnNames: _col0, _col1
                       Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE
-                      Map Join Operator
-                        condition map:
-                             Inner Join 0 to 1
-                        condition expressions:
-                          0 {_col0} {_col1}
-                          1 {_col0} {_col1}
-                        keys:
-                          0 _col0 (type: string)
-                          1 _col0 (type: string)
-                        outputColumnNames: _col0, _col1, _col2, _col3
-                        input vertices:
-                          0 Map 1
-                        Statistics: Num rows: 182 Data size: 1939 Basic stats: COMPLETE Column stats: NONE
-                        Spark HashTable Sink Operator
-                          condition expressions:
-                            0 {_col0} {_col2} {_col3}
-                            1 {_col0}
-                          keys:
-                            0 _col1 (type: string)
-                            1 _col1 (type: string)
-            Local Work:
-              Map Reduce Local Work
-
-  Stage: Stage-1
-    Spark
-#### A masked pattern was here ####
-      Vertices:
-        Map 3 
+                      Reduce Output Operator
+                        key expressions: _col0 (type: string)
+                        sort order: +
+                        Map-reduce partition columns: _col0 (type: string)
+                        Statistics: Num rows: 166 Data size: 1763 Basic stats: COMPLETE Column stats: NONE
+                        value expressions: _col1 (type: string)
+        Map 5 
             Map Operator Tree:
                 TableScan
                   alias: src
@@ -1818,35 +1779,52 @@ STAGE PLANS:
                       expressions: key (type: string), value (type: string)
                       outputColumnNames: _col0, _col1
                       Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
-                      Map Join Operator
-                        condition map:
-                             Inner Join 0 to 1
-                        condition expressions:
-                          0 {_col0} {_col1} {_col2} {_col3}
-                          1 {_col0}
-                        keys:
-                          0 _col1 (type: string)
-                          1 _col1 (type: string)
-                        outputColumnNames: _col0, _col1, _col2, _col3, _col4
-                        input vertices:
-                          0 Map 2
-                        Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE
-                        Filter Operator
-                          predicate: (((((_col0 <> '311') and ((_col1 <> 'val_50') or (_col0 > '1'))) and ((_col2 <> '10') or (_col0 <> '10'))) and (_col2 <> '14')) and (sqrt(_col4) <> 13)) (type: boolean)
-                          Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE
-                          Select Operator
-                            expressions: _col0 (type: string), _col3 (type: string)
-                            outputColumnNames: _col0, _col1
-                            Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE
-                            File Output Operator
-                              compressed: false
-                              Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE
-                              table:
-                                  input format: org.apache.hadoop.mapred.TextInputFormat
-                                  output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
-                                  serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
-            Local Work:
-              Map Reduce Local Work
+                      Reduce Output Operator
+                        key expressions: _col1 (type: string)
+                        sort order: +
+                        Map-reduce partition columns: _col1 (type: string)
+                        Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
+                        value expressions: _col0 (type: string)
+        Reducer 2 
+            Reduce Operator Tree:
+              Join Operator
+                condition map:
+                     Inner Join 0 to 1
+                condition expressions:
+                  0 {KEY.reducesinkkey0} {VALUE._col0}
+                  1 {KEY.reducesinkkey0} {VALUE._col0}
+                outputColumnNames: _col0, _col1, _col2, _col3
+                Statistics: Num rows: 182 Data size: 1939 Basic stats: COMPLETE Column stats: NONE
+                Reduce Output Operator
+                  key expressions: _col1 (type: string)
+                  sort order: +
+                  Map-reduce partition columns: _col1 (type: string)
+                  Statistics: Num rows: 182 Data size: 1939 Basic stats: COMPLETE Column stats: NONE
+                  value expressions: _col0 (type: string), _col2 (type: string), _col3 (type: string)
+        Reducer 3 
+            Reduce Operator Tree:
+              Join Operator
+                condition map:
+                     Inner Join 0 to 1
+                condition expressions:
+                  0 {VALUE._col0} {KEY.reducesinkkey0} {VALUE._col1} {VALUE._col2}
+                  1 {VALUE._col0}
+                outputColumnNames: _col0, _col1, _col2, _col3, _col4
+                Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE
+                Filter Operator
+                  predicate: (((((_col0 <> '311') and ((_col1 <> 'val_50') or (_col0 > '1'))) and ((_col2 <> '10') or (_col0 <> '10'))) and (_col2 <> '14')) and (sqrt(_col4) <> 13)) (type: boolean)
+                  Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE
+                  Select Operator
+                    expressions: _col0 (type: string), _col3 (type: string)
+                    outputColumnNames: _col0, _col1
+                    Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE
+                    File Output Operator
+                      compressed: false
+                      Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE
+                      table:
+                          input format: org.apache.hadoop.mapred.TextInputFormat
+                          output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+                          serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
 
   Stage: Stage-0
     Fetch Operator

Modified: hive/branches/spark/ql/src/test/results/clientpositive/spark/ppd_join3.q.out
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/results/clientpositive/spark/ppd_join3.q.out?rev=1642997&r1=1642996&r2=1642997&view=diff
==============================================================================
--- hive/branches/spark/ql/src/test/results/clientpositive/spark/ppd_join3.q.out (original)
+++ hive/branches/spark/ql/src/test/results/clientpositive/spark/ppd_join3.q.out Tue Dec  2 19:57:10 2014
@@ -27,120 +27,104 @@ ON src1.c1 = src3.c5
 WHERE src1.c1 > '0' and (src1.c2 <> 'val_500' or src1.c1 > '1') and (src2.c3 > '10' or src1.c1 <> '10') and (src2.c3 <> '4') and (src3.c5 <> '1')
 POSTHOOK: type: QUERY
 STAGE DEPENDENCIES:
-  Stage-2 is a root stage
-  Stage-1 depends on stages: Stage-2
+  Stage-1 is a root stage
   Stage-0 depends on stages: Stage-1
 
 STAGE PLANS:
-  Stage: Stage-2
+  Stage: Stage-1
     Spark
+      Edges:
+        Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1), Map 3 (PARTITION-LEVEL SORT, 1), Map 4 (PARTITION-LEVEL SORT, 1)
 #### A masked pattern was here ####
       Vertices:
-        Map 2 
+        Map 1 
             Map Operator Tree:
                 TableScan
                   alias: src
                   Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
                   Filter Operator
-                    predicate: (((((key <> '12') and (key < '400')) and (key <> '4')) and (key > '0')) and (key <> '1')) (type: boolean)
-                    Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE
+                    predicate: ((((((key <> '11') and (key < '400')) and (key > '0')) and ((value <> 'val_500') or (key > '1'))) and (key <> '4')) and (key <> '1')) (type: boolean)
+                    Statistics: Num rows: 73 Data size: 775 Basic stats: COMPLETE Column stats: NONE
                     Select Operator
                       expressions: key (type: string), value (type: string)
                       outputColumnNames: _col0, _col1
-                      Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE
+                      Statistics: Num rows: 73 Data size: 775 Basic stats: COMPLETE Column stats: NONE
                       Filter Operator
                         predicate: (_col0 < '400') (type: boolean)
-                        Statistics: Num rows: 18 Data size: 191 Basic stats: COMPLETE Column stats: NONE
-                        Spark HashTable Sink Operator
-                          condition expressions:
-                            0 {_col0} {_col1}
-                            1 {_col1}
-                            2 {_col0}
-                          keys:
-                            0 _col0 (type: string)
-                            1 _col0 (type: string)
-                            2 _col0 (type: string)
-            Local Work:
-              Map Reduce Local Work
+                        Statistics: Num rows: 24 Data size: 254 Basic stats: COMPLETE Column stats: NONE
+                        Reduce Output Operator
+                          key expressions: _col0 (type: string)
+                          sort order: +
+                          Map-reduce partition columns: _col0 (type: string)
+                          Statistics: Num rows: 24 Data size: 254 Basic stats: COMPLETE Column stats: NONE
+                          value expressions: _col1 (type: string)
         Map 3 
             Map Operator Tree:
                 TableScan
                   alias: src
                   Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
                   Filter Operator
-                    predicate: (((((key <> '13') and (key < '400')) and (key <> '1')) and (key > '0')) and (key <> '4')) (type: boolean)
+                    predicate: (((((key <> '12') and (key < '400')) and (key <> '4')) and (key > '0')) and (key <> '1')) (type: boolean)
                     Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE
                     Select Operator
-                      expressions: key (type: string)
-                      outputColumnNames: _col0
+                      expressions: key (type: string), value (type: string)
+                      outputColumnNames: _col0, _col1
                       Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE
                       Filter Operator
                         predicate: (_col0 < '400') (type: boolean)
                         Statistics: Num rows: 18 Data size: 191 Basic stats: COMPLETE Column stats: NONE
-                        Spark HashTable Sink Operator
-                          condition expressions:
-                            0 {_col0} {_col1}
-                            1 {_col0} {_col1}
-                            2 
-                          keys:
-                            0 _col0 (type: string)
-                            1 _col0 (type: string)
-                            2 _col0 (type: string)
-            Local Work:
-              Map Reduce Local Work
-
-  Stage: Stage-1
-    Spark
-#### A masked pattern was here ####
-      Vertices:
-        Map 1 
+                        Reduce Output Operator
+                          key expressions: _col0 (type: string)
+                          sort order: +
+                          Map-reduce partition columns: _col0 (type: string)
+                          Statistics: Num rows: 18 Data size: 191 Basic stats: COMPLETE Column stats: NONE
+                          value expressions: _col1 (type: string)
+        Map 4 
             Map Operator Tree:
                 TableScan
                   alias: src
                   Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
                   Filter Operator
-                    predicate: ((((((key <> '11') and (key < '400')) and (key > '0')) and ((value <> 'val_500') or (key > '1'))) and (key <> '4')) and (key <> '1')) (type: boolean)
-                    Statistics: Num rows: 73 Data size: 775 Basic stats: COMPLETE Column stats: NONE
+                    predicate: (((((key <> '13') and (key < '400')) and (key <> '1')) and (key > '0')) and (key <> '4')) (type: boolean)
+                    Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE
                     Select Operator
-                      expressions: key (type: string), value (type: string)
-                      outputColumnNames: _col0, _col1
-                      Statistics: Num rows: 73 Data size: 775 Basic stats: COMPLETE Column stats: NONE
+                      expressions: key (type: string)
+                      outputColumnNames: _col0
+                      Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE
                       Filter Operator
                         predicate: (_col0 < '400') (type: boolean)
-                        Statistics: Num rows: 24 Data size: 254 Basic stats: COMPLETE Column stats: NONE
-                        Map Join Operator
-                          condition map:
-                               Inner Join 0 to 1
-                               Inner Join 0 to 2
-                          condition expressions:
-                            0 {_col0} {_col1}
-                            1 {_col0} {_col1}
-                            2 {_col0}
-                          keys:
-                            0 _col0 (type: string)
-                            1 _col0 (type: string)
-                            2 _col0 (type: string)
-                          outputColumnNames: _col0, _col1, _col2, _col3, _col4
-                          input vertices:
-                            1 Map 2
-                            2 Map 3
-                          Statistics: Num rows: 52 Data size: 558 Basic stats: COMPLETE Column stats: NONE
-                          Filter Operator
-                            predicate: (((((_col0 > '0') and ((_col1 <> 'val_500') or (_col0 > '1'))) and ((_col2 > '10') or (_col0 <> '10'))) and (_col2 <> '4')) and (_col4 <> '1')) (type: boolean)
-                            Statistics: Num rows: 29 Data size: 311 Basic stats: COMPLETE Column stats: NONE
-                            Select Operator
-                              expressions: _col0 (type: string), _col3 (type: string)
-                              outputColumnNames: _col0, _col1
-                              Statistics: Num rows: 29 Data size: 311 Basic stats: COMPLETE Column stats: NONE
-                              File Output Operator
-                                compressed: false
-                                Statistics: Num rows: 29 Data size: 311 Basic stats: COMPLETE Column stats: NONE
-                                table:
-                                    input format: org.apache.hadoop.mapred.TextInputFormat
-                                    output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
-                                    serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
-            Local Work:
-              Map Reduce Local Work
+                        Statistics: Num rows: 18 Data size: 191 Basic stats: COMPLETE Column stats: NONE
+                        Reduce Output Operator
+                          key expressions: _col0 (type: string)
+                          sort order: +
+                          Map-reduce partition columns: _col0 (type: string)
+                          Statistics: Num rows: 18 Data size: 191 Basic stats: COMPLETE Column stats: NONE
+        Reducer 2 
+            Reduce Operator Tree:
+              Join Operator
+                condition map:
+                     Inner Join 0 to 1
+                     Inner Join 0 to 2
+                condition expressions:
+                  0 {KEY.reducesinkkey0} {VALUE._col0}
+                  1 {KEY.reducesinkkey0} {VALUE._col0}
+                  2 {KEY.reducesinkkey0}
+                outputColumnNames: _col0, _col1, _col2, _col3, _col4
+                Statistics: Num rows: 52 Data size: 558 Basic stats: COMPLETE Column stats: NONE
+                Filter Operator
+                  predicate: (((((_col0 > '0') and ((_col1 <> 'val_500') or (_col0 > '1'))) and ((_col2 > '10') or (_col0 <> '10'))) and (_col2 <> '4')) and (_col4 <> '1')) (type: boolean)
+                  Statistics: Num rows: 29 Data size: 311 Basic stats: COMPLETE Column stats: NONE
+                  Select Operator
+                    expressions: _col0 (type: string), _col3 (type: string)
+                    outputColumnNames: _col0, _col1
+                    Statistics: Num rows: 29 Data size: 311 Basic stats: COMPLETE Column stats: NONE
+                    File Output Operator
+                      compressed: false
+                      Statistics: Num rows: 29 Data size: 311 Basic stats: COMPLETE Column stats: NONE
+                      table:
+                          input format: org.apache.hadoop.mapred.TextInputFormat
+                          output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+                          serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
 
   Stage: Stage-0
     Fetch Operator
@@ -1774,111 +1758,95 @@ ON src1.c1 = src3.c5
 WHERE src1.c1 > '0' and (src1.c2 <> 'val_500' or src1.c1 > '1') and (src2.c3 > '10' or src1.c1 <> '10') and (src2.c3 <> '4') and (src3.c5 <> '1')
 POSTHOOK: type: QUERY
 STAGE DEPENDENCIES:
-  Stage-2 is a root stage
-  Stage-1 depends on stages: Stage-2
+  Stage-1 is a root stage
   Stage-0 depends on stages: Stage-1
 
 STAGE PLANS:
-  Stage: Stage-2
+  Stage: Stage-1
     Spark
+      Edges:
+        Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3), Map 3 (PARTITION-LEVEL SORT, 3), Map 4 (PARTITION-LEVEL SORT, 3)
 #### A masked pattern was here ####
       Vertices:
-        Map 2 
+        Map 1 
             Map Operator Tree:
                 TableScan
                   alias: src
                   Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
                   Filter Operator
-                    predicate: (((((key <> '12') and (key < '400')) and (key <> '4')) and (key > '0')) and (key <> '1')) (type: boolean)
-                    Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE
+                    predicate: ((((((key <> '11') and (key < '400')) and (key > '0')) and ((value <> 'val_500') or (key > '1'))) and (key <> '4')) and (key <> '1')) (type: boolean)
+                    Statistics: Num rows: 73 Data size: 775 Basic stats: COMPLETE Column stats: NONE
                     Select Operator
                       expressions: key (type: string), value (type: string)
                       outputColumnNames: _col0, _col1
-                      Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE
-                      Spark HashTable Sink Operator
-                        condition expressions:
-                          0 {_col0} {_col1}
-                          1 {_col1}
-                          2 {_col0}
-                        keys:
-                          0 _col0 (type: string)
-                          1 _col0 (type: string)
-                          2 _col0 (type: string)
-            Local Work:
-              Map Reduce Local Work
+                      Statistics: Num rows: 73 Data size: 775 Basic stats: COMPLETE Column stats: NONE
+                      Reduce Output Operator
+                        key expressions: _col0 (type: string)
+                        sort order: +
+                        Map-reduce partition columns: _col0 (type: string)
+                        Statistics: Num rows: 73 Data size: 775 Basic stats: COMPLETE Column stats: NONE
+                        value expressions: _col1 (type: string)
         Map 3 
             Map Operator Tree:
                 TableScan
                   alias: src
                   Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
                   Filter Operator
-                    predicate: (((((key <> '13') and (key < '400')) and (key <> '1')) and (key > '0')) and (key <> '4')) (type: boolean)
+                    predicate: (((((key <> '12') and (key < '400')) and (key <> '4')) and (key > '0')) and (key <> '1')) (type: boolean)
                     Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE
                     Select Operator
-                      expressions: key (type: string)
-                      outputColumnNames: _col0
+                      expressions: key (type: string), value (type: string)
+                      outputColumnNames: _col0, _col1
                       Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE
-                      Spark HashTable Sink Operator
-                        condition expressions:
-                          0 {_col0} {_col1}
-                          1 {_col0} {_col1}
-                          2 
-                        keys:
-                          0 _col0 (type: string)
-                          1 _col0 (type: string)
-                          2 _col0 (type: string)
-            Local Work:
-              Map Reduce Local Work
-
-  Stage: Stage-1
-    Spark
-#### A masked pattern was here ####
-      Vertices:
-        Map 1 
+                      Reduce Output Operator
+                        key expressions: _col0 (type: string)
+                        sort order: +
+                        Map-reduce partition columns: _col0 (type: string)
+                        Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE
+                        value expressions: _col1 (type: string)
+        Map 4 
             Map Operator Tree:
                 TableScan
                   alias: src
                   Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
                   Filter Operator
-                    predicate: ((((((key <> '11') and (key < '400')) and (key > '0')) and ((value <> 'val_500') or (key > '1'))) and (key <> '4')) and (key <> '1')) (type: boolean)
-                    Statistics: Num rows: 73 Data size: 775 Basic stats: COMPLETE Column stats: NONE
+                    predicate: (((((key <> '13') and (key < '400')) and (key <> '1')) and (key > '0')) and (key <> '4')) (type: boolean)
+                    Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE
                     Select Operator
-                      expressions: key (type: string), value (type: string)
-                      outputColumnNames: _col0, _col1
-                      Statistics: Num rows: 73 Data size: 775 Basic stats: COMPLETE Column stats: NONE
-                      Map Join Operator
-                        condition map:
-                             Inner Join 0 to 1
-                             Inner Join 0 to 2
-                        condition expressions:
-                          0 {_col0} {_col1}
-                          1 {_col0} {_col1}
-                          2 {_col0}
-                        keys:
-                          0 _col0 (type: string)
-                          1 _col0 (type: string)
-                          2 _col0 (type: string)
-                        outputColumnNames: _col0, _col1, _col2, _col3, _col4
-                        input vertices:
-                          1 Map 2
-                          2 Map 3
-                        Statistics: Num rows: 160 Data size: 1705 Basic stats: COMPLETE Column stats: NONE
-                        Filter Operator
-                          predicate: (((((_col0 > '0') and ((_col1 <> 'val_500') or (_col0 > '1'))) and ((_col2 > '10') or (_col0 <> '10'))) and (_col2 <> '4')) and (_col4 <> '1')) (type: boolean)
-                          Statistics: Num rows: 93 Data size: 991 Basic stats: COMPLETE Column stats: NONE
-                          Select Operator
-                            expressions: _col0 (type: string), _col3 (type: string)
-                            outputColumnNames: _col0, _col1
-                            Statistics: Num rows: 93 Data size: 991 Basic stats: COMPLETE Column stats: NONE
-                            File Output Operator
-                              compressed: false
-                              Statistics: Num rows: 93 Data size: 991 Basic stats: COMPLETE Column stats: NONE
-                              table:
-                                  input format: org.apache.hadoop.mapred.TextInputFormat
-                                  output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
-                                  serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
-            Local Work:
-              Map Reduce Local Work
+                      expressions: key (type: string)
+                      outputColumnNames: _col0
+                      Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE
+                      Reduce Output Operator
+                        key expressions: _col0 (type: string)
+                        sort order: +
+                        Map-reduce partition columns: _col0 (type: string)
+                        Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE
+        Reducer 2 
+            Reduce Operator Tree:
+              Join Operator
+                condition map:
+                     Inner Join 0 to 1
+                     Inner Join 0 to 2
+                condition expressions:
+                  0 {KEY.reducesinkkey0} {VALUE._col0}
+                  1 {KEY.reducesinkkey0} {VALUE._col0}
+                  2 {KEY.reducesinkkey0}
+                outputColumnNames: _col0, _col1, _col2, _col3, _col4
+                Statistics: Num rows: 160 Data size: 1705 Basic stats: COMPLETE Column stats: NONE
+                Filter Operator
+                  predicate: (((((_col0 > '0') and ((_col1 <> 'val_500') or (_col0 > '1'))) and ((_col2 > '10') or (_col0 <> '10'))) and (_col2 <> '4')) and (_col4 <> '1')) (type: boolean)
+                  Statistics: Num rows: 93 Data size: 991 Basic stats: COMPLETE Column stats: NONE
+                  Select Operator
+                    expressions: _col0 (type: string), _col3 (type: string)
+                    outputColumnNames: _col0, _col1
+                    Statistics: Num rows: 93 Data size: 991 Basic stats: COMPLETE Column stats: NONE
+                    File Output Operator
+                      compressed: false
+                      Statistics: Num rows: 93 Data size: 991 Basic stats: COMPLETE Column stats: NONE
+                      table:
+                          input format: org.apache.hadoop.mapred.TextInputFormat
+                          output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+                          serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
 
   Stage: Stage-0
     Fetch Operator

Modified: hive/branches/spark/ql/src/test/results/clientpositive/spark/ppd_join5.q.out
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/results/clientpositive/spark/ppd_join5.q.out?rev=1642997&r1=1642996&r2=1642997&view=diff
==============================================================================
--- hive/branches/spark/ql/src/test/results/clientpositive/spark/ppd_join5.q.out (original)
+++ hive/branches/spark/ql/src/test/results/clientpositive/spark/ppd_join5.q.out Tue Dec  2 19:57:10 2014
@@ -43,16 +43,31 @@ select a.*,b.d d1,c.d d2 from
        join t2 c on (a.id2 = b.id) where b.d <= 1 and c.d <= 1
 POSTHOOK: type: QUERY
 STAGE DEPENDENCIES:
-  Stage-2 is a root stage
-  Stage-1 depends on stages: Stage-2
+  Stage-1 is a root stage
   Stage-0 depends on stages: Stage-1
 
 STAGE PLANS:
-  Stage: Stage-2
+  Stage: Stage-1
     Spark
+      Edges:
+        Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3), Map 4 (PARTITION-LEVEL SORT, 3)
+        Reducer 3 <- Map 5 (PARTITION-LEVEL SORT, 1), Reducer 2 (PARTITION-LEVEL SORT, 1)
 #### A masked pattern was here ####
       Vertices:
-        Map 2 
+        Map 1 
+            Map Operator Tree:
+                TableScan
+                  alias: a
+                  Statistics: Num rows: 0 Data size: 4 Basic stats: PARTIAL Column stats: NONE
+                  Filter Operator
+                    predicate: (id1 is not null and id2 is not null) (type: boolean)
+                    Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
+                    Reduce Output Operator
+                      key expressions: id1 (type: string), id2 (type: string)
+                      sort order: ++
+                      Map-reduce partition columns: id1 (type: string), id2 (type: string)
+                      Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
+        Map 4 
             Map Operator Tree:
                 TableScan
                   alias: b
@@ -60,16 +75,13 @@ STAGE PLANS:
                   Filter Operator
                     predicate: (id is not null and (d <= 1)) (type: boolean)
                     Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
-                    Spark HashTable Sink Operator
-                      condition expressions:
-                        0 {id1} {id2}
-                        1 {d}
-                      keys:
-                        0 id1 (type: string), id2 (type: string)
-                        1 id (type: string), id (type: string)
-            Local Work:
-              Map Reduce Local Work
-        Map 3 
+                    Reduce Output Operator
+                      key expressions: id (type: string), id (type: string)
+                      sort order: ++
+                      Map-reduce partition columns: id (type: string), id (type: string)
+                      Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
+                      value expressions: d (type: int)
+        Map 5 
             Map Operator Tree:
                 TableScan
                   alias: c
@@ -77,67 +89,45 @@ STAGE PLANS:
                   Filter Operator
                     predicate: (d <= 1) (type: boolean)
                     Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
-                    Spark HashTable Sink Operator
-                      condition expressions:
-                        0 {_col0} {_col1} {_col6}
-                        1 {d}
-                      keys:
-                        0 
-                        1 
-            Local Work:
-              Map Reduce Local Work
-
-  Stage: Stage-1
-    Spark
-#### A masked pattern was here ####
-      Vertices:
-        Map 1 
-            Map Operator Tree:
-                TableScan
-                  alias: a
-                  Statistics: Num rows: 0 Data size: 4 Basic stats: PARTIAL Column stats: NONE
-                  Filter Operator
-                    predicate: (id1 is not null and id2 is not null) (type: boolean)
-                    Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
-                    Map Join Operator
-                      condition map:
-                           Inner Join 0 to 1
-                      condition expressions:
-                        0 {id1} {id2}
-                        1 {d}
-                      keys:
-                        0 id1 (type: string), id2 (type: string)
-                        1 id (type: string), id (type: string)
-                      outputColumnNames: _col0, _col1, _col6
-                      input vertices:
-                        1 Map 2
-                      Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
-                      Map Join Operator
-                        condition map:
-                             Inner Join 0 to 1
-                        condition expressions:
-                          0 {_col0} {_col1} {_col6}
-                          1 {d}
-                        keys:
-                          0 
-                          1 
-                        outputColumnNames: _col0, _col1, _col6, _col11
-                        input vertices:
-                          1 Map 3
-                        Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
-                        Select Operator
-                          expressions: _col0 (type: string), _col1 (type: string), _col6 (type: int), _col11 (type: int)
-                          outputColumnNames: _col0, _col1, _col2, _col3
-                          Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
-                          File Output Operator
-                            compressed: false
-                            Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
-                            table:
-                                input format: org.apache.hadoop.mapred.TextInputFormat
-                                output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
-                                serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
-            Local Work:
-              Map Reduce Local Work
+                    Reduce Output Operator
+                      sort order: 
+                      Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
+                      value expressions: d (type: int)
+        Reducer 2 
+            Reduce Operator Tree:
+              Join Operator
+                condition map:
+                     Inner Join 0 to 1
+                condition expressions:
+                  0 {KEY.reducesinkkey0} {KEY.reducesinkkey1}
+                  1 {VALUE._col0}
+                outputColumnNames: _col0, _col1, _col6
+                Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
+                Reduce Output Operator
+                  sort order: 
+                  Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
+                  value expressions: _col0 (type: string), _col1 (type: string), _col6 (type: int)
+        Reducer 3 
+            Reduce Operator Tree:
+              Join Operator
+                condition map:
+                     Inner Join 0 to 1
+                condition expressions:
+                  0 {VALUE._col0} {VALUE._col1} {VALUE._col6}
+                  1 {VALUE._col1}
+                outputColumnNames: _col0, _col1, _col6, _col11
+                Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
+                Select Operator
+                  expressions: _col0 (type: string), _col1 (type: string), _col6 (type: int), _col11 (type: int)
+                  outputColumnNames: _col0, _col1, _col2, _col3
+                  Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
+                  File Output Operator
+                    compressed: false
+                    Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
+                    table:
+                        input format: org.apache.hadoop.mapred.TextInputFormat
+                        output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+                        serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
 
   Stage: Stage-0
     Fetch Operator
@@ -160,16 +150,31 @@ select a.*,b.d d1,c.d d2 from
 ) z where d1 > 1 or d2 > 1
 POSTHOOK: type: QUERY
 STAGE DEPENDENCIES:
-  Stage-2 is a root stage
-  Stage-1 depends on stages: Stage-2
+  Stage-1 is a root stage
   Stage-0 depends on stages: Stage-1
 
 STAGE PLANS:
-  Stage: Stage-2
+  Stage: Stage-1
     Spark
+      Edges:
+        Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 3), Map 4 (PARTITION-LEVEL SORT, 3)
+        Reducer 3 <- Map 5 (PARTITION-LEVEL SORT, 1), Reducer 2 (PARTITION-LEVEL SORT, 1)
 #### A masked pattern was here ####
       Vertices:
-        Map 2 
+        Map 1 
+            Map Operator Tree:
+                TableScan
+                  alias: a
+                  Statistics: Num rows: 0 Data size: 4 Basic stats: PARTIAL Column stats: NONE
+                  Filter Operator
+                    predicate: (id1 is not null and id2 is not null) (type: boolean)
+                    Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
+                    Reduce Output Operator
+                      key expressions: id1 (type: string), id2 (type: string)
+                      sort order: ++
+                      Map-reduce partition columns: id1 (type: string), id2 (type: string)
+                      Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
+        Map 4 
             Map Operator Tree:
                 TableScan
                   alias: b
@@ -177,16 +182,13 @@ STAGE PLANS:
                   Filter Operator
                     predicate: (id is not null and (d <= 1)) (type: boolean)
                     Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
-                    Spark HashTable Sink Operator
-                      condition expressions:
-                        0 {id1} {id2}
-                        1 {d}
-                      keys:
-                        0 id1 (type: string), id2 (type: string)
-                        1 id (type: string), id (type: string)
-            Local Work:
-              Map Reduce Local Work
-        Map 3 
+                    Reduce Output Operator
+                      key expressions: id (type: string), id (type: string)
+                      sort order: ++
+                      Map-reduce partition columns: id (type: string), id (type: string)
+                      Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
+                      value expressions: d (type: int)
+        Map 5 
             Map Operator Tree:
                 TableScan
                   alias: c
@@ -194,70 +196,48 @@ STAGE PLANS:
                   Filter Operator
                     predicate: (d <= 1) (type: boolean)
                     Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
-                    Spark HashTable Sink Operator
-                      condition expressions:
-                        0 {_col0} {_col1} {_col6}
-                        1 {d}
-                      keys:
-                        0 
-                        1 
-            Local Work:
-              Map Reduce Local Work
-
-  Stage: Stage-1
-    Spark
-#### A masked pattern was here ####
-      Vertices:
-        Map 1 
-            Map Operator Tree:
-                TableScan
-                  alias: a
-                  Statistics: Num rows: 0 Data size: 4 Basic stats: PARTIAL Column stats: NONE
-                  Filter Operator
-                    predicate: (id1 is not null and id2 is not null) (type: boolean)
+                    Reduce Output Operator
+                      sort order: 
+                      Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
+                      value expressions: d (type: int)
+        Reducer 2 
+            Reduce Operator Tree:
+              Join Operator
+                condition map:
+                     Inner Join 0 to 1
+                condition expressions:
+                  0 {KEY.reducesinkkey0} {KEY.reducesinkkey1}
+                  1 {VALUE._col0}
+                outputColumnNames: _col0, _col1, _col6
+                Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
+                Reduce Output Operator
+                  sort order: 
+                  Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
+                  value expressions: _col0 (type: string), _col1 (type: string), _col6 (type: int)
+        Reducer 3 
+            Reduce Operator Tree:
+              Join Operator
+                condition map:
+                     Inner Join 0 to 1
+                condition expressions:
+                  0 {VALUE._col0} {VALUE._col1} {VALUE._col6}
+                  1 {VALUE._col1}
+                outputColumnNames: _col0, _col1, _col6, _col11
+                Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
+                Filter Operator
+                  predicate: ((_col6 > 1) or (_col11 > 1)) (type: boolean)
+                  Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
+                  Select Operator
+                    expressions: _col0 (type: string), _col1 (type: string), _col6 (type: int), _col11 (type: int)
+                    outputColumnNames: _col0, _col1, _col2, _col3
                     Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
-                    Map Join Operator
-                      condition map:
-                           Inner Join 0 to 1
-                      condition expressions:
-                        0 {id1} {id2}
-                        1 {d}
-                      keys:
-                        0 id1 (type: string), id2 (type: string)
-                        1 id (type: string), id (type: string)
-                      outputColumnNames: _col0, _col1, _col6
-                      input vertices:
-                        1 Map 2
-                      Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
-                      Map Join Operator
-                        condition map:
-                             Inner Join 0 to 1
-                        condition expressions:
-                          0 {_col0} {_col1} {_col6}
-                          1 {d}
-                        keys:
-                          0 
-                          1 
-                        outputColumnNames: _col0, _col1, _col6, _col11
-                        input vertices:
-                          1 Map 3
-                        Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
-                        Filter Operator
-                          predicate: ((_col6 > 1) or (_col11 > 1)) (type: boolean)
-                          Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
-                          Select Operator
-                            expressions: _col0 (type: string), _col1 (type: string), _col6 (type: int), _col11 (type: int)
-                            outputColumnNames: _col0, _col1, _col2, _col3
-                            Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
-                            File Output Operator
-                              compressed: false
-                              Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
-                              table:
-                                  input format: org.apache.hadoop.mapred.TextInputFormat
-                                  output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
-                                  serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
-            Local Work:
-              Map Reduce Local Work
+                    File Output Operator
+                      compressed: false
+                      Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
+                      table:
+                          input format: org.apache.hadoop.mapred.TextInputFormat
+                          output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+                          serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
 
   Stage: Stage-0
     Fetch Operator



Mime
View raw message