hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From xu...@apache.org
Subject svn commit: r1646523 [5/6] - in /hive/branches/spark/ql/src: java/org/apache/hadoop/hive/ql/optimizer/ java/org/apache/hadoop/hive/ql/optimizer/spark/ java/org/apache/hadoop/hive/ql/parse/spark/ test/results/clientpositive/spark/
Date Thu, 18 Dec 2014 19:58:10 GMT
Modified: hive/branches/spark/ql/src/test/results/clientpositive/spark/bucketsortoptimize_insert_2.q.out
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/results/clientpositive/spark/bucketsortoptimize_insert_2.q.out?rev=1646523&r1=1646522&r2=1646523&view=diff
==============================================================================
--- hive/branches/spark/ql/src/test/results/clientpositive/spark/bucketsortoptimize_insert_2.q.out (original)
+++ hive/branches/spark/ql/src/test/results/clientpositive/spark/bucketsortoptimize_insert_2.q.out Thu Dec 18 19:58:10 2014
@@ -95,12 +95,13 @@ FROM test_table1 a JOIN test_table2 b
 ON a.key = b.key WHERE a.ds = '1' and b.ds = '1'
 POSTHOOK: type: QUERY
 STAGE DEPENDENCIES:
-  Stage-1 is a root stage
+  Stage-3 is a root stage
+  Stage-1 depends on stages: Stage-3
   Stage-0 depends on stages: Stage-1
   Stage-2 depends on stages: Stage-0
 
 STAGE PLANS:
-  Stage: Stage-1
+  Stage: Stage-3
     Spark
 #### A masked pattern was here ####
       Vertices:
@@ -112,7 +113,31 @@ STAGE PLANS:
                   Filter Operator
                     predicate: key is not null (type: boolean)
                     Statistics: Num rows: 5 Data size: 35 Basic stats: COMPLETE Column stats: NONE
-                    Sorted Merge Bucket Map Join Operator
+                    Spark HashTable Sink Operator
+                      condition expressions:
+                        0 {value}
+                        1 {value}
+                      keys:
+                        0 key (type: int)
+                        1 key (type: int)
+            Local Work:
+              Map Reduce Local Work
+
+  Stage: Stage-1
+    Spark
+      Edges:
+        Reducer 3 <- Map 2 (PARTITION-LEVEL SORT, 1)
+#### A masked pattern was here ####
+      Vertices:
+        Map 2 
+            Map Operator Tree:
+                TableScan
+                  alias: b
+                  Statistics: Num rows: 84 Data size: 736 Basic stats: COMPLETE Column stats: NONE
+                  Filter Operator
+                    predicate: key is not null (type: boolean)
+                    Statistics: Num rows: 42 Data size: 368 Basic stats: COMPLETE Column stats: NONE
+                    Map Join Operator
                       condition map:
                            Inner Join 0 to 1
                       condition expressions:
@@ -122,16 +147,33 @@ STAGE PLANS:
                         0 key (type: int)
                         1 key (type: int)
                       outputColumnNames: _col0, _col1, _col7
+                      input vertices:
+                        0 Map 1
+                      Statistics: Num rows: 46 Data size: 404 Basic stats: COMPLETE Column stats: NONE
                       Select Operator
                         expressions: _col0 (type: int), concat(_col1, _col7) (type: string)
                         outputColumnNames: _col0, _col1
-                        File Output Operator
-                          compressed: false
-                          table:
-                              input format: org.apache.hadoop.mapred.TextInputFormat
-                              output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
-                              serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
-                              name: default.test_table3
+                        Statistics: Num rows: 46 Data size: 404 Basic stats: COMPLETE Column stats: NONE
+                        Reduce Output Operator
+                          key expressions: _col0 (type: int)
+                          sort order: +
+                          Map-reduce partition columns: _col0 (type: int)
+                          Statistics: Num rows: 46 Data size: 404 Basic stats: COMPLETE Column stats: NONE
+                          value expressions: _col0 (type: int), _col1 (type: string)
+            Local Work:
+              Map Reduce Local Work
+        Reducer 3 
+            Reduce Operator Tree:
+              Extract
+                Statistics: Num rows: 46 Data size: 404 Basic stats: COMPLETE Column stats: NONE
+                File Output Operator
+                  compressed: false
+                  Statistics: Num rows: 46 Data size: 404 Basic stats: COMPLETE Column stats: NONE
+                  table:
+                      input format: org.apache.hadoop.mapred.TextInputFormat
+                      output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+                      serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+                      name: default.test_table3
 
   Stage: Stage-0
     Move Operator
@@ -229,15 +271,14 @@ FROM test_table1 a JOIN test_table2 b
 ON a.key = b.key WHERE a.ds is not null and b.ds = '1'
 POSTHOOK: type: QUERY
 STAGE DEPENDENCIES:
-  Stage-1 is a root stage
+  Stage-3 is a root stage
+  Stage-1 depends on stages: Stage-3
   Stage-0 depends on stages: Stage-1
   Stage-2 depends on stages: Stage-0
 
 STAGE PLANS:
-  Stage: Stage-1
+  Stage: Stage-3
     Spark
-      Edges:
-        Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1)
 #### A masked pattern was here ####
       Vertices:
         Map 1 
@@ -248,7 +289,31 @@ STAGE PLANS:
                   Filter Operator
                     predicate: key is not null (type: boolean)
                     Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE
-                    Sorted Merge Bucket Map Join Operator
+                    Spark HashTable Sink Operator
+                      condition expressions:
+                        0 {value}
+                        1 {value}
+                      keys:
+                        0 key (type: int)
+                        1 key (type: int)
+            Local Work:
+              Map Reduce Local Work
+
+  Stage: Stage-1
+    Spark
+      Edges:
+        Reducer 3 <- Map 2 (PARTITION-LEVEL SORT, 1)
+#### A masked pattern was here ####
+      Vertices:
+        Map 2 
+            Map Operator Tree:
+                TableScan
+                  alias: b
+                  Statistics: Num rows: 84 Data size: 736 Basic stats: COMPLETE Column stats: NONE
+                  Filter Operator
+                    predicate: key is not null (type: boolean)
+                    Statistics: Num rows: 42 Data size: 368 Basic stats: COMPLETE Column stats: NONE
+                    Map Join Operator
                       condition map:
                            Inner Join 0 to 1
                       condition expressions:
@@ -258,19 +323,28 @@ STAGE PLANS:
                         0 key (type: int)
                         1 key (type: int)
                       outputColumnNames: _col0, _col1, _col7
+                      input vertices:
+                        0 Map 1
+                      Statistics: Num rows: 46 Data size: 404 Basic stats: COMPLETE Column stats: NONE
                       Select Operator
                         expressions: _col0 (type: int), concat(_col1, _col7) (type: string)
                         outputColumnNames: _col0, _col1
+                        Statistics: Num rows: 46 Data size: 404 Basic stats: COMPLETE Column stats: NONE
                         Reduce Output Operator
                           key expressions: _col0 (type: int)
                           sort order: +
                           Map-reduce partition columns: _col0 (type: int)
+                          Statistics: Num rows: 46 Data size: 404 Basic stats: COMPLETE Column stats: NONE
                           value expressions: _col0 (type: int), _col1 (type: string)
-        Reducer 2 
+            Local Work:
+              Map Reduce Local Work
+        Reducer 3 
             Reduce Operator Tree:
               Extract
+                Statistics: Num rows: 46 Data size: 404 Basic stats: COMPLETE Column stats: NONE
                 File Output Operator
                   compressed: false
+                  Statistics: Num rows: 46 Data size: 404 Basic stats: COMPLETE Column stats: NONE
                   table:
                       input format: org.apache.hadoop.mapred.TextInputFormat
                       output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
@@ -397,12 +471,13 @@ FROM test_table1 a JOIN test_table2 b
 ON a.key = b.key WHERE a.ds = '1' and b.ds is not null
 POSTHOOK: type: QUERY
 STAGE DEPENDENCIES:
-  Stage-1 is a root stage
+  Stage-3 is a root stage
+  Stage-1 depends on stages: Stage-3
   Stage-0 depends on stages: Stage-1
   Stage-2 depends on stages: Stage-0
 
 STAGE PLANS:
-  Stage: Stage-1
+  Stage: Stage-3
     Spark
 #### A masked pattern was here ####
       Vertices:
@@ -414,7 +489,31 @@ STAGE PLANS:
                   Filter Operator
                     predicate: key is not null (type: boolean)
                     Statistics: Num rows: 5 Data size: 35 Basic stats: COMPLETE Column stats: NONE
-                    Sorted Merge Bucket Map Join Operator
+                    Spark HashTable Sink Operator
+                      condition expressions:
+                        0 {value}
+                        1 {value}
+                      keys:
+                        0 key (type: int)
+                        1 key (type: int)
+            Local Work:
+              Map Reduce Local Work
+
+  Stage: Stage-1
+    Spark
+      Edges:
+        Reducer 3 <- Map 2 (PARTITION-LEVEL SORT, 1)
+#### A masked pattern was here ####
+      Vertices:
+        Map 2 
+            Map Operator Tree:
+                TableScan
+                  alias: b
+                  Statistics: Num rows: 168 Data size: 1472 Basic stats: COMPLETE Column stats: NONE
+                  Filter Operator
+                    predicate: key is not null (type: boolean)
+                    Statistics: Num rows: 84 Data size: 736 Basic stats: COMPLETE Column stats: NONE
+                    Map Join Operator
                       condition map:
                            Inner Join 0 to 1
                       condition expressions:
@@ -424,16 +523,33 @@ STAGE PLANS:
                         0 key (type: int)
                         1 key (type: int)
                       outputColumnNames: _col0, _col1, _col7
+                      input vertices:
+                        0 Map 1
+                      Statistics: Num rows: 92 Data size: 809 Basic stats: COMPLETE Column stats: NONE
                       Select Operator
                         expressions: _col0 (type: int), concat(_col1, _col7) (type: string)
                         outputColumnNames: _col0, _col1
-                        File Output Operator
-                          compressed: false
-                          table:
-                              input format: org.apache.hadoop.mapred.TextInputFormat
-                              output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
-                              serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
-                              name: default.test_table3
+                        Statistics: Num rows: 92 Data size: 809 Basic stats: COMPLETE Column stats: NONE
+                        Reduce Output Operator
+                          key expressions: _col0 (type: int)
+                          sort order: +
+                          Map-reduce partition columns: _col0 (type: int)
+                          Statistics: Num rows: 92 Data size: 809 Basic stats: COMPLETE Column stats: NONE
+                          value expressions: _col0 (type: int), _col1 (type: string)
+            Local Work:
+              Map Reduce Local Work
+        Reducer 3 
+            Reduce Operator Tree:
+              Extract
+                Statistics: Num rows: 92 Data size: 809 Basic stats: COMPLETE Column stats: NONE
+                File Output Operator
+                  compressed: false
+                  Statistics: Num rows: 92 Data size: 809 Basic stats: COMPLETE Column stats: NONE
+                  table:
+                      input format: org.apache.hadoop.mapred.TextInputFormat
+                      output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+                      serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+                      name: default.test_table3
 
   Stage: Stage-0
     Move Operator
@@ -559,12 +675,13 @@ JOIN
 ON a.key = b.key
 POSTHOOK: type: QUERY
 STAGE DEPENDENCIES:
-  Stage-1 is a root stage
+  Stage-3 is a root stage
+  Stage-1 depends on stages: Stage-3
   Stage-0 depends on stages: Stage-1
   Stage-2 depends on stages: Stage-0
 
 STAGE PLANS:
-  Stage: Stage-1
+  Stage: Stage-3
     Spark
 #### A masked pattern was here ####
       Vertices:
@@ -580,7 +697,35 @@ STAGE PLANS:
                       expressions: key (type: int), value (type: string)
                       outputColumnNames: _col0, _col1
                       Statistics: Num rows: 5 Data size: 35 Basic stats: COMPLETE Column stats: NONE
-                      Sorted Merge Bucket Map Join Operator
+                      Spark HashTable Sink Operator
+                        condition expressions:
+                          0 {_col1}
+                          1 {_col1}
+                        keys:
+                          0 _col0 (type: int)
+                          1 _col0 (type: int)
+            Local Work:
+              Map Reduce Local Work
+
+  Stage: Stage-1
+    Spark
+      Edges:
+        Reducer 3 <- Map 2 (PARTITION-LEVEL SORT, 1)
+#### A masked pattern was here ####
+      Vertices:
+        Map 2 
+            Map Operator Tree:
+                TableScan
+                  alias: test_table2
+                  Statistics: Num rows: 84 Data size: 736 Basic stats: COMPLETE Column stats: NONE
+                  Filter Operator
+                    predicate: key is not null (type: boolean)
+                    Statistics: Num rows: 42 Data size: 368 Basic stats: COMPLETE Column stats: NONE
+                    Select Operator
+                      expressions: key (type: int), value (type: string)
+                      outputColumnNames: _col0, _col1
+                      Statistics: Num rows: 42 Data size: 368 Basic stats: COMPLETE Column stats: NONE
+                      Map Join Operator
                         condition map:
                              Inner Join 0 to 1
                         condition expressions:
@@ -590,16 +735,33 @@ STAGE PLANS:
                           0 _col0 (type: int)
                           1 _col0 (type: int)
                         outputColumnNames: _col0, _col1, _col3
+                        input vertices:
+                          0 Map 1
+                        Statistics: Num rows: 46 Data size: 404 Basic stats: COMPLETE Column stats: NONE
                         Select Operator
                           expressions: _col0 (type: int), concat(_col1, _col3) (type: string)
                           outputColumnNames: _col0, _col1
-                          File Output Operator
-                            compressed: false
-                            table:
-                                input format: org.apache.hadoop.mapred.TextInputFormat
-                                output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
-                                serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
-                                name: default.test_table3
+                          Statistics: Num rows: 46 Data size: 404 Basic stats: COMPLETE Column stats: NONE
+                          Reduce Output Operator
+                            key expressions: _col0 (type: int)
+                            sort order: +
+                            Map-reduce partition columns: _col0 (type: int)
+                            Statistics: Num rows: 46 Data size: 404 Basic stats: COMPLETE Column stats: NONE
+                            value expressions: _col0 (type: int), _col1 (type: string)
+            Local Work:
+              Map Reduce Local Work
+        Reducer 3 
+            Reduce Operator Tree:
+              Extract
+                Statistics: Num rows: 46 Data size: 404 Basic stats: COMPLETE Column stats: NONE
+                File Output Operator
+                  compressed: false
+                  Statistics: Num rows: 46 Data size: 404 Basic stats: COMPLETE Column stats: NONE
+                  table:
+                      input format: org.apache.hadoop.mapred.TextInputFormat
+                      output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+                      serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+                      name: default.test_table3
 
   Stage: Stage-0
     Move Operator
@@ -707,12 +869,13 @@ JOIN
 ON a.key = b.key
 POSTHOOK: type: QUERY
 STAGE DEPENDENCIES:
-  Stage-1 is a root stage
+  Stage-3 is a root stage
+  Stage-1 depends on stages: Stage-3
   Stage-0 depends on stages: Stage-1
   Stage-2 depends on stages: Stage-0
 
 STAGE PLANS:
-  Stage: Stage-1
+  Stage: Stage-3
     Spark
 #### A masked pattern was here ####
       Vertices:
@@ -728,7 +891,35 @@ STAGE PLANS:
                       expressions: key (type: int), concat(value, value) (type: string)
                       outputColumnNames: _col0, _col1
                       Statistics: Num rows: 5 Data size: 35 Basic stats: COMPLETE Column stats: NONE
-                      Sorted Merge Bucket Map Join Operator
+                      Spark HashTable Sink Operator
+                        condition expressions:
+                          0 {_col1}
+                          1 {_col1}
+                        keys:
+                          0 _col0 (type: int)
+                          1 _col0 (type: int)
+            Local Work:
+              Map Reduce Local Work
+
+  Stage: Stage-1
+    Spark
+      Edges:
+        Reducer 3 <- Map 2 (PARTITION-LEVEL SORT, 1)
+#### A masked pattern was here ####
+      Vertices:
+        Map 2 
+            Map Operator Tree:
+                TableScan
+                  alias: test_table2
+                  Statistics: Num rows: 84 Data size: 736 Basic stats: COMPLETE Column stats: NONE
+                  Filter Operator
+                    predicate: key is not null (type: boolean)
+                    Statistics: Num rows: 42 Data size: 368 Basic stats: COMPLETE Column stats: NONE
+                    Select Operator
+                      expressions: key (type: int), concat(value, value) (type: string)
+                      outputColumnNames: _col0, _col1
+                      Statistics: Num rows: 42 Data size: 368 Basic stats: COMPLETE Column stats: NONE
+                      Map Join Operator
                         condition map:
                              Inner Join 0 to 1
                         condition expressions:
@@ -738,16 +929,33 @@ STAGE PLANS:
                           0 _col0 (type: int)
                           1 _col0 (type: int)
                         outputColumnNames: _col0, _col1, _col3
+                        input vertices:
+                          0 Map 1
+                        Statistics: Num rows: 46 Data size: 404 Basic stats: COMPLETE Column stats: NONE
                         Select Operator
                           expressions: _col0 (type: int), concat(_col1, _col3) (type: string)
                           outputColumnNames: _col0, _col1
-                          File Output Operator
-                            compressed: false
-                            table:
-                                input format: org.apache.hadoop.mapred.TextInputFormat
-                                output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
-                                serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
-                                name: default.test_table3
+                          Statistics: Num rows: 46 Data size: 404 Basic stats: COMPLETE Column stats: NONE
+                          Reduce Output Operator
+                            key expressions: _col0 (type: int)
+                            sort order: +
+                            Map-reduce partition columns: _col0 (type: int)
+                            Statistics: Num rows: 46 Data size: 404 Basic stats: COMPLETE Column stats: NONE
+                            value expressions: _col0 (type: int), _col1 (type: string)
+            Local Work:
+              Map Reduce Local Work
+        Reducer 3 
+            Reduce Operator Tree:
+              Extract
+                Statistics: Num rows: 46 Data size: 404 Basic stats: COMPLETE Column stats: NONE
+                File Output Operator
+                  compressed: false
+                  Statistics: Num rows: 46 Data size: 404 Basic stats: COMPLETE Column stats: NONE
+                  table:
+                      input format: org.apache.hadoop.mapred.TextInputFormat
+                      output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+                      serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+                      name: default.test_table3
 
   Stage: Stage-0
     Move Operator
@@ -855,15 +1063,14 @@ JOIN
 ON a.key = b.key
 POSTHOOK: type: QUERY
 STAGE DEPENDENCIES:
-  Stage-1 is a root stage
+  Stage-3 is a root stage
+  Stage-1 depends on stages: Stage-3
   Stage-0 depends on stages: Stage-1
   Stage-2 depends on stages: Stage-0
 
 STAGE PLANS:
-  Stage: Stage-1
+  Stage: Stage-3
     Spark
-      Edges:
-        Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1)
 #### A masked pattern was here ####
       Vertices:
         Map 1 
@@ -878,7 +1085,35 @@ STAGE PLANS:
                       expressions: key (type: int), value (type: string)
                       outputColumnNames: _col0, _col1
                       Statistics: Num rows: 5 Data size: 35 Basic stats: COMPLETE Column stats: NONE
-                      Sorted Merge Bucket Map Join Operator
+                      Spark HashTable Sink Operator
+                        condition expressions:
+                          0 {_col1}
+                          1 {_col1}
+                        keys:
+                          0 _col0 (type: int)
+                          1 _col0 (type: int)
+            Local Work:
+              Map Reduce Local Work
+
+  Stage: Stage-1
+    Spark
+      Edges:
+        Reducer 3 <- Map 2 (PARTITION-LEVEL SORT, 1)
+#### A masked pattern was here ####
+      Vertices:
+        Map 2 
+            Map Operator Tree:
+                TableScan
+                  alias: test_table2
+                  Statistics: Num rows: 84 Data size: 736 Basic stats: COMPLETE Column stats: NONE
+                  Filter Operator
+                    predicate: key is not null (type: boolean)
+                    Statistics: Num rows: 42 Data size: 368 Basic stats: COMPLETE Column stats: NONE
+                    Select Operator
+                      expressions: key (type: int), value (type: string)
+                      outputColumnNames: _col0, _col1
+                      Statistics: Num rows: 42 Data size: 368 Basic stats: COMPLETE Column stats: NONE
+                      Map Join Operator
                         condition map:
                              Inner Join 0 to 1
                         condition expressions:
@@ -888,19 +1123,28 @@ STAGE PLANS:
                           0 _col0 (type: int)
                           1 _col0 (type: int)
                         outputColumnNames: _col0, _col1, _col3
+                        input vertices:
+                          0 Map 1
+                        Statistics: Num rows: 46 Data size: 404 Basic stats: COMPLETE Column stats: NONE
                         Select Operator
                           expressions: (_col0 + _col0) (type: int), concat(_col1, _col3) (type: string)
                           outputColumnNames: _col0, _col1
+                          Statistics: Num rows: 46 Data size: 404 Basic stats: COMPLETE Column stats: NONE
                           Reduce Output Operator
                             key expressions: _col0 (type: int)
                             sort order: +
                             Map-reduce partition columns: _col0 (type: int)
+                            Statistics: Num rows: 46 Data size: 404 Basic stats: COMPLETE Column stats: NONE
                             value expressions: _col0 (type: int), _col1 (type: string)
-        Reducer 2 
+            Local Work:
+              Map Reduce Local Work
+        Reducer 3 
             Reduce Operator Tree:
               Extract
+                Statistics: Num rows: 46 Data size: 404 Basic stats: COMPLETE Column stats: NONE
                 File Output Operator
                   compressed: false
+                  Statistics: Num rows: 46 Data size: 404 Basic stats: COMPLETE Column stats: NONE
                   table:
                       input format: org.apache.hadoop.mapred.TextInputFormat
                       output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat

Modified: hive/branches/spark/ql/src/test/results/clientpositive/spark/bucketsortoptimize_insert_4.q.out
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/results/clientpositive/spark/bucketsortoptimize_insert_4.q.out?rev=1646523&r1=1646522&r2=1646523&view=diff
==============================================================================
--- hive/branches/spark/ql/src/test/results/clientpositive/spark/bucketsortoptimize_insert_4.q.out (original)
+++ hive/branches/spark/ql/src/test/results/clientpositive/spark/bucketsortoptimize_insert_4.q.out Thu Dec 18 19:58:10 2014
@@ -71,12 +71,13 @@ FROM test_table1 a JOIN test_table2 b
 ON a.key = b.key WHERE a.ds = '1' and b.ds = '1'
 POSTHOOK: type: QUERY
 STAGE DEPENDENCIES:
-  Stage-1 is a root stage
+  Stage-3 is a root stage
+  Stage-1 depends on stages: Stage-3
   Stage-0 depends on stages: Stage-1
   Stage-2 depends on stages: Stage-0
 
 STAGE PLANS:
-  Stage: Stage-1
+  Stage: Stage-3
     Spark
 #### A masked pattern was here ####
       Vertices:
@@ -88,7 +89,31 @@ STAGE PLANS:
                   Filter Operator
                     predicate: key is not null (type: boolean)
                     Statistics: Num rows: 5 Data size: 35 Basic stats: COMPLETE Column stats: NONE
-                    Sorted Merge Bucket Map Join Operator
+                    Spark HashTable Sink Operator
+                      condition expressions:
+                        0 {value}
+                        1 {value}
+                      keys:
+                        0 key (type: int)
+                        1 key (type: int)
+            Local Work:
+              Map Reduce Local Work
+
+  Stage: Stage-1
+    Spark
+      Edges:
+        Reducer 3 <- Map 2 (PARTITION-LEVEL SORT, 1)
+#### A masked pattern was here ####
+      Vertices:
+        Map 2 
+            Map Operator Tree:
+                TableScan
+                  alias: b
+                  Statistics: Num rows: 84 Data size: 736 Basic stats: COMPLETE Column stats: NONE
+                  Filter Operator
+                    predicate: key is not null (type: boolean)
+                    Statistics: Num rows: 42 Data size: 368 Basic stats: COMPLETE Column stats: NONE
+                    Map Join Operator
                       condition map:
                            Inner Join 0 to 1
                       condition expressions:
@@ -98,16 +123,33 @@ STAGE PLANS:
                         0 key (type: int)
                         1 key (type: int)
                       outputColumnNames: _col0, _col1, _col7
+                      input vertices:
+                        0 Map 1
+                      Statistics: Num rows: 46 Data size: 404 Basic stats: COMPLETE Column stats: NONE
                       Select Operator
                         expressions: _col0 (type: int), _col0 (type: int), concat(_col1, _col7) (type: string)
                         outputColumnNames: _col0, _col1, _col2
-                        File Output Operator
-                          compressed: false
-                          table:
-                              input format: org.apache.hadoop.mapred.TextInputFormat
-                              output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
-                              serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
-                              name: default.test_table3
+                        Statistics: Num rows: 46 Data size: 404 Basic stats: COMPLETE Column stats: NONE
+                        Reduce Output Operator
+                          key expressions: _col1 (type: int)
+                          sort order: +
+                          Map-reduce partition columns: _col1 (type: int)
+                          Statistics: Num rows: 46 Data size: 404 Basic stats: COMPLETE Column stats: NONE
+                          value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string)
+            Local Work:
+              Map Reduce Local Work
+        Reducer 3 
+            Reduce Operator Tree:
+              Extract
+                Statistics: Num rows: 46 Data size: 404 Basic stats: COMPLETE Column stats: NONE
+                File Output Operator
+                  compressed: false
+                  Statistics: Num rows: 46 Data size: 404 Basic stats: COMPLETE Column stats: NONE
+                  table:
+                      input format: org.apache.hadoop.mapred.TextInputFormat
+                      output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+                      serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+                      name: default.test_table3
 
   Stage: Stage-0
     Move Operator
@@ -224,15 +266,14 @@ FROM test_table1 a JOIN test_table2 b
 ON a.key = b.key WHERE a.ds = '1' and b.ds = '1'
 POSTHOOK: type: QUERY
 STAGE DEPENDENCIES:
-  Stage-1 is a root stage
+  Stage-3 is a root stage
+  Stage-1 depends on stages: Stage-3
   Stage-0 depends on stages: Stage-1
   Stage-2 depends on stages: Stage-0
 
 STAGE PLANS:
-  Stage: Stage-1
+  Stage: Stage-3
     Spark
-      Edges:
-        Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1)
 #### A masked pattern was here ####
       Vertices:
         Map 1 
@@ -243,7 +284,31 @@ STAGE PLANS:
                   Filter Operator
                     predicate: key is not null (type: boolean)
                     Statistics: Num rows: 5 Data size: 35 Basic stats: COMPLETE Column stats: NONE
-                    Sorted Merge Bucket Map Join Operator
+                    Spark HashTable Sink Operator
+                      condition expressions:
+                        0 {value}
+                        1 
+                      keys:
+                        0 key (type: int)
+                        1 key (type: int)
+            Local Work:
+              Map Reduce Local Work
+
+  Stage: Stage-1
+    Spark
+      Edges:
+        Reducer 3 <- Map 2 (PARTITION-LEVEL SORT, 1)
+#### A masked pattern was here ####
+      Vertices:
+        Map 2 
+            Map Operator Tree:
+                TableScan
+                  alias: b
+                  Statistics: Num rows: 84 Data size: 736 Basic stats: COMPLETE Column stats: NONE
+                  Filter Operator
+                    predicate: key is not null (type: boolean)
+                    Statistics: Num rows: 42 Data size: 368 Basic stats: COMPLETE Column stats: NONE
+                    Map Join Operator
                       condition map:
                            Inner Join 0 to 1
                       condition expressions:
@@ -253,16 +318,24 @@ STAGE PLANS:
                         0 key (type: int)
                         1 key (type: int)
                       outputColumnNames: _col0, _col1
+                      input vertices:
+                        0 Map 1
+                      Statistics: Num rows: 46 Data size: 404 Basic stats: COMPLETE Column stats: NONE
                       Reduce Output Operator
                         key expressions: _col1 (type: string)
                         sort order: +
                         Map-reduce partition columns: _col1 (type: string)
+                        Statistics: Num rows: 46 Data size: 404 Basic stats: COMPLETE Column stats: NONE
                         value expressions: _col0 (type: int), _col1 (type: string)
-        Reducer 2 
+            Local Work:
+              Map Reduce Local Work
+        Reducer 3 
             Reduce Operator Tree:
               Extract
+                Statistics: Num rows: 46 Data size: 404 Basic stats: COMPLETE Column stats: NONE
                 File Output Operator
                   compressed: false
+                  Statistics: Num rows: 46 Data size: 404 Basic stats: COMPLETE Column stats: NONE
                   table:
                       input format: org.apache.hadoop.mapred.TextInputFormat
                       output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat

Modified: hive/branches/spark/ql/src/test/results/clientpositive/spark/bucketsortoptimize_insert_6.q.out
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/results/clientpositive/spark/bucketsortoptimize_insert_6.q.out?rev=1646523&r1=1646522&r2=1646523&view=diff
==============================================================================
--- hive/branches/spark/ql/src/test/results/clientpositive/spark/bucketsortoptimize_insert_6.q.out (original)
+++ hive/branches/spark/ql/src/test/results/clientpositive/spark/bucketsortoptimize_insert_6.q.out Thu Dec 18 19:58:10 2014
@@ -73,12 +73,13 @@ FROM test_table1 a JOIN test_table2 b
 ON a.key = b.key and a.key2 = b.key2 WHERE a.ds = '1' and b.ds = '1'
 POSTHOOK: type: QUERY
 STAGE DEPENDENCIES:
-  Stage-1 is a root stage
+  Stage-3 is a root stage
+  Stage-1 depends on stages: Stage-3
   Stage-0 depends on stages: Stage-1
   Stage-2 depends on stages: Stage-0
 
 STAGE PLANS:
-  Stage: Stage-1
+  Stage: Stage-3
     Spark
 #### A masked pattern was here ####
       Vertices:
@@ -90,7 +91,31 @@ STAGE PLANS:
                   Filter Operator
                     predicate: (key is not null and key2 is not null) (type: boolean)
                     Statistics: Num rows: 3 Data size: 27 Basic stats: COMPLETE Column stats: NONE
-                    Sorted Merge Bucket Map Join Operator
+                    Spark HashTable Sink Operator
+                      condition expressions:
+                        0 {value}
+                        1 {value}
+                      keys:
+                        0 key (type: int), key2 (type: int)
+                        1 key (type: int), key2 (type: int)
+            Local Work:
+              Map Reduce Local Work
+
+  Stage: Stage-1
+    Spark
+      Edges:
+        Reducer 3 <- Map 2 (PARTITION-LEVEL SORT, 1)
+#### A masked pattern was here ####
+      Vertices:
+        Map 2 
+            Map Operator Tree:
+                TableScan
+                  alias: b
+                  Statistics: Num rows: 84 Data size: 979 Basic stats: COMPLETE Column stats: NONE
+                  Filter Operator
+                    predicate: (key is not null and key2 is not null) (type: boolean)
+                    Statistics: Num rows: 21 Data size: 244 Basic stats: COMPLETE Column stats: NONE
+                    Map Join Operator
                       condition map:
                            Inner Join 0 to 1
                       condition expressions:
@@ -100,16 +125,33 @@ STAGE PLANS:
                         0 key (type: int), key2 (type: int)
                         1 key (type: int), key2 (type: int)
                       outputColumnNames: _col0, _col1, _col2, _col9
+                      input vertices:
+                        0 Map 1
+                      Statistics: Num rows: 23 Data size: 268 Basic stats: COMPLETE Column stats: NONE
                       Select Operator
                         expressions: _col0 (type: int), _col1 (type: int), concat(_col2, _col9) (type: string)
                         outputColumnNames: _col0, _col1, _col2
-                        File Output Operator
-                          compressed: false
-                          table:
-                              input format: org.apache.hadoop.mapred.TextInputFormat
-                              output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
-                              serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
-                              name: default.test_table3
+                        Statistics: Num rows: 23 Data size: 268 Basic stats: COMPLETE Column stats: NONE
+                        Reduce Output Operator
+                          key expressions: _col0 (type: int), _col1 (type: int)
+                          sort order: +-
+                          Map-reduce partition columns: _col0 (type: int), _col1 (type: int)
+                          Statistics: Num rows: 23 Data size: 268 Basic stats: COMPLETE Column stats: NONE
+                          value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string)
+            Local Work:
+              Map Reduce Local Work
+        Reducer 3 
+            Reduce Operator Tree:
+              Extract
+                Statistics: Num rows: 23 Data size: 268 Basic stats: COMPLETE Column stats: NONE
+                File Output Operator
+                  compressed: false
+                  Statistics: Num rows: 23 Data size: 268 Basic stats: COMPLETE Column stats: NONE
+                  table:
+                      input format: org.apache.hadoop.mapred.TextInputFormat
+                      output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+                      serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+                      name: default.test_table3
 
   Stage: Stage-0
     Move Operator
@@ -214,12 +256,13 @@ ON a.key = b.key and a.key2 = b.key2 WHE
 )subq1
 POSTHOOK: type: QUERY
 STAGE DEPENDENCIES:
-  Stage-1 is a root stage
+  Stage-3 is a root stage
+  Stage-1 depends on stages: Stage-3
   Stage-0 depends on stages: Stage-1
   Stage-2 depends on stages: Stage-0
 
 STAGE PLANS:
-  Stage: Stage-1
+  Stage: Stage-3
     Spark
 #### A masked pattern was here ####
       Vertices:
@@ -231,7 +274,31 @@ STAGE PLANS:
                   Filter Operator
                     predicate: (key is not null and key2 is not null) (type: boolean)
                     Statistics: Num rows: 3 Data size: 27 Basic stats: COMPLETE Column stats: NONE
-                    Sorted Merge Bucket Map Join Operator
+                    Spark HashTable Sink Operator
+                      condition expressions:
+                        0 {value}
+                        1 {value}
+                      keys:
+                        0 key (type: int), key2 (type: int)
+                        1 key (type: int), key2 (type: int)
+            Local Work:
+              Map Reduce Local Work
+
+  Stage: Stage-1
+    Spark
+      Edges:
+        Reducer 3 <- Map 2 (PARTITION-LEVEL SORT, 1)
+#### A masked pattern was here ####
+      Vertices:
+        Map 2 
+            Map Operator Tree:
+                TableScan
+                  alias: b
+                  Statistics: Num rows: 84 Data size: 979 Basic stats: COMPLETE Column stats: NONE
+                  Filter Operator
+                    predicate: (key is not null and key2 is not null) (type: boolean)
+                    Statistics: Num rows: 21 Data size: 244 Basic stats: COMPLETE Column stats: NONE
+                    Map Join Operator
                       condition map:
                            Inner Join 0 to 1
                       condition expressions:
@@ -241,16 +308,33 @@ STAGE PLANS:
                         0 key (type: int), key2 (type: int)
                         1 key (type: int), key2 (type: int)
                       outputColumnNames: _col0, _col1, _col2, _col9
+                      input vertices:
+                        0 Map 1
+                      Statistics: Num rows: 23 Data size: 268 Basic stats: COMPLETE Column stats: NONE
                       Select Operator
                         expressions: _col0 (type: int), _col1 (type: int), concat(_col2, _col9) (type: string)
                         outputColumnNames: _col0, _col1, _col2
-                        File Output Operator
-                          compressed: false
-                          table:
-                              input format: org.apache.hadoop.mapred.TextInputFormat
-                              output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
-                              serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
-                              name: default.test_table3
+                        Statistics: Num rows: 23 Data size: 268 Basic stats: COMPLETE Column stats: NONE
+                        Reduce Output Operator
+                          key expressions: _col0 (type: int), _col1 (type: int)
+                          sort order: +-
+                          Map-reduce partition columns: _col0 (type: int), _col1 (type: int)
+                          Statistics: Num rows: 23 Data size: 268 Basic stats: COMPLETE Column stats: NONE
+                          value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string)
+            Local Work:
+              Map Reduce Local Work
+        Reducer 3 
+            Reduce Operator Tree:
+              Extract
+                Statistics: Num rows: 23 Data size: 268 Basic stats: COMPLETE Column stats: NONE
+                File Output Operator
+                  compressed: false
+                  Statistics: Num rows: 23 Data size: 268 Basic stats: COMPLETE Column stats: NONE
+                  table:
+                      input format: org.apache.hadoop.mapred.TextInputFormat
+                      output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+                      serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+                      name: default.test_table3
 
   Stage: Stage-0
     Move Operator
@@ -355,15 +439,14 @@ FROM test_table1 a JOIN test_table2 b
 ON a.key = b.key and a.key2 = b.key2 WHERE a.ds = '1' and b.ds = '1'
 POSTHOOK: type: QUERY
 STAGE DEPENDENCIES:
-  Stage-1 is a root stage
+  Stage-3 is a root stage
+  Stage-1 depends on stages: Stage-3
   Stage-0 depends on stages: Stage-1
   Stage-2 depends on stages: Stage-0
 
 STAGE PLANS:
-  Stage: Stage-1
+  Stage: Stage-3
     Spark
-      Edges:
-        Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1)
 #### A masked pattern was here ####
       Vertices:
         Map 1 
@@ -374,7 +457,31 @@ STAGE PLANS:
                   Filter Operator
                     predicate: (key is not null and key2 is not null) (type: boolean)
                     Statistics: Num rows: 3 Data size: 27 Basic stats: COMPLETE Column stats: NONE
-                    Sorted Merge Bucket Map Join Operator
+                    Spark HashTable Sink Operator
+                      condition expressions:
+                        0 {value}
+                        1 {value}
+                      keys:
+                        0 key (type: int), key2 (type: int)
+                        1 key (type: int), key2 (type: int)
+            Local Work:
+              Map Reduce Local Work
+
+  Stage: Stage-1
+    Spark
+      Edges:
+        Reducer 3 <- Map 2 (PARTITION-LEVEL SORT, 1)
+#### A masked pattern was here ####
+      Vertices:
+        Map 2 
+            Map Operator Tree:
+                TableScan
+                  alias: b
+                  Statistics: Num rows: 84 Data size: 979 Basic stats: COMPLETE Column stats: NONE
+                  Filter Operator
+                    predicate: (key is not null and key2 is not null) (type: boolean)
+                    Statistics: Num rows: 21 Data size: 244 Basic stats: COMPLETE Column stats: NONE
+                    Map Join Operator
                       condition map:
                            Inner Join 0 to 1
                       condition expressions:
@@ -384,19 +491,28 @@ STAGE PLANS:
                         0 key (type: int), key2 (type: int)
                         1 key (type: int), key2 (type: int)
                       outputColumnNames: _col0, _col1, _col2, _col9
+                      input vertices:
+                        0 Map 1
+                      Statistics: Num rows: 23 Data size: 268 Basic stats: COMPLETE Column stats: NONE
                       Select Operator
                         expressions: _col1 (type: int), _col0 (type: int), concat(_col2, _col9) (type: string)
                         outputColumnNames: _col0, _col1, _col2
+                        Statistics: Num rows: 23 Data size: 268 Basic stats: COMPLETE Column stats: NONE
                         Reduce Output Operator
                           key expressions: _col0 (type: int), _col1 (type: int)
                           sort order: +-
                           Map-reduce partition columns: _col0 (type: int), _col1 (type: int)
+                          Statistics: Num rows: 23 Data size: 268 Basic stats: COMPLETE Column stats: NONE
                           value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string)
-        Reducer 2 
+            Local Work:
+              Map Reduce Local Work
+        Reducer 3 
             Reduce Operator Tree:
               Extract
+                Statistics: Num rows: 23 Data size: 268 Basic stats: COMPLETE Column stats: NONE
                 File Output Operator
                   compressed: false
+                  Statistics: Num rows: 23 Data size: 268 Basic stats: COMPLETE Column stats: NONE
                   table:
                       input format: org.apache.hadoop.mapred.TextInputFormat
                       output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
@@ -441,15 +557,14 @@ ON a.key = b.key and a.key2 = b.key2 WHE
 )subq1
 POSTHOOK: type: QUERY
 STAGE DEPENDENCIES:
-  Stage-1 is a root stage
+  Stage-3 is a root stage
+  Stage-1 depends on stages: Stage-3
   Stage-0 depends on stages: Stage-1
   Stage-2 depends on stages: Stage-0
 
 STAGE PLANS:
-  Stage: Stage-1
+  Stage: Stage-3
     Spark
-      Edges:
-        Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1)
 #### A masked pattern was here ####
       Vertices:
         Map 1 
@@ -460,7 +575,31 @@ STAGE PLANS:
                   Filter Operator
                     predicate: (key is not null and key2 is not null) (type: boolean)
                     Statistics: Num rows: 3 Data size: 27 Basic stats: COMPLETE Column stats: NONE
-                    Sorted Merge Bucket Map Join Operator
+                    Spark HashTable Sink Operator
+                      condition expressions:
+                        0 {value}
+                        1 {value}
+                      keys:
+                        0 key (type: int), key2 (type: int)
+                        1 key (type: int), key2 (type: int)
+            Local Work:
+              Map Reduce Local Work
+
+  Stage: Stage-1
+    Spark
+      Edges:
+        Reducer 3 <- Map 2 (PARTITION-LEVEL SORT, 1)
+#### A masked pattern was here ####
+      Vertices:
+        Map 2 
+            Map Operator Tree:
+                TableScan
+                  alias: b
+                  Statistics: Num rows: 84 Data size: 979 Basic stats: COMPLETE Column stats: NONE
+                  Filter Operator
+                    predicate: (key is not null and key2 is not null) (type: boolean)
+                    Statistics: Num rows: 21 Data size: 244 Basic stats: COMPLETE Column stats: NONE
+                    Map Join Operator
                       condition map:
                            Inner Join 0 to 1
                       condition expressions:
@@ -470,19 +609,28 @@ STAGE PLANS:
                         0 key (type: int), key2 (type: int)
                         1 key (type: int), key2 (type: int)
                       outputColumnNames: _col0, _col1, _col2, _col9
+                      input vertices:
+                        0 Map 1
+                      Statistics: Num rows: 23 Data size: 268 Basic stats: COMPLETE Column stats: NONE
                       Select Operator
                         expressions: _col1 (type: int), _col0 (type: int), concat(_col2, _col9) (type: string)
                         outputColumnNames: _col0, _col1, _col2
+                        Statistics: Num rows: 23 Data size: 268 Basic stats: COMPLETE Column stats: NONE
                         Reduce Output Operator
                           key expressions: _col0 (type: int), _col1 (type: int)
                           sort order: +-
                           Map-reduce partition columns: _col0 (type: int), _col1 (type: int)
+                          Statistics: Num rows: 23 Data size: 268 Basic stats: COMPLETE Column stats: NONE
                           value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string)
-        Reducer 2 
+            Local Work:
+              Map Reduce Local Work
+        Reducer 3 
             Reduce Operator Tree:
               Extract
+                Statistics: Num rows: 23 Data size: 268 Basic stats: COMPLETE Column stats: NONE
                 File Output Operator
                   compressed: false
+                  Statistics: Num rows: 23 Data size: 268 Basic stats: COMPLETE Column stats: NONE
                   table:
                       input format: org.apache.hadoop.mapred.TextInputFormat
                       output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
@@ -533,12 +681,13 @@ ON a.key = b.key and a.key2 = b.key2 WHE
 )subq2
 POSTHOOK: type: QUERY
 STAGE DEPENDENCIES:
-  Stage-1 is a root stage
+  Stage-3 is a root stage
+  Stage-1 depends on stages: Stage-3
   Stage-0 depends on stages: Stage-1
   Stage-2 depends on stages: Stage-0
 
 STAGE PLANS:
-  Stage: Stage-1
+  Stage: Stage-3
     Spark
 #### A masked pattern was here ####
       Vertices:
@@ -550,7 +699,31 @@ STAGE PLANS:
                   Filter Operator
                     predicate: (key is not null and key2 is not null) (type: boolean)
                     Statistics: Num rows: 3 Data size: 27 Basic stats: COMPLETE Column stats: NONE
-                    Sorted Merge Bucket Map Join Operator
+                    Spark HashTable Sink Operator
+                      condition expressions:
+                        0 {value}
+                        1 {value}
+                      keys:
+                        0 key (type: int), key2 (type: int)
+                        1 key (type: int), key2 (type: int)
+            Local Work:
+              Map Reduce Local Work
+
+  Stage: Stage-1
+    Spark
+      Edges:
+        Reducer 3 <- Map 2 (PARTITION-LEVEL SORT, 1)
+#### A masked pattern was here ####
+      Vertices:
+        Map 2 
+            Map Operator Tree:
+                TableScan
+                  alias: b
+                  Statistics: Num rows: 84 Data size: 979 Basic stats: COMPLETE Column stats: NONE
+                  Filter Operator
+                    predicate: (key is not null and key2 is not null) (type: boolean)
+                    Statistics: Num rows: 21 Data size: 244 Basic stats: COMPLETE Column stats: NONE
+                    Map Join Operator
                       condition map:
                            Inner Join 0 to 1
                       condition expressions:
@@ -560,16 +733,33 @@ STAGE PLANS:
                         0 key (type: int), key2 (type: int)
                         1 key (type: int), key2 (type: int)
                       outputColumnNames: _col0, _col1, _col2, _col9
+                      input vertices:
+                        0 Map 1
+                      Statistics: Num rows: 23 Data size: 268 Basic stats: COMPLETE Column stats: NONE
                       Select Operator
                         expressions: _col0 (type: int), _col1 (type: int), concat(_col2, _col9) (type: string)
                         outputColumnNames: _col0, _col1, _col2
-                        File Output Operator
-                          compressed: false
-                          table:
-                              input format: org.apache.hadoop.mapred.TextInputFormat
-                              output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
-                              serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
-                              name: default.test_table3
+                        Statistics: Num rows: 23 Data size: 268 Basic stats: COMPLETE Column stats: NONE
+                        Reduce Output Operator
+                          key expressions: _col0 (type: int), _col1 (type: int)
+                          sort order: +-
+                          Map-reduce partition columns: _col0 (type: int), _col1 (type: int)
+                          Statistics: Num rows: 23 Data size: 268 Basic stats: COMPLETE Column stats: NONE
+                          value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string)
+            Local Work:
+              Map Reduce Local Work
+        Reducer 3 
+            Reduce Operator Tree:
+              Extract
+                Statistics: Num rows: 23 Data size: 268 Basic stats: COMPLETE Column stats: NONE
+                File Output Operator
+                  compressed: false
+                  Statistics: Num rows: 23 Data size: 268 Basic stats: COMPLETE Column stats: NONE
+                  table:
+                      input format: org.apache.hadoop.mapred.TextInputFormat
+                      output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+                      serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+                      name: default.test_table3
 
   Stage: Stage-0
     Move Operator
@@ -692,12 +882,13 @@ ON a.key = b.key and a.key2 = b.key2 WHE
 )subq2
 POSTHOOK: type: QUERY
 STAGE DEPENDENCIES:
-  Stage-1 is a root stage
+  Stage-3 is a root stage
+  Stage-1 depends on stages: Stage-3
   Stage-0 depends on stages: Stage-1
   Stage-2 depends on stages: Stage-0
 
 STAGE PLANS:
-  Stage: Stage-1
+  Stage: Stage-3
     Spark
 #### A masked pattern was here ####
       Vertices:
@@ -709,7 +900,31 @@ STAGE PLANS:
                   Filter Operator
                     predicate: (key is not null and key2 is not null) (type: boolean)
                     Statistics: Num rows: 3 Data size: 27 Basic stats: COMPLETE Column stats: NONE
-                    Sorted Merge Bucket Map Join Operator
+                    Spark HashTable Sink Operator
+                      condition expressions:
+                        0 {value}
+                        1 {value}
+                      keys:
+                        0 key (type: int), key2 (type: int)
+                        1 key (type: int), key2 (type: int)
+            Local Work:
+              Map Reduce Local Work
+
+  Stage: Stage-1
+    Spark
+      Edges:
+        Reducer 3 <- Map 2 (PARTITION-LEVEL SORT, 1)
+#### A masked pattern was here ####
+      Vertices:
+        Map 2 
+            Map Operator Tree:
+                TableScan
+                  alias: b
+                  Statistics: Num rows: 84 Data size: 979 Basic stats: COMPLETE Column stats: NONE
+                  Filter Operator
+                    predicate: (key is not null and key2 is not null) (type: boolean)
+                    Statistics: Num rows: 21 Data size: 244 Basic stats: COMPLETE Column stats: NONE
+                    Map Join Operator
                       condition map:
                            Inner Join 0 to 1
                       condition expressions:
@@ -719,16 +934,33 @@ STAGE PLANS:
                         0 key (type: int), key2 (type: int)
                         1 key (type: int), key2 (type: int)
                       outputColumnNames: _col0, _col1, _col2, _col9
+                      input vertices:
+                        0 Map 1
+                      Statistics: Num rows: 23 Data size: 268 Basic stats: COMPLETE Column stats: NONE
                       Select Operator
                         expressions: _col0 (type: int), _col1 (type: int), concat(_col2, _col9) (type: string)
                         outputColumnNames: _col0, _col1, _col2
-                        File Output Operator
-                          compressed: false
-                          table:
-                              input format: org.apache.hadoop.mapred.TextInputFormat
-                              output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
-                              serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
-                              name: default.test_table3
+                        Statistics: Num rows: 23 Data size: 268 Basic stats: COMPLETE Column stats: NONE
+                        Reduce Output Operator
+                          key expressions: _col0 (type: int), _col1 (type: int)
+                          sort order: +-
+                          Map-reduce partition columns: _col0 (type: int), _col1 (type: int)
+                          Statistics: Num rows: 23 Data size: 268 Basic stats: COMPLETE Column stats: NONE
+                          value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string)
+            Local Work:
+              Map Reduce Local Work
+        Reducer 3 
+            Reduce Operator Tree:
+              Extract
+                Statistics: Num rows: 23 Data size: 268 Basic stats: COMPLETE Column stats: NONE
+                File Output Operator
+                  compressed: false
+                  Statistics: Num rows: 23 Data size: 268 Basic stats: COMPLETE Column stats: NONE
+                  table:
+                      input format: org.apache.hadoop.mapred.TextInputFormat
+                      output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+                      serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+                      name: default.test_table3
 
   Stage: Stage-0
     Move Operator
@@ -861,15 +1093,14 @@ ON a.key = b.key and a.key2 = b.key2 WHE
 )subq2
 POSTHOOK: type: QUERY
 STAGE DEPENDENCIES:
-  Stage-1 is a root stage
+  Stage-3 is a root stage
+  Stage-1 depends on stages: Stage-3
   Stage-0 depends on stages: Stage-1
   Stage-2 depends on stages: Stage-0
 
 STAGE PLANS:
-  Stage: Stage-1
+  Stage: Stage-3
     Spark
-      Edges:
-        Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 1)
 #### A masked pattern was here ####
       Vertices:
         Map 1 
@@ -880,7 +1111,31 @@ STAGE PLANS:
                   Filter Operator
                     predicate: (key is not null and key2 is not null) (type: boolean)
                     Statistics: Num rows: 3 Data size: 27 Basic stats: COMPLETE Column stats: NONE
-                    Sorted Merge Bucket Map Join Operator
+                    Spark HashTable Sink Operator
+                      condition expressions:
+                        0 {value}
+                        1 {value}
+                      keys:
+                        0 key (type: int), key2 (type: int)
+                        1 key (type: int), key2 (type: int)
+            Local Work:
+              Map Reduce Local Work
+
+  Stage: Stage-1
+    Spark
+      Edges:
+        Reducer 3 <- Map 2 (PARTITION-LEVEL SORT, 1)
+#### A masked pattern was here ####
+      Vertices:
+        Map 2 
+            Map Operator Tree:
+                TableScan
+                  alias: b
+                  Statistics: Num rows: 84 Data size: 979 Basic stats: COMPLETE Column stats: NONE
+                  Filter Operator
+                    predicate: (key is not null and key2 is not null) (type: boolean)
+                    Statistics: Num rows: 21 Data size: 244 Basic stats: COMPLETE Column stats: NONE
+                    Map Join Operator
                       condition map:
                            Inner Join 0 to 1
                       condition expressions:
@@ -890,19 +1145,28 @@ STAGE PLANS:
                         0 key (type: int), key2 (type: int)
                         1 key (type: int), key2 (type: int)
                       outputColumnNames: _col0, _col1, _col2, _col9
+                      input vertices:
+                        0 Map 1
+                      Statistics: Num rows: 23 Data size: 268 Basic stats: COMPLETE Column stats: NONE
                       Select Operator
                         expressions: _col0 (type: int), _col1 (type: int), concat(_col2, _col9) (type: string)
                         outputColumnNames: _col0, _col1, _col2
+                        Statistics: Num rows: 23 Data size: 268 Basic stats: COMPLETE Column stats: NONE
                         Reduce Output Operator
                           key expressions: _col0 (type: int), _col1 (type: int)
                           sort order: --
                           Map-reduce partition columns: _col0 (type: int), _col1 (type: int)
+                          Statistics: Num rows: 23 Data size: 268 Basic stats: COMPLETE Column stats: NONE
                           value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string)
-        Reducer 2 
+            Local Work:
+              Map Reduce Local Work
+        Reducer 3 
             Reduce Operator Tree:
               Extract
+                Statistics: Num rows: 23 Data size: 268 Basic stats: COMPLETE Column stats: NONE
                 File Output Operator
                   compressed: false
+                  Statistics: Num rows: 23 Data size: 268 Basic stats: COMPLETE Column stats: NONE
                   table:
                       input format: org.apache.hadoop.mapred.TextInputFormat
                       output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat

Modified: hive/branches/spark/ql/src/test/results/clientpositive/spark/bucketsortoptimize_insert_7.q.out
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/results/clientpositive/spark/bucketsortoptimize_insert_7.q.out?rev=1646523&r1=1646522&r2=1646523&view=diff
==============================================================================
--- hive/branches/spark/ql/src/test/results/clientpositive/spark/bucketsortoptimize_insert_7.q.out (original)
+++ hive/branches/spark/ql/src/test/results/clientpositive/spark/bucketsortoptimize_insert_7.q.out Thu Dec 18 19:58:10 2014
@@ -73,12 +73,13 @@ ON a.key = b.key WHERE a.ds = '1' and b.
 and (a.key = 0 or a.key = 5)
 POSTHOOK: type: QUERY
 STAGE DEPENDENCIES:
-  Stage-1 is a root stage
+  Stage-3 is a root stage
+  Stage-1 depends on stages: Stage-3
   Stage-0 depends on stages: Stage-1
   Stage-2 depends on stages: Stage-0
 
 STAGE PLANS:
-  Stage: Stage-1
+  Stage: Stage-3
     Spark
 #### A masked pattern was here ####
       Vertices:
@@ -90,7 +91,31 @@ STAGE PLANS:
                   Filter Operator
                     predicate: (key is not null and ((key = 0) or (key = 5))) (type: boolean)
                     Statistics: Num rows: 4 Data size: 28 Basic stats: COMPLETE Column stats: NONE
-                    Sorted Merge Bucket Map Join Operator
+                    Spark HashTable Sink Operator
+                      condition expressions:
+                        0 {value}
+                        1 {value}
+                      keys:
+                        0 key (type: int)
+                        1 key (type: int)
+            Local Work:
+              Map Reduce Local Work
+
+  Stage: Stage-1
+    Spark
+      Edges:
+        Reducer 3 <- Map 2 (PARTITION-LEVEL SORT, 1)
+#### A masked pattern was here ####
+      Vertices:
+        Map 2 
+            Map Operator Tree:
+                TableScan
+                  alias: b
+                  Statistics: Num rows: 84 Data size: 736 Basic stats: COMPLETE Column stats: NONE
+                  Filter Operator
+                    predicate: (key is not null and ((key = 0) or (key = 5))) (type: boolean)
+                    Statistics: Num rows: 42 Data size: 368 Basic stats: COMPLETE Column stats: NONE
+                    Map Join Operator
                       condition map:
                            Inner Join 0 to 1
                       condition expressions:
@@ -100,16 +125,33 @@ STAGE PLANS:
                         0 key (type: int)
                         1 key (type: int)
                       outputColumnNames: _col0, _col1, _col7
+                      input vertices:
+                        0 Map 1
+                      Statistics: Num rows: 46 Data size: 404 Basic stats: COMPLETE Column stats: NONE
                       Select Operator
                         expressions: _col0 (type: int), concat(_col1, _col7) (type: string)
                         outputColumnNames: _col0, _col1
-                        File Output Operator
-                          compressed: false
-                          table:
-                              input format: org.apache.hadoop.mapred.TextInputFormat
-                              output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
-                              serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
-                              name: default.test_table3
+                        Statistics: Num rows: 46 Data size: 404 Basic stats: COMPLETE Column stats: NONE
+                        Reduce Output Operator
+                          key expressions: _col0 (type: int)
+                          sort order: +
+                          Map-reduce partition columns: _col0 (type: int)
+                          Statistics: Num rows: 46 Data size: 404 Basic stats: COMPLETE Column stats: NONE
+                          value expressions: _col0 (type: int), _col1 (type: string)
+            Local Work:
+              Map Reduce Local Work
+        Reducer 3 
+            Reduce Operator Tree:
+              Extract
+                Statistics: Num rows: 46 Data size: 404 Basic stats: COMPLETE Column stats: NONE
+                File Output Operator
+                  compressed: false
+                  Statistics: Num rows: 46 Data size: 404 Basic stats: COMPLETE Column stats: NONE
+                  table:
+                      input format: org.apache.hadoop.mapred.TextInputFormat
+                      output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+                      serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+                      name: default.test_table3
 
   Stage: Stage-0
     Move Operator
@@ -209,12 +251,13 @@ JOIN
 ON a.key = b.key
 POSTHOOK: type: QUERY
 STAGE DEPENDENCIES:
-  Stage-1 is a root stage
+  Stage-3 is a root stage
+  Stage-1 depends on stages: Stage-3
   Stage-0 depends on stages: Stage-1
   Stage-2 depends on stages: Stage-0
 
 STAGE PLANS:
-  Stage: Stage-1
+  Stage: Stage-3
     Spark
 #### A masked pattern was here ####
       Vertices:
@@ -230,7 +273,35 @@ STAGE PLANS:
                       expressions: key (type: int), value (type: string)
                       outputColumnNames: _col0, _col1
                       Statistics: Num rows: 5 Data size: 35 Basic stats: COMPLETE Column stats: NONE
-                      Sorted Merge Bucket Map Join Operator
+                      Spark HashTable Sink Operator
+                        condition expressions:
+                          0 {_col1}
+                          1 {_col1}
+                        keys:
+                          0 _col0 (type: int)
+                          1 _col0 (type: int)
+            Local Work:
+              Map Reduce Local Work
+
+  Stage: Stage-1
+    Spark
+      Edges:
+        Reducer 3 <- Map 2 (PARTITION-LEVEL SORT, 1)
+#### A masked pattern was here ####
+      Vertices:
+        Map 2 
+            Map Operator Tree:
+                TableScan
+                  alias: test_table2
+                  Statistics: Num rows: 84 Data size: 736 Basic stats: COMPLETE Column stats: NONE
+                  Filter Operator
+                    predicate: (((key = 0) or (key = 5)) and key is not null) (type: boolean)
+                    Statistics: Num rows: 42 Data size: 368 Basic stats: COMPLETE Column stats: NONE
+                    Select Operator
+                      expressions: key (type: int), value (type: string)
+                      outputColumnNames: _col0, _col1
+                      Statistics: Num rows: 42 Data size: 368 Basic stats: COMPLETE Column stats: NONE
+                      Map Join Operator
                         condition map:
                              Inner Join 0 to 1
                         condition expressions:
@@ -240,16 +311,33 @@ STAGE PLANS:
                           0 _col0 (type: int)
                           1 _col0 (type: int)
                         outputColumnNames: _col0, _col1, _col3
+                        input vertices:
+                          0 Map 1
+                        Statistics: Num rows: 46 Data size: 404 Basic stats: COMPLETE Column stats: NONE
                         Select Operator
                           expressions: _col0 (type: int), concat(_col1, _col3) (type: string)
                           outputColumnNames: _col0, _col1
-                          File Output Operator
-                            compressed: false
-                            table:
-                                input format: org.apache.hadoop.mapred.TextInputFormat
-                                output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
-                                serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
-                                name: default.test_table3
+                          Statistics: Num rows: 46 Data size: 404 Basic stats: COMPLETE Column stats: NONE
+                          Reduce Output Operator
+                            key expressions: _col0 (type: int)
+                            sort order: +
+                            Map-reduce partition columns: _col0 (type: int)
+                            Statistics: Num rows: 46 Data size: 404 Basic stats: COMPLETE Column stats: NONE
+                            value expressions: _col0 (type: int), _col1 (type: string)
+            Local Work:
+              Map Reduce Local Work
+        Reducer 3 
+            Reduce Operator Tree:
+              Extract
+                Statistics: Num rows: 46 Data size: 404 Basic stats: COMPLETE Column stats: NONE
+                File Output Operator
+                  compressed: false
+                  Statistics: Num rows: 46 Data size: 404 Basic stats: COMPLETE Column stats: NONE
+                  table:
+                      input format: org.apache.hadoop.mapred.TextInputFormat
+                      output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+                      serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+                      name: default.test_table3
 
   Stage: Stage-0
     Move Operator
@@ -355,12 +443,13 @@ ON a.key = b.key
 WHERE a.key = 0 or a.key = 5
 POSTHOOK: type: QUERY
 STAGE DEPENDENCIES:
-  Stage-1 is a root stage
+  Stage-3 is a root stage
+  Stage-1 depends on stages: Stage-3
   Stage-0 depends on stages: Stage-1
   Stage-2 depends on stages: Stage-0
 
 STAGE PLANS:
-  Stage: Stage-1
+  Stage: Stage-3
     Spark
 #### A masked pattern was here ####
       Vertices:
@@ -376,7 +465,35 @@ STAGE PLANS:
                       expressions: key (type: int), value (type: string)
                       outputColumnNames: _col0, _col1
                       Statistics: Num rows: 2 Data size: 14 Basic stats: COMPLETE Column stats: NONE
-                      Sorted Merge Bucket Map Join Operator
+                      Spark HashTable Sink Operator
+                        condition expressions:
+                          0 {_col1}
+                          1 {_col1}
+                        keys:
+                          0 _col0 (type: int)
+                          1 _col0 (type: int)
+            Local Work:
+              Map Reduce Local Work
+
+  Stage: Stage-1
+    Spark
+      Edges:
+        Reducer 3 <- Map 2 (PARTITION-LEVEL SORT, 1)
+#### A masked pattern was here ####
+      Vertices:
+        Map 2 
+            Map Operator Tree:
+                TableScan
+                  alias: test_table2
+                  Statistics: Num rows: 84 Data size: 736 Basic stats: COMPLETE Column stats: NONE
+                  Filter Operator
+                    predicate: (((key < 8) and key is not null) and ((key = 0) or (key = 5))) (type: boolean)
+                    Statistics: Num rows: 14 Data size: 122 Basic stats: COMPLETE Column stats: NONE
+                    Select Operator
+                      expressions: key (type: int), value (type: string)
+                      outputColumnNames: _col0, _col1
+                      Statistics: Num rows: 14 Data size: 122 Basic stats: COMPLETE Column stats: NONE
+                      Map Join Operator
                         condition map:
                              Inner Join 0 to 1
                         condition expressions:
@@ -386,16 +503,33 @@ STAGE PLANS:
                           0 _col0 (type: int)
                           1 _col0 (type: int)
                         outputColumnNames: _col0, _col1, _col3
+                        input vertices:
+                          0 Map 1
+                        Statistics: Num rows: 15 Data size: 134 Basic stats: COMPLETE Column stats: NONE
                         Select Operator
                           expressions: _col0 (type: int), concat(_col1, _col3) (type: string)
                           outputColumnNames: _col0, _col1
-                          File Output Operator
-                            compressed: false
-                            table:
-                                input format: org.apache.hadoop.mapred.TextInputFormat
-                                output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
-                                serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
-                                name: default.test_table3
+                          Statistics: Num rows: 15 Data size: 134 Basic stats: COMPLETE Column stats: NONE
+                          Reduce Output Operator
+                            key expressions: _col0 (type: int)
+                            sort order: +
+                            Map-reduce partition columns: _col0 (type: int)
+                            Statistics: Num rows: 15 Data size: 134 Basic stats: COMPLETE Column stats: NONE
+                            value expressions: _col0 (type: int), _col1 (type: string)
+            Local Work:
+              Map Reduce Local Work
+        Reducer 3 
+            Reduce Operator Tree:
+              Extract
+                Statistics: Num rows: 15 Data size: 134 Basic stats: COMPLETE Column stats: NONE
+                File Output Operator
+                  compressed: false
+                  Statistics: Num rows: 15 Data size: 134 Basic stats: COMPLETE Column stats: NONE
+                  table:
+                      input format: org.apache.hadoop.mapred.TextInputFormat
+                      output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+                      serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+                      name: default.test_table3
 
   Stage: Stage-0
     Move Operator

Modified: hive/branches/spark/ql/src/test/results/clientpositive/spark/bucketsortoptimize_insert_8.q.out
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/results/clientpositive/spark/bucketsortoptimize_insert_8.q.out?rev=1646523&r1=1646522&r2=1646523&view=diff
==============================================================================
--- hive/branches/spark/ql/src/test/results/clientpositive/spark/bucketsortoptimize_insert_8.q.out (original)
+++ hive/branches/spark/ql/src/test/results/clientpositive/spark/bucketsortoptimize_insert_8.q.out Thu Dec 18 19:58:10 2014
@@ -71,12 +71,13 @@ FROM test_table1 a JOIN test_table2 b
 ON a.key = b.key WHERE a.ds = '1' and b.ds = '1'
 POSTHOOK: type: QUERY
 STAGE DEPENDENCIES:
-  Stage-1 is a root stage
+  Stage-3 is a root stage
+  Stage-1 depends on stages: Stage-3
   Stage-0 depends on stages: Stage-1
   Stage-2 depends on stages: Stage-0
 
 STAGE PLANS:
-  Stage: Stage-1
+  Stage: Stage-3
     Spark
 #### A masked pattern was here ####
       Vertices:
@@ -88,7 +89,31 @@ STAGE PLANS:
                   Filter Operator
                     predicate: key is not null (type: boolean)
                     Statistics: Num rows: 5 Data size: 35 Basic stats: COMPLETE Column stats: NONE
-                    Sorted Merge Bucket Map Join Operator
+                    Spark HashTable Sink Operator
+                      condition expressions:
+                        0 {value}
+                        1 {key} {value}
+                      keys:
+                        0 key (type: int)
+                        1 key (type: int)
+            Local Work:
+              Map Reduce Local Work
+
+  Stage: Stage-1
+    Spark
+      Edges:
+        Reducer 3 <- Map 2 (PARTITION-LEVEL SORT, 1)
+#### A masked pattern was here ####
+      Vertices:
+        Map 2 
+            Map Operator Tree:
+                TableScan
+                  alias: b
+                  Statistics: Num rows: 84 Data size: 736 Basic stats: COMPLETE Column stats: NONE
+                  Filter Operator
+                    predicate: key is not null (type: boolean)
+                    Statistics: Num rows: 42 Data size: 368 Basic stats: COMPLETE Column stats: NONE
+                    Map Join Operator
                       condition map:
                            Inner Join 0 to 1
                       condition expressions:
@@ -98,16 +123,33 @@ STAGE PLANS:
                         0 key (type: int)
                         1 key (type: int)
                       outputColumnNames: _col0, _col1, _col6, _col7
+                      input vertices:
+                        0 Map 1
+                      Statistics: Num rows: 46 Data size: 404 Basic stats: COMPLETE Column stats: NONE
                       Select Operator
                         expressions: _col0 (type: int), _col6 (type: int), concat(_col1, _col7) (type: string)
                         outputColumnNames: _col0, _col1, _col2
-                        File Output Operator
-                          compressed: false
-                          table:
-                              input format: org.apache.hadoop.mapred.TextInputFormat
-                              output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
-                              serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
-                              name: default.test_table3
+                        Statistics: Num rows: 46 Data size: 404 Basic stats: COMPLETE Column stats: NONE
+                        Reduce Output Operator
+                          key expressions: _col0 (type: int)
+                          sort order: +
+                          Map-reduce partition columns: _col0 (type: int)
+                          Statistics: Num rows: 46 Data size: 404 Basic stats: COMPLETE Column stats: NONE
+                          value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string)
+            Local Work:
+              Map Reduce Local Work
+        Reducer 3 
+            Reduce Operator Tree:
+              Extract
+                Statistics: Num rows: 46 Data size: 404 Basic stats: COMPLETE Column stats: NONE
+                File Output Operator
+                  compressed: false
+                  Statistics: Num rows: 46 Data size: 404 Basic stats: COMPLETE Column stats: NONE
+                  table:
+                      input format: org.apache.hadoop.mapred.TextInputFormat
+                      output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+                      serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+                      name: default.test_table3
 
   Stage: Stage-0
     Move Operator
@@ -206,12 +248,13 @@ FROM test_table1 a JOIN test_table2 b
 ON a.key = b.key WHERE a.ds = '1' and b.ds = '1'
 POSTHOOK: type: QUERY
 STAGE DEPENDENCIES:
-  Stage-1 is a root stage
+  Stage-3 is a root stage
+  Stage-1 depends on stages: Stage-3
   Stage-0 depends on stages: Stage-1
   Stage-2 depends on stages: Stage-0
 
 STAGE PLANS:
-  Stage: Stage-1
+  Stage: Stage-3
     Spark
 #### A masked pattern was here ####
       Vertices:
@@ -223,7 +266,31 @@ STAGE PLANS:
                   Filter Operator
                     predicate: key is not null (type: boolean)
                     Statistics: Num rows: 5 Data size: 35 Basic stats: COMPLETE Column stats: NONE
-                    Sorted Merge Bucket Map Join Operator
+                    Spark HashTable Sink Operator
+                      condition expressions:
+                        0 {value}
+                        1 {key} {value}
+                      keys:
+                        0 key (type: int)
+                        1 key (type: int)
+            Local Work:
+              Map Reduce Local Work
+
+  Stage: Stage-1
+    Spark
+      Edges:
+        Reducer 3 <- Map 2 (PARTITION-LEVEL SORT, 1)
+#### A masked pattern was here ####
+      Vertices:
+        Map 2 
+            Map Operator Tree:
+                TableScan
+                  alias: b
+                  Statistics: Num rows: 84 Data size: 736 Basic stats: COMPLETE Column stats: NONE
+                  Filter Operator
+                    predicate: key is not null (type: boolean)
+                    Statistics: Num rows: 42 Data size: 368 Basic stats: COMPLETE Column stats: NONE
+                    Map Join Operator
                       condition map:
                            Inner Join 0 to 1
                       condition expressions:
@@ -233,16 +300,33 @@ STAGE PLANS:
                         0 key (type: int)
                         1 key (type: int)
                       outputColumnNames: _col0, _col1, _col6, _col7
+                      input vertices:
+                        0 Map 1
+                      Statistics: Num rows: 46 Data size: 404 Basic stats: COMPLETE Column stats: NONE
                       Select Operator
                         expressions: _col6 (type: int), _col0 (type: int), concat(_col1, _col7) (type: string)
                         outputColumnNames: _col0, _col1, _col2
-                        File Output Operator
-                          compressed: false
-                          table:
-                              input format: org.apache.hadoop.mapred.TextInputFormat
-                              output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
-                              serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
-                              name: default.test_table3
+                        Statistics: Num rows: 46 Data size: 404 Basic stats: COMPLETE Column stats: NONE
+                        Reduce Output Operator
+                          key expressions: _col0 (type: int)
+                          sort order: +
+                          Map-reduce partition columns: _col0 (type: int)
+                          Statistics: Num rows: 46 Data size: 404 Basic stats: COMPLETE Column stats: NONE
+                          value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: string)
+            Local Work:
+              Map Reduce Local Work
+        Reducer 3 
+            Reduce Operator Tree:
+              Extract
+                Statistics: Num rows: 46 Data size: 404 Basic stats: COMPLETE Column stats: NONE
+                File Output Operator
+                  compressed: false
+                  Statistics: Num rows: 46 Data size: 404 Basic stats: COMPLETE Column stats: NONE
+                  table:
+                      input format: org.apache.hadoop.mapred.TextInputFormat
+                      output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+                      serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+                      name: default.test_table3
 
   Stage: Stage-0
     Move Operator

Modified: hive/branches/spark/ql/src/test/results/clientpositive/spark/cross_product_check_2.q.out
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/results/clientpositive/spark/cross_product_check_2.q.out?rev=1646523&r1=1646522&r2=1646523&view=diff
==============================================================================
--- hive/branches/spark/ql/src/test/results/clientpositive/spark/cross_product_check_2.q.out (original)
+++ hive/branches/spark/ql/src/test/results/clientpositive/spark/cross_product_check_2.q.out Thu Dec 18 19:58:10 2014
@@ -96,7 +96,7 @@ STAGE PLANS:
       Processor Tree:
         ListSink
 
-Warning: Map Join MAPJOIN[16][bigTable=a] in task 'Stage-1:MAPRED' is a cross product
+Warning: Map Join MAPJOIN[15][bigTable=a] in task 'Stage-1:MAPRED' is a cross product
 PREHOOK: query: explain select * from B d1 join B d2 on d1.key = d2.key join A
 PREHOOK: type: QUERY
 POSTHOOK: query: explain select * from B d1 join B d2 on d1.key = d2.key join A
@@ -207,7 +207,7 @@ STAGE PLANS:
       Processor Tree:
         ListSink
 
-Warning: Map Join MAPJOIN[22][bigTable=a] in task 'Stage-1:MAPRED' is a cross product
+Warning: Map Join MAPJOIN[21][bigTable=a] in task 'Stage-1:MAPRED' is a cross product
 PREHOOK: query: explain select * from A join 
          (select d1.key 
           from B d1 join B d2 on d1.key = d2.key 
@@ -345,8 +345,8 @@ STAGE PLANS:
       Processor Tree:
         ListSink
 
-Warning: Map Join MAPJOIN[18][bigTable=a] in task 'Stage-1:MAPRED' is a cross product
-Warning: Map Join MAPJOIN[17][bigTable=d1] in task 'Stage-2:MAPRED' is a cross product
+Warning: Map Join MAPJOIN[17][bigTable=a] in task 'Stage-1:MAPRED' is a cross product
+Warning: Map Join MAPJOIN[18][bigTable=d1] in task 'Stage-2:MAPRED' is a cross product
 PREHOOK: query: explain select * from A join (select d1.key from B d1 join B d2 where 1 = 1 group by d1.key) od1
 PREHOOK: type: QUERY
 POSTHOOK: query: explain select * from A join (select d1.key from B d1 join B d2 where 1 = 1 group by d1.key) od1
@@ -472,7 +472,7 @@ STAGE PLANS:
       Processor Tree:
         ListSink
 
-Warning: Map Join MAPJOIN[27][bigTable=?] in task 'Stage-1:MAPRED' is a cross product
+Warning: Map Join MAPJOIN[26][bigTable=?] in task 'Stage-1:MAPRED' is a cross product
 PREHOOK: query: explain select * from 
 (select A.key from A group by key) ss join 
 (select d1.key from B d1 join B d2 on d1.key = d2.key where 1 = 1 group by d1.key) od1

Modified: hive/branches/spark/ql/src/test/results/clientpositive/spark/parquet_join.q.out
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/results/clientpositive/spark/parquet_join.q.out?rev=1646523&r1=1646522&r2=1646523&view=diff
==============================================================================
--- hive/branches/spark/ql/src/test/results/clientpositive/spark/parquet_join.q.out (original)
+++ hive/branches/spark/ql/src/test/results/clientpositive/spark/parquet_join.q.out Thu Dec 18 19:58:10 2014
@@ -319,11 +319,14 @@ STAGE PLANS:
                         0 key (type: int)
                         1 key (type: int)
                       outputColumnNames: _col1, _col7
+                      Statistics: Num rows: 1 Data size: 2 Basic stats: COMPLETE Column stats: NONE
                       Select Operator
                         expressions: _col1 (type: string), _col7 (type: string)
                         outputColumnNames: _col0, _col1
+                        Statistics: Num rows: 1 Data size: 2 Basic stats: COMPLETE Column stats: NONE
                         File Output Operator
                           compressed: false
+                          Statistics: Num rows: 1 Data size: 2 Basic stats: COMPLETE Column stats: NONE
                           table:
                               input format: org.apache.hadoop.mapred.TextInputFormat
                               output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat



Mime
View raw message