hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From sze...@apache.org
Subject svn commit: r1657741 - in /hive/branches/spark: itests/src/test/resources/testconfiguration.properties ql/src/java/org/apache/hadoop/hive/ql/parse/spark/GenSparkUtils.java ql/src/test/results/clientpositive/spark/union_view.q.out
Date Fri, 06 Feb 2015 03:32:02 GMT
Author: szehon
Date: Fri Feb  6 03:32:02 2015
New Revision: 1657741

URL: http://svn.apache.org/r1657741
Log:
HIVE-9570 : Investigate test failure on union_view.q [Spark Branch] (Chao Sun via Szehon)

Added:
    hive/branches/spark/ql/src/test/results/clientpositive/spark/union_view.q.out
Modified:
    hive/branches/spark/itests/src/test/resources/testconfiguration.properties
    hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/GenSparkUtils.java

Modified: hive/branches/spark/itests/src/test/resources/testconfiguration.properties
URL: http://svn.apache.org/viewvc/hive/branches/spark/itests/src/test/resources/testconfiguration.properties?rev=1657741&r1=1657740&r2=1657741&view=diff
==============================================================================
--- hive/branches/spark/itests/src/test/resources/testconfiguration.properties (original)
+++ hive/branches/spark/itests/src/test/resources/testconfiguration.properties Fri Feb  6
03:32:02 2015
@@ -951,6 +951,7 @@ spark.query.files=add_part_multiple.q, \
   union_remove_8.q, \
   union_remove_9.q, \
   uniquejoin.q, \
+  union_view.q, \
   varchar_join1.q, \
   vector_between_in.q, \
   vector_cast_constant.q, \

Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/GenSparkUtils.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/GenSparkUtils.java?rev=1657741&r1=1657740&r2=1657741&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/GenSparkUtils.java
(original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/GenSparkUtils.java
Fri Feb  6 03:32:02 2015
@@ -189,7 +189,15 @@ public class GenSparkUtils {
     throws SemanticException {
 
     List<Operator<?>> roots = new ArrayList<Operator<?>>();
-    roots.addAll(work.getAllRootOperators());
+
+    // For MapWork, getAllRootOperators is not suitable, since it checks
+    // getPathToAliases, and will return null if this is empty. Here we are
+    // replacing getAliasToWork, so should use that information instead.
+    if (work instanceof MapWork) {
+      roots.addAll(((MapWork) work).getAliasToWork().values());
+    } else {
+      roots.addAll(work.getAllRootOperators());
+    }
     if (work.getDummyOps() != null) {
       roots.addAll(work.getDummyOps());
     }

Added: hive/branches/spark/ql/src/test/results/clientpositive/spark/union_view.q.out
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/results/clientpositive/spark/union_view.q.out?rev=1657741&view=auto
==============================================================================
--- hive/branches/spark/ql/src/test/results/clientpositive/spark/union_view.q.out (added)
+++ hive/branches/spark/ql/src/test/results/clientpositive/spark/union_view.q.out Fri Feb
 6 03:32:02 2015
@@ -0,0 +1,700 @@
+PREHOOK: query: CREATE TABLE src_union_1 (key int, value string) PARTITIONED BY (ds string)
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@src_union_1
+POSTHOOK: query: CREATE TABLE src_union_1 (key int, value string) PARTITIONED BY (ds string)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@src_union_1
+PREHOOK: query: CREATE INDEX src_union_1_key_idx ON TABLE src_union_1(key) AS 'COMPACT' WITH
DEFERRED REBUILD
+PREHOOK: type: CREATEINDEX
+PREHOOK: Input: default@src_union_1
+POSTHOOK: query: CREATE INDEX src_union_1_key_idx ON TABLE src_union_1(key) AS 'COMPACT'
WITH DEFERRED REBUILD
+POSTHOOK: type: CREATEINDEX
+POSTHOOK: Input: default@src_union_1
+POSTHOOK: Output: default@default__src_union_1_src_union_1_key_idx__
+PREHOOK: query: CREATE TABLE src_union_2 (key int, value string) PARTITIONED BY (ds string,
part_1 string)
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@src_union_2
+POSTHOOK: query: CREATE TABLE src_union_2 (key int, value string) PARTITIONED BY (ds string,
part_1 string)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@src_union_2
+PREHOOK: query: CREATE INDEX src_union_2_key_idx ON TABLE src_union_2(key) AS 'COMPACT' WITH
DEFERRED REBUILD
+PREHOOK: type: CREATEINDEX
+PREHOOK: Input: default@src_union_2
+POSTHOOK: query: CREATE INDEX src_union_2_key_idx ON TABLE src_union_2(key) AS 'COMPACT'
WITH DEFERRED REBUILD
+POSTHOOK: type: CREATEINDEX
+POSTHOOK: Input: default@src_union_2
+POSTHOOK: Output: default@default__src_union_2_src_union_2_key_idx__
+PREHOOK: query: CREATE TABLE src_union_3(key int, value string) PARTITIONED BY (ds string,
part_1 string, part_2 string)
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@src_union_3
+POSTHOOK: query: CREATE TABLE src_union_3(key int, value string) PARTITIONED BY (ds string,
part_1 string, part_2 string)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@src_union_3
+PREHOOK: query: CREATE INDEX src_union_3_key_idx ON TABLE src_union_3(key) AS 'COMPACT' WITH
DEFERRED REBUILD
+PREHOOK: type: CREATEINDEX
+PREHOOK: Input: default@src_union_3
+POSTHOOK: query: CREATE INDEX src_union_3_key_idx ON TABLE src_union_3(key) AS 'COMPACT'
WITH DEFERRED REBUILD
+POSTHOOK: type: CREATEINDEX
+POSTHOOK: Input: default@src_union_3
+POSTHOOK: Output: default@default__src_union_3_src_union_3_key_idx__
+STAGE DEPENDENCIES:
+  Stage-0 is a root stage
+
+STAGE PLANS:
+  Stage: Stage-0
+    Fetch Operator
+      limit: -1
+      Processor Tree:
+        TableScan
+          alias: src_union_1
+          filterExpr: ((key = 86) and (ds = '1')) (type: boolean)
+          Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+          Filter Operator
+            predicate: (key = 86) (type: boolean)
+            Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats:
NONE
+            Select Operator
+              expressions: 86 (type: int), value (type: string), '1' (type: string)
+              outputColumnNames: _col0, _col1, _col2
+              Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats:
NONE
+              ListSink
+
+STAGE DEPENDENCIES:
+  Stage-0 is a root stage
+
+STAGE PLANS:
+  Stage: Stage-0
+    Fetch Operator
+      limit: -1
+      Processor Tree:
+        TableScan
+          alias: src_union_2
+          filterExpr: ((key = 86) and (ds = '2')) (type: boolean)
+          Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats:
NONE
+          Filter Operator
+            predicate: (key = 86) (type: boolean)
+            Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats:
NONE
+            Select Operator
+              expressions: 86 (type: int), value (type: string), '2' (type: string)
+              outputColumnNames: _col0, _col1, _col2
+              Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats:
NONE
+              ListSink
+
+STAGE DEPENDENCIES:
+  Stage-0 is a root stage
+
+STAGE PLANS:
+  Stage: Stage-0
+    Fetch Operator
+      limit: -1
+      Processor Tree:
+        TableScan
+          alias: src_union_3
+          filterExpr: ((key = 86) and (ds = '3')) (type: boolean)
+          Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats:
NONE
+          Filter Operator
+            predicate: (key = 86) (type: boolean)
+            Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats:
NONE
+            Select Operator
+              expressions: 86 (type: int), value (type: string), '3' (type: string)
+              outputColumnNames: _col0, _col1, _col2
+              Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats:
NONE
+              ListSink
+
+86	val_86	1
+86	val_86	2
+86	val_86	2
+86	val_86	3
+86	val_86	3
+STAGE DEPENDENCIES:
+  Stage-1 is a root stage
+  Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+  Stage: Stage-1
+    Spark
+      Edges:
+        Reducer 2 <- Map 1 (GROUP, 1)
+#### A masked pattern was here ####
+      Vertices:
+        Map 1 
+            Map Operator Tree:
+                TableScan
+                  alias: src_union_1
+                  filterExpr: (ds = '1') (type: boolean)
+                  Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column
stats: NONE
+                  Select Operator
+                    Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column
stats: NONE
+                    Group By Operator
+                      aggregations: count(1)
+                      mode: hash
+                      outputColumnNames: _col0
+                      Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats:
NONE
+                      Reduce Output Operator
+                        sort order: 
+                        Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column
stats: NONE
+                        value expressions: _col0 (type: bigint)
+        Reducer 2 
+            Reduce Operator Tree:
+              Group By Operator
+                aggregations: count(VALUE._col0)
+                mode: mergepartial
+                outputColumnNames: _col0
+                Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats:
NONE
+                File Output Operator
+                  compressed: false
+                  Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats:
NONE
+                  table:
+                      input format: org.apache.hadoop.mapred.TextInputFormat
+                      output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+                      serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+  Stage: Stage-0
+    Fetch Operator
+      limit: -1
+      Processor Tree:
+        ListSink
+
+STAGE DEPENDENCIES:
+  Stage-1 is a root stage
+  Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+  Stage: Stage-1
+    Spark
+      Edges:
+        Reducer 2 <- Map 1 (GROUP, 1)
+#### A masked pattern was here ####
+      Vertices:
+        Map 1 
+            Map Operator Tree:
+                TableScan
+                  alias: src_union_2
+                  filterExpr: (ds = '2') (type: boolean)
+                  Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column
stats: NONE
+                  Select Operator
+                    Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column
stats: NONE
+                    Group By Operator
+                      aggregations: count(1)
+                      mode: hash
+                      outputColumnNames: _col0
+                      Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats:
NONE
+                      Reduce Output Operator
+                        sort order: 
+                        Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column
stats: NONE
+                        value expressions: _col0 (type: bigint)
+        Reducer 2 
+            Reduce Operator Tree:
+              Group By Operator
+                aggregations: count(VALUE._col0)
+                mode: mergepartial
+                outputColumnNames: _col0
+                Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats:
NONE
+                File Output Operator
+                  compressed: false
+                  Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats:
NONE
+                  table:
+                      input format: org.apache.hadoop.mapred.TextInputFormat
+                      output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+                      serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+  Stage: Stage-0
+    Fetch Operator
+      limit: -1
+      Processor Tree:
+        ListSink
+
+STAGE DEPENDENCIES:
+  Stage-1 is a root stage
+  Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+  Stage: Stage-1
+    Spark
+      Edges:
+        Reducer 2 <- Map 1 (GROUP, 1)
+#### A masked pattern was here ####
+      Vertices:
+        Map 1 
+            Map Operator Tree:
+                TableScan
+                  alias: src_union_3
+                  filterExpr: (ds = '3') (type: boolean)
+                  Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column
stats: NONE
+                  Select Operator
+                    Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column
stats: NONE
+                    Group By Operator
+                      aggregations: count(1)
+                      mode: hash
+                      outputColumnNames: _col0
+                      Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats:
NONE
+                      Reduce Output Operator
+                        sort order: 
+                        Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column
stats: NONE
+                        value expressions: _col0 (type: bigint)
+        Reducer 2 
+            Reduce Operator Tree:
+              Group By Operator
+                aggregations: count(VALUE._col0)
+                mode: mergepartial
+                outputColumnNames: _col0
+                Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats:
NONE
+                File Output Operator
+                  compressed: false
+                  Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats:
NONE
+                  table:
+                      input format: org.apache.hadoop.mapred.TextInputFormat
+                      output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+                      serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+  Stage: Stage-0
+    Fetch Operator
+      limit: -1
+      Processor Tree:
+        ListSink
+
+500
+1000
+1000
+STAGE DEPENDENCIES:
+  Stage-1 is a root stage
+  Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+  Stage: Stage-1
+    Spark
+#### A masked pattern was here ####
+      Vertices:
+        Map 1 
+            Map Operator Tree:
+                TableScan
+                  alias: src_union_1
+                  filterExpr: ((key = 86) and (ds = '1')) (type: boolean)
+                  Filter Operator
+                    predicate: (key = 86) (type: boolean)
+                    Select Operator
+                      expressions: value (type: string)
+                      outputColumnNames: _col1
+                      Select Operator
+                        expressions: 86 (type: int), _col1 (type: string), '1' (type: string)
+                        outputColumnNames: _col0, _col1, _col2
+                        File Output Operator
+                          compressed: false
+                          table:
+                              input format: org.apache.hadoop.mapred.TextInputFormat
+                              output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+                              serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+        Map 2 
+        Map 3 
+
+  Stage: Stage-0
+    Fetch Operator
+      limit: -1
+      Processor Tree:
+        ListSink
+
+STAGE DEPENDENCIES:
+  Stage-1 is a root stage
+  Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+  Stage: Stage-1
+    Spark
+#### A masked pattern was here ####
+      Vertices:
+        Map 1 
+        Map 2 
+            Map Operator Tree:
+                TableScan
+                  alias: src_union_2
+                  filterExpr: ((key = 86) and (ds = '2')) (type: boolean)
+                  Filter Operator
+                    predicate: (key = 86) (type: boolean)
+                    Select Operator
+                      expressions: value (type: string)
+                      outputColumnNames: _col1
+                      Select Operator
+                        expressions: 86 (type: int), _col1 (type: string), '2' (type: string)
+                        outputColumnNames: _col0, _col1, _col2
+                        File Output Operator
+                          compressed: false
+                          table:
+                              input format: org.apache.hadoop.mapred.TextInputFormat
+                              output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+                              serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+        Map 3 
+
+  Stage: Stage-0
+    Fetch Operator
+      limit: -1
+      Processor Tree:
+        ListSink
+
+STAGE DEPENDENCIES:
+  Stage-1 is a root stage
+  Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+  Stage: Stage-1
+    Spark
+#### A masked pattern was here ####
+      Vertices:
+        Map 1 
+        Map 2 
+        Map 3 
+            Map Operator Tree:
+                TableScan
+                  alias: src_union_3
+                  filterExpr: ((key = 86) and (ds = '3')) (type: boolean)
+                  Filter Operator
+                    predicate: (key = 86) (type: boolean)
+                    Select Operator
+                      expressions: value (type: string)
+                      outputColumnNames: _col1
+                      Select Operator
+                        expressions: 86 (type: int), _col1 (type: string), '3' (type: string)
+                        outputColumnNames: _col0, _col1, _col2
+                        File Output Operator
+                          compressed: false
+                          table:
+                              input format: org.apache.hadoop.mapred.TextInputFormat
+                              output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+                              serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+  Stage: Stage-0
+    Fetch Operator
+      limit: -1
+      Processor Tree:
+        ListSink
+
+STAGE DEPENDENCIES:
+  Stage-1 is a root stage
+  Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+  Stage: Stage-1
+    Spark
+      Edges:
+        Reducer 2 <- Map 1 (SORT, 1), Map 3 (SORT, 1), Map 4 (SORT, 1)
+#### A masked pattern was here ####
+      Vertices:
+        Map 1 
+            Map Operator Tree:
+                TableScan
+                  alias: src_union_1
+                  filterExpr: ((key = 86) and ds is not null) (type: boolean)
+                  Filter Operator
+                    predicate: (key = 86) (type: boolean)
+                    Select Operator
+                      expressions: value (type: string), ds (type: string)
+                      outputColumnNames: _col1, _col2
+                      Select Operator
+                        expressions: _col1 (type: string), _col2 (type: string)
+                        outputColumnNames: _col1, _col2
+                        Reduce Output Operator
+                          key expressions: _col2 (type: string)
+                          sort order: +
+                          value expressions: _col1 (type: string)
+        Map 3 
+            Map Operator Tree:
+                TableScan
+                  alias: src_union_2
+                  filterExpr: ((key = 86) and ds is not null) (type: boolean)
+                  Filter Operator
+                    predicate: (key = 86) (type: boolean)
+                    Select Operator
+                      expressions: value (type: string), ds (type: string)
+                      outputColumnNames: _col1, _col2
+                      Select Operator
+                        expressions: _col1 (type: string), _col2 (type: string)
+                        outputColumnNames: _col1, _col2
+                        Reduce Output Operator
+                          key expressions: _col2 (type: string)
+                          sort order: +
+                          value expressions: _col1 (type: string)
+        Map 4 
+            Map Operator Tree:
+                TableScan
+                  alias: src_union_3
+                  filterExpr: ((key = 86) and ds is not null) (type: boolean)
+                  Filter Operator
+                    predicate: (key = 86) (type: boolean)
+                    Select Operator
+                      expressions: value (type: string), ds (type: string)
+                      outputColumnNames: _col1, _col2
+                      Select Operator
+                        expressions: _col1 (type: string), _col2 (type: string)
+                        outputColumnNames: _col1, _col2
+                        Reduce Output Operator
+                          key expressions: _col2 (type: string)
+                          sort order: +
+                          value expressions: _col1 (type: string)
+        Reducer 2 
+            Reduce Operator Tree:
+              Select Operator
+                expressions: 86 (type: int), VALUE._col1 (type: string), KEY.reducesinkkey0
(type: string)
+                outputColumnNames: _col0, _col1, _col2
+                Statistics: Num rows: 1250 Data size: 13280 Basic stats: COMPLETE Column
stats: NONE
+                File Output Operator
+                  compressed: false
+                  Statistics: Num rows: 1250 Data size: 13280 Basic stats: COMPLETE Column
stats: NONE
+                  table:
+                      input format: org.apache.hadoop.mapred.TextInputFormat
+                      output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+                      serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+  Stage: Stage-0
+    Fetch Operator
+      limit: -1
+      Processor Tree:
+        ListSink
+
+86	val_86	1
+86	val_86	2
+86	val_86	2
+86	val_86	3
+86	val_86	3
+86	val_86	1
+86	val_86	2
+86	val_86	2
+86	val_86	3
+86	val_86	3
+STAGE DEPENDENCIES:
+  Stage-1 is a root stage
+  Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+  Stage: Stage-1
+    Spark
+      Edges:
+        Reducer 2 <- Map 1 (GROUP, 1), Map 3 (GROUP, 1), Map 4 (GROUP, 1)
+#### A masked pattern was here ####
+      Vertices:
+        Map 1 
+            Map Operator Tree:
+                TableScan
+                  alias: src_union_1
+                  filterExpr: (ds = '1') (type: boolean)
+                  Select Operator
+                    Select Operator
+                      Group By Operator
+                        aggregations: count(1)
+                        mode: hash
+                        outputColumnNames: _col0
+                        Reduce Output Operator
+                          sort order: 
+                          value expressions: _col0 (type: bigint)
+        Map 3 
+        Map 4 
+        Reducer 2 
+            Reduce Operator Tree:
+              Group By Operator
+                aggregations: count(VALUE._col0)
+                mode: mergepartial
+                outputColumnNames: _col0
+                Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats:
NONE
+                File Output Operator
+                  compressed: false
+                  Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats:
NONE
+                  table:
+                      input format: org.apache.hadoop.mapred.TextInputFormat
+                      output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+                      serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+  Stage: Stage-0
+    Fetch Operator
+      limit: -1
+      Processor Tree:
+        ListSink
+
+STAGE DEPENDENCIES:
+  Stage-1 is a root stage
+  Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+  Stage: Stage-1
+    Spark
+      Edges:
+        Reducer 2 <- Map 1 (GROUP, 1), Map 3 (GROUP, 1), Map 4 (GROUP, 1)
+#### A masked pattern was here ####
+      Vertices:
+        Map 1 
+        Map 3 
+            Map Operator Tree:
+                TableScan
+                  alias: src_union_2
+                  filterExpr: (ds = '2') (type: boolean)
+                  Select Operator
+                    Select Operator
+                      Group By Operator
+                        aggregations: count(1)
+                        mode: hash
+                        outputColumnNames: _col0
+                        Reduce Output Operator
+                          sort order: 
+                          value expressions: _col0 (type: bigint)
+        Map 4 
+        Reducer 2 
+            Reduce Operator Tree:
+              Group By Operator
+                aggregations: count(VALUE._col0)
+                mode: mergepartial
+                outputColumnNames: _col0
+                Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats:
NONE
+                File Output Operator
+                  compressed: false
+                  Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats:
NONE
+                  table:
+                      input format: org.apache.hadoop.mapred.TextInputFormat
+                      output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+                      serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+  Stage: Stage-0
+    Fetch Operator
+      limit: -1
+      Processor Tree:
+        ListSink
+
+STAGE DEPENDENCIES:
+  Stage-1 is a root stage
+  Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+  Stage: Stage-1
+    Spark
+      Edges:
+        Reducer 2 <- Map 1 (GROUP, 1), Map 3 (GROUP, 1), Map 4 (GROUP, 1)
+#### A masked pattern was here ####
+      Vertices:
+        Map 1 
+        Map 3 
+        Map 4 
+            Map Operator Tree:
+                TableScan
+                  alias: src_union_3
+                  filterExpr: (ds = '3') (type: boolean)
+                  Select Operator
+                    Select Operator
+                      Group By Operator
+                        aggregations: count(1)
+                        mode: hash
+                        outputColumnNames: _col0
+                        Reduce Output Operator
+                          sort order: 
+                          value expressions: _col0 (type: bigint)
+        Reducer 2 
+            Reduce Operator Tree:
+              Group By Operator
+                aggregations: count(VALUE._col0)
+                mode: mergepartial
+                outputColumnNames: _col0
+                Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats:
NONE
+                File Output Operator
+                  compressed: false
+                  Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats:
NONE
+                  table:
+                      input format: org.apache.hadoop.mapred.TextInputFormat
+                      output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+                      serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+  Stage: Stage-0
+    Fetch Operator
+      limit: -1
+      Processor Tree:
+        ListSink
+
+500
+1000
+1000
+STAGE DEPENDENCIES:
+  Stage-1 is a root stage
+  Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+  Stage: Stage-1
+    Spark
+#### A masked pattern was here ####
+      Vertices:
+        Map 1 
+        Map 2 
+        Map 3 
+            Map Operator Tree:
+                TableScan
+                  alias: src_union_3
+                  filterExpr: ((key = 86) and (ds = '4')) (type: boolean)
+                  Filter Operator
+                    predicate: (key = 86) (type: boolean)
+                    Select Operator
+                      expressions: value (type: string)
+                      outputColumnNames: _col1
+                      Select Operator
+                        expressions: 86 (type: int), _col1 (type: string), '4' (type: string)
+                        outputColumnNames: _col0, _col1, _col2
+                        File Output Operator
+                          compressed: false
+                          table:
+                              input format: org.apache.hadoop.mapred.TextInputFormat
+                              output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+                              serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+  Stage: Stage-0
+    Fetch Operator
+      limit: -1
+      Processor Tree:
+        ListSink
+
+86	val_86	4
+STAGE DEPENDENCIES:
+  Stage-1 is a root stage
+  Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+  Stage: Stage-1
+    Spark
+      Edges:
+        Reducer 2 <- Map 1 (GROUP, 1), Map 3 (GROUP, 1), Map 4 (GROUP, 1)
+#### A masked pattern was here ####
+      Vertices:
+        Map 1 
+        Map 3 
+        Map 4 
+            Map Operator Tree:
+                TableScan
+                  alias: src_union_3
+                  filterExpr: (ds = '4') (type: boolean)
+                  Select Operator
+                    Select Operator
+                      Group By Operator
+                        aggregations: count(1)
+                        mode: hash
+                        outputColumnNames: _col0
+                        Reduce Output Operator
+                          sort order: 
+                          value expressions: _col0 (type: bigint)
+        Reducer 2 
+            Reduce Operator Tree:
+              Group By Operator
+                aggregations: count(VALUE._col0)
+                mode: mergepartial
+                outputColumnNames: _col0
+                Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats:
NONE
+                File Output Operator
+                  compressed: false
+                  Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats:
NONE
+                  table:
+                      input format: org.apache.hadoop.mapred.TextInputFormat
+                      output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+                      serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+  Stage: Stage-0
+    Fetch Operator
+      limit: -1
+      Processor Tree:
+        ListSink
+
+500



Mime
View raw message