hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From stak...@apache.org
Subject [01/35] hive git commit: HIVE-17543: Enable PerfCliDriver for HoS (Sahil Takiar, reviewed by Peter Vary)
Date Thu, 28 Sep 2017 18:12:17 GMT
Repository: hive
Updated Branches:
  refs/heads/master 365c03107 -> bd371246e


http://git-wip-us.apache.org/repos/asf/hive/blob/bd371246/ql/src/test/results/clientpositive/perf/spark/query97.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/perf/spark/query97.q.out b/ql/src/test/results/clientpositive/perf/spark/query97.q.out
new file mode 100644
index 0000000..14a2aed
--- /dev/null
+++ b/ql/src/test/results/clientpositive/perf/spark/query97.q.out
@@ -0,0 +1,250 @@
+PREHOOK: query: explain
+with ssci as (
+select ss_customer_sk customer_sk
+      ,ss_item_sk item_sk
+from store_sales,date_dim
+where ss_sold_date_sk = d_date_sk
+  and d_month_seq between 1212 and 1212 + 11
+group by ss_customer_sk
+        ,ss_item_sk),
+csci as(
+ select cs_bill_customer_sk customer_sk
+      ,cs_item_sk item_sk
+from catalog_sales,date_dim
+where cs_sold_date_sk = d_date_sk
+  and d_month_seq between 1212 and 1212 + 11
+group by cs_bill_customer_sk
+        ,cs_item_sk)
+ select  sum(case when ssci.customer_sk is not null and csci.customer_sk is null then 1 else
0 end) store_only
+      ,sum(case when ssci.customer_sk is null and csci.customer_sk is not null then 1 else
0 end) catalog_only
+      ,sum(case when ssci.customer_sk is not null and csci.customer_sk is not null then 1
else 0 end) store_and_catalog
+from ssci full outer join csci on (ssci.customer_sk=csci.customer_sk
+                               and ssci.item_sk = csci.item_sk)
+limit 100
+PREHOOK: type: QUERY
+POSTHOOK: query: explain
+with ssci as (
+select ss_customer_sk customer_sk
+      ,ss_item_sk item_sk
+from store_sales,date_dim
+where ss_sold_date_sk = d_date_sk
+  and d_month_seq between 1212 and 1212 + 11
+group by ss_customer_sk
+        ,ss_item_sk),
+csci as(
+ select cs_bill_customer_sk customer_sk
+      ,cs_item_sk item_sk
+from catalog_sales,date_dim
+where cs_sold_date_sk = d_date_sk
+  and d_month_seq between 1212 and 1212 + 11
+group by cs_bill_customer_sk
+        ,cs_item_sk)
+ select  sum(case when ssci.customer_sk is not null and csci.customer_sk is null then 1 else
0 end) store_only
+      ,sum(case when ssci.customer_sk is null and csci.customer_sk is not null then 1 else
0 end) catalog_only
+      ,sum(case when ssci.customer_sk is not null and csci.customer_sk is not null then 1
else 0 end) store_and_catalog
+from ssci full outer join csci on (ssci.customer_sk=csci.customer_sk
+                               and ssci.item_sk = csci.item_sk)
+limit 100
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+  Stage-2 is a root stage
+  Stage-3 depends on stages: Stage-2
+  Stage-1 depends on stages: Stage-3
+  Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+  Stage: Stage-2
+    Spark
+#### A masked pattern was here ####
+      Vertices:
+        Map 5 
+            Map Operator Tree:
+                TableScan
+                  alias: date_dim
+                  Statistics: Num rows: 73049 Data size: 81741831 Basic stats: COMPLETE Column
stats: NONE
+                  Filter Operator
+                    predicate: (d_date_sk is not null and d_month_seq BETWEEN 1212 AND 1223)
(type: boolean)
+                    Statistics: Num rows: 8116 Data size: 9081804 Basic stats: COMPLETE Column
stats: NONE
+                    Select Operator
+                      expressions: d_date_sk (type: int)
+                      outputColumnNames: _col0
+                      Statistics: Num rows: 8116 Data size: 9081804 Basic stats: COMPLETE
Column stats: NONE
+                      Spark HashTable Sink Operator
+                        keys:
+                          0 _col0 (type: int)
+                          1 _col0 (type: int)
+            Local Work:
+              Map Reduce Local Work
+
+  Stage: Stage-3
+    Spark
+#### A masked pattern was here ####
+      Vertices:
+        Map 8 
+            Map Operator Tree:
+                TableScan
+                  alias: date_dim
+                  Statistics: Num rows: 73049 Data size: 81741831 Basic stats: COMPLETE Column
stats: NONE
+                  Filter Operator
+                    predicate: (d_date_sk is not null and d_month_seq BETWEEN 1212 AND 1223)
(type: boolean)
+                    Statistics: Num rows: 8116 Data size: 9081804 Basic stats: COMPLETE Column
stats: NONE
+                    Select Operator
+                      expressions: d_date_sk (type: int)
+                      outputColumnNames: _col0
+                      Statistics: Num rows: 8116 Data size: 9081804 Basic stats: COMPLETE
Column stats: NONE
+                      Spark HashTable Sink Operator
+                        keys:
+                          0 _col0 (type: int)
+                          1 _col0 (type: int)
+            Local Work:
+              Map Reduce Local Work
+
+  Stage: Stage-1
+    Spark
+      Edges:
+        Reducer 2 <- Map 1 (GROUP, 437)
+        Reducer 3 <- Reducer 2 (PARTITION-LEVEL SORT, 386), Reducer 7 (PARTITION-LEVEL
SORT, 386)
+        Reducer 4 <- Reducer 3 (GROUP, 1)
+        Reducer 7 <- Map 6 (GROUP, 336)
+#### A masked pattern was here ####
+      Vertices:
+        Map 1 
+            Map Operator Tree:
+                TableScan
+                  alias: store_sales
+                  Statistics: Num rows: 575995635 Data size: 50814502088 Basic stats: COMPLETE
Column stats: NONE
+                  Filter Operator
+                    predicate: ss_sold_date_sk is not null (type: boolean)
+                    Statistics: Num rows: 575995635 Data size: 50814502088 Basic stats: COMPLETE
Column stats: NONE
+                    Select Operator
+                      expressions: ss_sold_date_sk (type: int), ss_item_sk (type: int), ss_customer_sk
(type: int)
+                      outputColumnNames: _col0, _col1, _col2
+                      Statistics: Num rows: 575995635 Data size: 50814502088 Basic stats:
COMPLETE Column stats: NONE
+                      Map Join Operator
+                        condition map:
+                             Inner Join 0 to 1
+                        keys:
+                          0 _col0 (type: int)
+                          1 _col0 (type: int)
+                        outputColumnNames: _col1, _col2
+                        input vertices:
+                          1 Map 5
+                        Statistics: Num rows: 633595212 Data size: 55895953508 Basic stats:
COMPLETE Column stats: NONE
+                        Group By Operator
+                          keys: _col2 (type: int), _col1 (type: int)
+                          mode: hash
+                          outputColumnNames: _col0, _col1
+                          Statistics: Num rows: 633595212 Data size: 55895953508 Basic stats:
COMPLETE Column stats: NONE
+                          Reduce Output Operator
+                            key expressions: _col0 (type: int), _col1 (type: int)
+                            sort order: ++
+                            Map-reduce partition columns: _col0 (type: int), _col1 (type:
int)
+                            Statistics: Num rows: 633595212 Data size: 55895953508 Basic
stats: COMPLETE Column stats: NONE
+            Local Work:
+              Map Reduce Local Work
+        Map 6 
+            Map Operator Tree:
+                TableScan
+                  alias: catalog_sales
+                  Statistics: Num rows: 287989836 Data size: 38999608952 Basic stats: COMPLETE
Column stats: NONE
+                  Filter Operator
+                    predicate: cs_sold_date_sk is not null (type: boolean)
+                    Statistics: Num rows: 287989836 Data size: 38999608952 Basic stats: COMPLETE
Column stats: NONE
+                    Select Operator
+                      expressions: cs_sold_date_sk (type: int), cs_bill_customer_sk (type:
int), cs_item_sk (type: int)
+                      outputColumnNames: _col0, _col1, _col2
+                      Statistics: Num rows: 287989836 Data size: 38999608952 Basic stats:
COMPLETE Column stats: NONE
+                      Map Join Operator
+                        condition map:
+                             Inner Join 0 to 1
+                        keys:
+                          0 _col0 (type: int)
+                          1 _col0 (type: int)
+                        outputColumnNames: _col1, _col2
+                        input vertices:
+                          1 Map 8
+                        Statistics: Num rows: 316788826 Data size: 42899570777 Basic stats:
COMPLETE Column stats: NONE
+                        Group By Operator
+                          keys: _col1 (type: int), _col2 (type: int)
+                          mode: hash
+                          outputColumnNames: _col0, _col1
+                          Statistics: Num rows: 316788826 Data size: 42899570777 Basic stats:
COMPLETE Column stats: NONE
+                          Reduce Output Operator
+                            key expressions: _col0 (type: int), _col1 (type: int)
+                            sort order: ++
+                            Map-reduce partition columns: _col0 (type: int), _col1 (type:
int)
+                            Statistics: Num rows: 316788826 Data size: 42899570777 Basic
stats: COMPLETE Column stats: NONE
+            Local Work:
+              Map Reduce Local Work
+        Reducer 2 
+            Reduce Operator Tree:
+              Group By Operator
+                keys: KEY._col0 (type: int), KEY._col1 (type: int)
+                mode: mergepartial
+                outputColumnNames: _col0, _col1
+                Statistics: Num rows: 316797606 Data size: 27947976754 Basic stats: COMPLETE
Column stats: NONE
+                Reduce Output Operator
+                  key expressions: _col0 (type: int), _col1 (type: int)
+                  sort order: ++
+                  Map-reduce partition columns: _col0 (type: int), _col1 (type: int)
+                  Statistics: Num rows: 316797606 Data size: 27947976754 Basic stats: COMPLETE
Column stats: NONE
+        Reducer 3 
+            Reduce Operator Tree:
+              Join Operator
+                condition map:
+                     Outer Join 0 to 1
+                keys:
+                  0 _col0 (type: int), _col1 (type: int)
+                  1 _col0 (type: int), _col1 (type: int)
+                outputColumnNames: _col0, _col2
+                Statistics: Num rows: 348477374 Data size: 30742775095 Basic stats: COMPLETE
Column stats: NONE
+                Select Operator
+                  expressions: CASE WHEN ((_col0 is not null and _col2 is null)) THEN (1)
ELSE (0) END (type: int), CASE WHEN ((_col0 is null and _col2 is not null)) THEN (1) ELSE
(0) END (type: int), CASE WHEN ((_col0 is not null and _col2 is not null)) THEN (1) ELSE (0)
END (type: int)
+                  outputColumnNames: _col0, _col1, _col2
+                  Statistics: Num rows: 348477374 Data size: 30742775095 Basic stats: COMPLETE
Column stats: NONE
+                  Group By Operator
+                    aggregations: sum(_col0), sum(_col1), sum(_col2)
+                    mode: hash
+                    outputColumnNames: _col0, _col1, _col2
+                    Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats:
NONE
+                    Reduce Output Operator
+                      sort order: 
+                      Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column
stats: NONE
+                      TopN Hash Memory Usage: 0.1
+                      value expressions: _col0 (type: bigint), _col1 (type: bigint), _col2
(type: bigint)
+        Reducer 4 
+            Reduce Operator Tree:
+              Group By Operator
+                aggregations: sum(VALUE._col0), sum(VALUE._col1), sum(VALUE._col2)
+                mode: mergepartial
+                outputColumnNames: _col0, _col1, _col2
+                Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats:
NONE
+                Limit
+                  Number of rows: 100
+                  Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats:
NONE
+                  File Output Operator
+                    compressed: false
+                    Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats:
NONE
+                    table:
+                        input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+                        output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+                        serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+        Reducer 7 
+            Reduce Operator Tree:
+              Group By Operator
+                keys: KEY._col0 (type: int), KEY._col1 (type: int)
+                mode: mergepartial
+                outputColumnNames: _col0, _col1
+                Statistics: Num rows: 158394413 Data size: 21449785388 Basic stats: COMPLETE
Column stats: NONE
+                Reduce Output Operator
+                  key expressions: _col0 (type: int), _col1 (type: int)
+                  sort order: ++
+                  Map-reduce partition columns: _col0 (type: int), _col1 (type: int)
+                  Statistics: Num rows: 158394413 Data size: 21449785388 Basic stats: COMPLETE
Column stats: NONE
+
+  Stage: Stage-0
+    Fetch Operator
+      limit: 100
+      Processor Tree:
+        ListSink
+

http://git-wip-us.apache.org/repos/asf/hive/blob/bd371246/ql/src/test/results/clientpositive/perf/spark/query98.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/perf/spark/query98.q.out b/ql/src/test/results/clientpositive/perf/spark/query98.q.out
new file mode 100644
index 0000000..24d7de8
--- /dev/null
+++ b/ql/src/test/results/clientpositive/perf/spark/query98.q.out
@@ -0,0 +1,243 @@
+PREHOOK: query: explain
+select i_item_desc 
+      ,i_category 
+      ,i_class 
+      ,i_current_price
+      ,sum(ss_ext_sales_price) as itemrevenue 
+      ,sum(ss_ext_sales_price)*100/sum(sum(ss_ext_sales_price)) over
+          (partition by i_class) as revenueratio
+from	
+	store_sales
+    	,item 
+    	,date_dim
+where 
+	ss_item_sk = i_item_sk 
+  	and i_category in ('Jewelry', 'Sports', 'Books')
+  	and ss_sold_date_sk = d_date_sk
+	and d_date between cast('2001-01-12' as date) 
+				and (cast('2001-01-12' as date) + 30 days)
+group by 
+	i_item_id
+        ,i_item_desc 
+        ,i_category
+        ,i_class
+        ,i_current_price
+order by 
+	i_category
+        ,i_class
+        ,i_item_id
+        ,i_item_desc
+        ,revenueratio
+PREHOOK: type: QUERY
+POSTHOOK: query: explain
+select i_item_desc 
+      ,i_category 
+      ,i_class 
+      ,i_current_price
+      ,sum(ss_ext_sales_price) as itemrevenue 
+      ,sum(ss_ext_sales_price)*100/sum(sum(ss_ext_sales_price)) over
+          (partition by i_class) as revenueratio
+from	
+	store_sales
+    	,item 
+    	,date_dim
+where 
+	ss_item_sk = i_item_sk 
+  	and i_category in ('Jewelry', 'Sports', 'Books')
+  	and ss_sold_date_sk = d_date_sk
+	and d_date between cast('2001-01-12' as date) 
+				and (cast('2001-01-12' as date) + 30 days)
+group by 
+	i_item_id
+        ,i_item_desc 
+        ,i_category
+        ,i_class
+        ,i_current_price
+order by 
+	i_category
+        ,i_class
+        ,i_item_id
+        ,i_item_desc
+        ,revenueratio
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+  Stage-2 is a root stage
+  Stage-1 depends on stages: Stage-2
+  Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+  Stage: Stage-2
+    Spark
+#### A masked pattern was here ####
+      Vertices:
+        Map 6 
+            Map Operator Tree:
+                TableScan
+                  alias: date_dim
+                  Statistics: Num rows: 73049 Data size: 81741831 Basic stats: COMPLETE Column
stats: NONE
+                  Filter Operator
+                    predicate: (CAST( d_date AS TIMESTAMP) BETWEEN 2001-01-12 00:00:00.0
AND 2001-02-11 00:00:00.0 and d_date_sk is not null) (type: boolean)
+                    Statistics: Num rows: 8116 Data size: 9081804 Basic stats: COMPLETE Column
stats: NONE
+                    Select Operator
+                      expressions: d_date_sk (type: int)
+                      outputColumnNames: _col0
+                      Statistics: Num rows: 8116 Data size: 9081804 Basic stats: COMPLETE
Column stats: NONE
+                      Spark HashTable Sink Operator
+                        keys:
+                          0 _col0 (type: int)
+                          1 _col0 (type: int)
+            Local Work:
+              Map Reduce Local Work
+
+  Stage: Stage-1
+    Spark
+      Edges:
+        Reducer 2 <- Map 1 (PARTITION-LEVEL SORT, 440), Map 7 (PARTITION-LEVEL SORT, 440)
+        Reducer 3 <- Reducer 2 (GROUP, 481)
+        Reducer 4 <- Reducer 3 (PARTITION-LEVEL SORT, 241)
+        Reducer 5 <- Reducer 4 (SORT, 1)
+#### A masked pattern was here ####
+      Vertices:
+        Map 1 
+            Map Operator Tree:
+                TableScan
+                  alias: store_sales
+                  Statistics: Num rows: 575995635 Data size: 50814502088 Basic stats: COMPLETE
Column stats: NONE
+                  Filter Operator
+                    predicate: (ss_item_sk is not null and ss_sold_date_sk is not null) (type:
boolean)
+                    Statistics: Num rows: 575995635 Data size: 50814502088 Basic stats: COMPLETE
Column stats: NONE
+                    Select Operator
+                      expressions: ss_sold_date_sk (type: int), ss_item_sk (type: int), ss_ext_sales_price
(type: decimal(7,2))
+                      outputColumnNames: _col0, _col1, _col2
+                      Statistics: Num rows: 575995635 Data size: 50814502088 Basic stats:
COMPLETE Column stats: NONE
+                      Map Join Operator
+                        condition map:
+                             Inner Join 0 to 1
+                        keys:
+                          0 _col0 (type: int)
+                          1 _col0 (type: int)
+                        outputColumnNames: _col1, _col2
+                        input vertices:
+                          1 Map 6
+                        Statistics: Num rows: 633595212 Data size: 55895953508 Basic stats:
COMPLETE Column stats: NONE
+                        Reduce Output Operator
+                          key expressions: _col1 (type: int)
+                          sort order: +
+                          Map-reduce partition columns: _col1 (type: int)
+                          Statistics: Num rows: 633595212 Data size: 55895953508 Basic stats:
COMPLETE Column stats: NONE
+                          value expressions: _col2 (type: decimal(7,2))
+            Local Work:
+              Map Reduce Local Work
+        Map 7 
+            Map Operator Tree:
+                TableScan
+                  alias: item
+                  Statistics: Num rows: 462000 Data size: 663560457 Basic stats: COMPLETE
Column stats: NONE
+                  Filter Operator
+                    predicate: ((i_category) IN ('Jewelry', 'Sports', 'Books') and i_item_sk
is not null) (type: boolean)
+                    Statistics: Num rows: 231000 Data size: 331780228 Basic stats: COMPLETE
Column stats: NONE
+                    Select Operator
+                      expressions: i_item_sk (type: int), i_item_id (type: string), i_item_desc
(type: string), i_current_price (type: decimal(7,2)), i_class (type: string), i_category (type:
string)
+                      outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
+                      Statistics: Num rows: 231000 Data size: 331780228 Basic stats: COMPLETE
Column stats: NONE
+                      Reduce Output Operator
+                        key expressions: _col0 (type: int)
+                        sort order: +
+                        Map-reduce partition columns: _col0 (type: int)
+                        Statistics: Num rows: 231000 Data size: 331780228 Basic stats: COMPLETE
Column stats: NONE
+                        value expressions: _col1 (type: string), _col2 (type: string), _col3
(type: decimal(7,2)), _col4 (type: string), _col5 (type: string)
+        Reducer 2 
+            Reduce Operator Tree:
+              Join Operator
+                condition map:
+                     Inner Join 0 to 1
+                keys:
+                  0 _col1 (type: int)
+                  1 _col0 (type: int)
+                outputColumnNames: _col2, _col6, _col7, _col8, _col9, _col10
+                Statistics: Num rows: 696954748 Data size: 61485550191 Basic stats: COMPLETE
Column stats: NONE
+                Group By Operator
+                  aggregations: sum(_col2)
+                  keys: _col10 (type: string), _col9 (type: string), _col6 (type: string),
_col7 (type: string), _col8 (type: decimal(7,2))
+                  mode: hash
+                  outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
+                  Statistics: Num rows: 696954748 Data size: 61485550191 Basic stats: COMPLETE
Column stats: NONE
+                  Reduce Output Operator
+                    key expressions: _col0 (type: string), _col1 (type: string), _col2 (type:
string), _col3 (type: string), _col4 (type: decimal(7,2))
+                    sort order: +++++
+                    Map-reduce partition columns: _col0 (type: string), _col1 (type: string),
_col2 (type: string), _col3 (type: string), _col4 (type: decimal(7,2))
+                    Statistics: Num rows: 696954748 Data size: 61485550191 Basic stats: COMPLETE
Column stats: NONE
+                    value expressions: _col5 (type: decimal(17,2))
+        Reducer 3 
+            Reduce Operator Tree:
+              Group By Operator
+                aggregations: sum(VALUE._col0)
+                keys: KEY._col0 (type: string), KEY._col1 (type: string), KEY._col2 (type:
string), KEY._col3 (type: string), KEY._col4 (type: decimal(7,2))
+                mode: mergepartial
+                outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
+                Statistics: Num rows: 348477374 Data size: 30742775095 Basic stats: COMPLETE
Column stats: NONE
+                Select Operator
+                  expressions: _col2 (type: string), _col3 (type: string), _col0 (type: string),
_col1 (type: string), _col4 (type: decimal(7,2)), _col5 (type: decimal(17,2))
+                  outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
+                  Statistics: Num rows: 348477374 Data size: 30742775095 Basic stats: COMPLETE
Column stats: NONE
+                  Reduce Output Operator
+                    key expressions: _col3 (type: string)
+                    sort order: +
+                    Map-reduce partition columns: _col3 (type: string)
+                    Statistics: Num rows: 348477374 Data size: 30742775095 Basic stats: COMPLETE
Column stats: NONE
+                    value expressions: _col0 (type: string), _col1 (type: string), _col2
(type: string), _col4 (type: decimal(7,2)), _col5 (type: decimal(17,2))
+        Reducer 4 
+            Reduce Operator Tree:
+              Select Operator
+                expressions: VALUE._col0 (type: string), VALUE._col1 (type: string), VALUE._col2
(type: string), KEY.reducesinkkey0 (type: string), VALUE._col3 (type: decimal(7,2)), VALUE._col4
(type: decimal(17,2))
+                outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
+                Statistics: Num rows: 348477374 Data size: 30742775095 Basic stats: COMPLETE
Column stats: NONE
+                PTF Operator
+                  Function definitions:
+                      Input definition
+                        input alias: ptf_0
+                        output shape: _col0: string, _col1: string, _col2: string, _col3:
string, _col4: decimal(7,2), _col5: decimal(17,2)
+                        type: WINDOWING
+                      Windowing table definition
+                        input alias: ptf_1
+                        name: windowingtablefunction
+                        order by: _col3 ASC NULLS FIRST
+                        partition by: _col3
+                        raw input shape:
+                        window functions:
+                            window function definition
+                              alias: sum_window_0
+                              arguments: _col5
+                              name: sum
+                              window function: GenericUDAFSumHiveDecimal
+                              window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX)
+                  Statistics: Num rows: 348477374 Data size: 30742775095 Basic stats: COMPLETE
Column stats: NONE
+                  Select Operator
+                    expressions: _col1 (type: string), _col2 (type: string), _col3 (type:
string), _col4 (type: decimal(7,2)), _col5 (type: decimal(17,2)), ((_col5 * 100) / sum_window_0)
(type: decimal(38,17)), _col0 (type: string)
+                    outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6
+                    Statistics: Num rows: 348477374 Data size: 30742775095 Basic stats: COMPLETE
Column stats: NONE
+                    Reduce Output Operator
+                      key expressions: _col1 (type: string), _col2 (type: string), _col6
(type: string), _col0 (type: string), _col5 (type: decimal(38,17))
+                      sort order: +++++
+                      Statistics: Num rows: 348477374 Data size: 30742775095 Basic stats:
COMPLETE Column stats: NONE
+                      value expressions: _col3 (type: decimal(7,2)), _col4 (type: decimal(17,2))
+        Reducer 5 
+            Reduce Operator Tree:
+              Select Operator
+                expressions: KEY.reducesinkkey3 (type: string), KEY.reducesinkkey0 (type:
string), KEY.reducesinkkey1 (type: string), VALUE._col0 (type: decimal(7,2)), VALUE._col1
(type: decimal(17,2)), KEY.reducesinkkey4 (type: decimal(38,17))
+                outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
+                Statistics: Num rows: 348477374 Data size: 30742775095 Basic stats: COMPLETE
Column stats: NONE
+                File Output Operator
+                  compressed: false
+                  Statistics: Num rows: 348477374 Data size: 30742775095 Basic stats: COMPLETE
Column stats: NONE
+                  table:
+                      input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+                      output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+                      serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+  Stage: Stage-0
+    Fetch Operator
+      limit: -1
+      Processor Tree:
+        ListSink
+

http://git-wip-us.apache.org/repos/asf/hive/blob/bd371246/ql/src/test/results/clientpositive/perf/spark/query99.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/perf/spark/query99.q.out b/ql/src/test/results/clientpositive/perf/spark/query99.q.out
new file mode 100644
index 0000000..c0c5f13
--- /dev/null
+++ b/ql/src/test/results/clientpositive/perf/spark/query99.q.out
@@ -0,0 +1,270 @@
+PREHOOK: query: explain
+select  
+   substr(w_warehouse_name,1,20)
+  ,sm_type
+  ,cc_name
+  ,sum(case when (cs_ship_date_sk - cs_sold_date_sk <= 30 ) then 1 else 0 end)  as `30
days` 
+  ,sum(case when (cs_ship_date_sk - cs_sold_date_sk > 30) and 
+                 (cs_ship_date_sk - cs_sold_date_sk <= 60) then 1 else 0 end )  as `31-60
days` 
+  ,sum(case when (cs_ship_date_sk - cs_sold_date_sk > 60) and 
+                 (cs_ship_date_sk - cs_sold_date_sk <= 90) then 1 else 0 end)  as `61-90
days` 
+  ,sum(case when (cs_ship_date_sk - cs_sold_date_sk > 90) and
+                 (cs_ship_date_sk - cs_sold_date_sk <= 120) then 1 else 0 end)  as `91-120
days` 
+  ,sum(case when (cs_ship_date_sk - cs_sold_date_sk  > 120) then 1 else 0 end)  as `>120
days` 
+from
+   catalog_sales
+  ,warehouse
+  ,ship_mode
+  ,call_center
+  ,date_dim
+where
+    d_month_seq between 1212 and 1212 + 11
+and cs_ship_date_sk   = d_date_sk
+and cs_warehouse_sk   = w_warehouse_sk
+and cs_ship_mode_sk   = sm_ship_mode_sk
+and cs_call_center_sk = cc_call_center_sk
+group by
+   substr(w_warehouse_name,1,20)
+  ,sm_type
+  ,cc_name
+order by substr(w_warehouse_name,1,20)
+        ,sm_type
+        ,cc_name
+limit 100
+PREHOOK: type: QUERY
+POSTHOOK: query: explain
+select  
+   substr(w_warehouse_name,1,20)
+  ,sm_type
+  ,cc_name
+  ,sum(case when (cs_ship_date_sk - cs_sold_date_sk <= 30 ) then 1 else 0 end)  as `30
days` 
+  ,sum(case when (cs_ship_date_sk - cs_sold_date_sk > 30) and 
+                 (cs_ship_date_sk - cs_sold_date_sk <= 60) then 1 else 0 end )  as `31-60
days` 
+  ,sum(case when (cs_ship_date_sk - cs_sold_date_sk > 60) and 
+                 (cs_ship_date_sk - cs_sold_date_sk <= 90) then 1 else 0 end)  as `61-90
days` 
+  ,sum(case when (cs_ship_date_sk - cs_sold_date_sk > 90) and
+                 (cs_ship_date_sk - cs_sold_date_sk <= 120) then 1 else 0 end)  as `91-120
days` 
+  ,sum(case when (cs_ship_date_sk - cs_sold_date_sk  > 120) then 1 else 0 end)  as `>120
days` 
+from
+   catalog_sales
+  ,warehouse
+  ,ship_mode
+  ,call_center
+  ,date_dim
+where
+    d_month_seq between 1212 and 1212 + 11
+and cs_ship_date_sk   = d_date_sk
+and cs_warehouse_sk   = w_warehouse_sk
+and cs_ship_mode_sk   = sm_ship_mode_sk
+and cs_call_center_sk = cc_call_center_sk
+group by
+   substr(w_warehouse_name,1,20)
+  ,sm_type
+  ,cc_name
+order by substr(w_warehouse_name,1,20)
+        ,sm_type
+        ,cc_name
+limit 100
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+  Stage-2 is a root stage
+  Stage-1 depends on stages: Stage-2
+  Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+  Stage: Stage-2
+    Spark
+#### A masked pattern was here ####
+      Vertices:
+        Map 4 
+            Map Operator Tree:
+                TableScan
+                  alias: date_dim
+                  Statistics: Num rows: 73049 Data size: 81741831 Basic stats: COMPLETE Column
stats: NONE
+                  Filter Operator
+                    predicate: (d_date_sk is not null and d_month_seq BETWEEN 1212 AND 1223)
(type: boolean)
+                    Statistics: Num rows: 8116 Data size: 9081804 Basic stats: COMPLETE Column
stats: NONE
+                    Select Operator
+                      expressions: d_date_sk (type: int)
+                      outputColumnNames: _col0
+                      Statistics: Num rows: 8116 Data size: 9081804 Basic stats: COMPLETE
Column stats: NONE
+                      Spark HashTable Sink Operator
+                        keys:
+                          0 _col1 (type: int)
+                          1 _col0 (type: int)
+            Local Work:
+              Map Reduce Local Work
+        Map 5 
+            Map Operator Tree:
+                TableScan
+                  alias: call_center
+                  Statistics: Num rows: 60 Data size: 122700 Basic stats: COMPLETE Column
stats: NONE
+                  Filter Operator
+                    predicate: cc_call_center_sk is not null (type: boolean)
+                    Statistics: Num rows: 60 Data size: 122700 Basic stats: COMPLETE Column
stats: NONE
+                    Select Operator
+                      expressions: cc_call_center_sk (type: int), cc_name (type: string)
+                      outputColumnNames: _col0, _col1
+                      Statistics: Num rows: 60 Data size: 122700 Basic stats: COMPLETE Column
stats: NONE
+                      Spark HashTable Sink Operator
+                        keys:
+                          0 _col2 (type: int)
+                          1 _col0 (type: int)
+            Local Work:
+              Map Reduce Local Work
+        Map 6 
+            Map Operator Tree:
+                TableScan
+                  alias: warehouse
+                  Statistics: Num rows: 27 Data size: 27802 Basic stats: COMPLETE Column
stats: NONE
+                  Filter Operator
+                    predicate: w_warehouse_sk is not null (type: boolean)
+                    Statistics: Num rows: 27 Data size: 27802 Basic stats: COMPLETE Column
stats: NONE
+                    Select Operator
+                      expressions: w_warehouse_sk (type: int), w_warehouse_name (type: string)
+                      outputColumnNames: _col0, _col1
+                      Statistics: Num rows: 27 Data size: 27802 Basic stats: COMPLETE Column
stats: NONE
+                      Spark HashTable Sink Operator
+                        keys:
+                          0 _col4 (type: int)
+                          1 _col0 (type: int)
+            Local Work:
+              Map Reduce Local Work
+        Map 7 
+            Map Operator Tree:
+                TableScan
+                  alias: ship_mode
+                  Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats:
NONE
+                  Filter Operator
+                    predicate: sm_ship_mode_sk is not null (type: boolean)
+                    Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats:
NONE
+                    Select Operator
+                      expressions: sm_ship_mode_sk (type: int), sm_type (type: string)
+                      outputColumnNames: _col0, _col1
+                      Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats:
NONE
+                      Spark HashTable Sink Operator
+                        keys:
+                          0 _col3 (type: int)
+                          1 _col0 (type: int)
+            Local Work:
+              Map Reduce Local Work
+
+  Stage: Stage-1
+    Spark
+      Edges:
+        Reducer 2 <- Map 1 (GROUP, 447)
+        Reducer 3 <- Reducer 2 (SORT, 1)
+#### A masked pattern was here ####
+      Vertices:
+        Map 1 
+            Map Operator Tree:
+                TableScan
+                  alias: catalog_sales
+                  Statistics: Num rows: 287989836 Data size: 38999608952 Basic stats: COMPLETE
Column stats: NONE
+                  Filter Operator
+                    predicate: (cs_call_center_sk is not null and cs_ship_date_sk is not
null and cs_ship_mode_sk is not null and cs_warehouse_sk is not null) (type: boolean)
+                    Statistics: Num rows: 287989836 Data size: 38999608952 Basic stats: COMPLETE
Column stats: NONE
+                    Select Operator
+                      expressions: cs_sold_date_sk (type: int), cs_ship_date_sk (type: int),
cs_call_center_sk (type: int), cs_ship_mode_sk (type: int), cs_warehouse_sk (type: int)
+                      outputColumnNames: _col0, _col1, _col2, _col3, _col4
+                      Statistics: Num rows: 287989836 Data size: 38999608952 Basic stats:
COMPLETE Column stats: NONE
+                      Map Join Operator
+                        condition map:
+                             Inner Join 0 to 1
+                        keys:
+                          0 _col1 (type: int)
+                          1 _col0 (type: int)
+                        outputColumnNames: _col0, _col1, _col2, _col3, _col4
+                        input vertices:
+                          1 Map 4
+                        Statistics: Num rows: 316788826 Data size: 42899570777 Basic stats:
COMPLETE Column stats: NONE
+                        Map Join Operator
+                          condition map:
+                               Inner Join 0 to 1
+                          keys:
+                            0 _col2 (type: int)
+                            1 _col0 (type: int)
+                          outputColumnNames: _col0, _col1, _col3, _col4, _col8
+                          input vertices:
+                            1 Map 5
+                          Statistics: Num rows: 348467716 Data size: 47189528877 Basic stats:
COMPLETE Column stats: NONE
+                          Map Join Operator
+                            condition map:
+                                 Inner Join 0 to 1
+                            keys:
+                              0 _col4 (type: int)
+                              1 _col0 (type: int)
+                            outputColumnNames: _col0, _col1, _col3, _col8, _col10
+                            input vertices:
+                              1 Map 6
+                            Statistics: Num rows: 383314495 Data size: 51908482889 Basic
stats: COMPLETE Column stats: NONE
+                            Map Join Operator
+                              condition map:
+                                   Inner Join 0 to 1
+                              keys:
+                                0 _col3 (type: int)
+                                1 _col0 (type: int)
+                              outputColumnNames: _col0, _col1, _col8, _col10, _col12
+                              input vertices:
+                                1 Map 7
+                              Statistics: Num rows: 421645953 Data size: 57099332415 Basic
stats: COMPLETE Column stats: NONE
+                              Select Operator
+                                expressions: substr(_col10, 1, 20) (type: string), _col12
(type: string), _col8 (type: string), CASE WHEN (((_col1 - _col0) <= 30)) THEN (1) ELSE
(0) END (type: int), CASE WHEN ((((_col1 - _col0) > 30) and ((_col1 - _col0) <= 60)))
THEN (1) ELSE (0) END (type: int), CASE WHEN ((((_col1 - _col0) > 60) and ((_col1 - _col0)
<= 90))) THEN (1) ELSE (0) END (type: int), CASE WHEN ((((_col1 - _col0) > 90) and ((_col1
- _col0) <= 120))) THEN (1) ELSE (0) END (type: int), CASE WHEN (((_col1 - _col0) >
120)) THEN (1) ELSE (0) END (type: int)
+                                outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5,
_col6, _col7
+                                Statistics: Num rows: 421645953 Data size: 57099332415 Basic
stats: COMPLETE Column stats: NONE
+                                Group By Operator
+                                  aggregations: sum(_col3), sum(_col4), sum(_col5), sum(_col6),
sum(_col7)
+                                  keys: _col0 (type: string), _col1 (type: string), _col2
(type: string)
+                                  mode: hash
+                                  outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5,
_col6, _col7
+                                  Statistics: Num rows: 421645953 Data size: 57099332415
Basic stats: COMPLETE Column stats: NONE
+                                  Reduce Output Operator
+                                    key expressions: _col0 (type: string), _col1 (type: string),
_col2 (type: string)
+                                    sort order: +++
+                                    Map-reduce partition columns: _col0 (type: string), _col1
(type: string), _col2 (type: string)
+                                    Statistics: Num rows: 421645953 Data size: 57099332415
Basic stats: COMPLETE Column stats: NONE
+                                    TopN Hash Memory Usage: 0.1
+                                    value expressions: _col3 (type: bigint), _col4 (type:
bigint), _col5 (type: bigint), _col6 (type: bigint), _col7 (type: bigint)
+            Local Work:
+              Map Reduce Local Work
+        Reducer 2 
+            Reduce Operator Tree:
+              Group By Operator
+                aggregations: sum(VALUE._col0), sum(VALUE._col1), sum(VALUE._col2), sum(VALUE._col3),
sum(VALUE._col4)
+                keys: KEY._col0 (type: string), KEY._col1 (type: string), KEY._col2 (type:
string)
+                mode: mergepartial
+                outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7
+                Statistics: Num rows: 210822976 Data size: 28549666139 Basic stats: COMPLETE
Column stats: NONE
+                Select Operator
+                  expressions: _col1 (type: string), _col2 (type: string), _col3 (type: bigint),
_col4 (type: bigint), _col5 (type: bigint), _col6 (type: bigint), _col7 (type: bigint), _col0
(type: string)
+                  outputColumnNames: _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8
+                  Statistics: Num rows: 210822976 Data size: 28549666139 Basic stats: COMPLETE
Column stats: NONE
+                  Reduce Output Operator
+                    key expressions: _col8 (type: string), _col1 (type: string), _col2 (type:
string)
+                    sort order: +++
+                    Statistics: Num rows: 210822976 Data size: 28549666139 Basic stats: COMPLETE
Column stats: NONE
+                    TopN Hash Memory Usage: 0.1
+                    value expressions: _col3 (type: bigint), _col4 (type: bigint), _col5
(type: bigint), _col6 (type: bigint), _col7 (type: bigint)
+        Reducer 3 
+            Reduce Operator Tree:
+              Select Operator
+                expressions: KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type:
string), KEY.reducesinkkey2 (type: string), VALUE._col0 (type: bigint), VALUE._col1 (type:
bigint), VALUE._col2 (type: bigint), VALUE._col3 (type: bigint), VALUE._col4 (type: bigint)
+                outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7
+                Statistics: Num rows: 210822976 Data size: 28549666139 Basic stats: COMPLETE
Column stats: NONE
+                Limit
+                  Number of rows: 100
+                  Statistics: Num rows: 100 Data size: 13500 Basic stats: COMPLETE Column
stats: NONE
+                  File Output Operator
+                    compressed: false
+                    Statistics: Num rows: 100 Data size: 13500 Basic stats: COMPLETE Column
stats: NONE
+                    table:
+                        input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+                        output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+                        serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+  Stage: Stage-0
+    Fetch Operator
+      limit: -1
+      Processor Tree:
+        ListSink
+


Mime
View raw message