hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From hashut...@apache.org
Subject [01/21] hive git commit: HIVE-15192 : Use Calcite to de-correlate and plan subqueries (Vineet Garg via Ashutosh Chauhan)
Date Fri, 16 Dec 2016 18:28:14 GMT
Repository: hive
Updated Branches:
  refs/heads/master 2ae78f01b -> 382dc2084


http://git-wip-us.apache.org/repos/asf/hive/blob/382dc208/ql/src/test/results/clientpositive/vector_groupby_mapjoin.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/vector_groupby_mapjoin.q.out b/ql/src/test/results/clientpositive/vector_groupby_mapjoin.q.out
index 3468657..af42e41 100644
--- a/ql/src/test/results/clientpositive/vector_groupby_mapjoin.q.out
+++ b/ql/src/test/results/clientpositive/vector_groupby_mapjoin.q.out
@@ -1,4 +1,4 @@
-Warning: Map Join MAPJOIN[32][bigTable=?] in task 'Stage-3:MAPRED' is a cross product
+Warning: Map Join MAPJOIN[35][bigTable=?] in task 'Stage-8:MAPRED' is a cross product
 PREHOOK: query: -- HIVE-12738 -- We are checking if a MapJoin after a GroupBy will work properly.
 explain
 select *
@@ -17,8 +17,14 @@ order by key
 POSTHOOK: type: QUERY
 STAGE DEPENDENCIES:
   Stage-4 is a root stage
-  Stage-8 depends on stages: Stage-4
-  Stage-3 depends on stages: Stage-8
+  Stage-10 depends on stages: Stage-4
+  Stage-8 depends on stages: Stage-10
+  Stage-7 depends on stages: Stage-5, Stage-8 , consists of Stage-9, Stage-2
+  Stage-9 has a backup stage: Stage-2
+  Stage-6 depends on stages: Stage-9
+  Stage-3 depends on stages: Stage-2, Stage-6
+  Stage-2
+  Stage-5 is a root stage
   Stage-0 depends on stages: Stage-3
 
 STAGE PLANS:
@@ -28,47 +34,38 @@ STAGE PLANS:
           TableScan
             alias: src
             Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats:
NONE
-            Filter Operator
-              predicate: key is null (type: boolean)
-              Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats:
NONE
-              Select Operator
-                Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats:
NONE
-                Group By Operator
-                  aggregations: count()
-                  mode: hash
-                  outputColumnNames: _col0
-                  Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats:
NONE
-                  Reduce Output Operator
-                    sort order: 
-                    Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats:
NONE
-                    value expressions: _col0 (type: bigint)
+            Select Operator
+              expressions: key (type: string)
+              outputColumnNames: key
+              Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats:
NONE
+              Group By Operator
+                aggregations: count(), count(key)
+                mode: hash
+                outputColumnNames: _col0, _col1
+                Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats:
NONE
+                Reduce Output Operator
+                  sort order: 
+                  Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats:
NONE
+                  value expressions: _col0 (type: bigint), _col1 (type: bigint)
       Reduce Operator Tree:
         Group By Operator
-          aggregations: count(VALUE._col0)
+          aggregations: count(VALUE._col0), count(VALUE._col1)
           mode: mergepartial
-          outputColumnNames: _col0
-          Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
-          Filter Operator
-            predicate: (_col0 = 0) (type: boolean)
-            Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
-            Select Operator
-              Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
-              File Output Operator
-                compressed: false
-                table:
-                    input format: org.apache.hadoop.mapred.SequenceFileInputFormat
-                    output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
-                    serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
+          outputColumnNames: _col0, _col1
+          Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE
+          File Output Operator
+            compressed: false
+            table:
+                input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+                output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+                serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
 
-  Stage: Stage-8
+  Stage: Stage-10
     Map Reduce Local Work
       Alias -> Map Local Tables:
         $hdt$_0:src 
           Fetch Operator
             limit: -1
-        $hdt$_2:src 
-          Fetch Operator
-            limit: -1
       Alias -> Map Local Operator Tree:
         $hdt$_0:src 
           TableScan
@@ -82,20 +79,8 @@ STAGE PLANS:
                 keys:
                   0 
                   1 
-        $hdt$_2:src 
-          TableScan
-            alias: src
-            Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats:
NONE
-            Select Operator
-              expressions: key (type: string)
-              outputColumnNames: _col0
-              Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats:
NONE
-              HashTable Sink Operator
-                keys:
-                  0 _col0 (type: string)
-                  1 _col0 (type: string)
 
-  Stage: Stage-3
+  Stage: Stage-8
     Map Reduce
       Map Operator Tree:
           TableScan
@@ -105,50 +90,164 @@ STAGE PLANS:
               keys:
                 0 
                 1 
-              outputColumnNames: _col0, _col1
-              Statistics: Num rows: 500 Data size: 9812 Basic stats: COMPLETE Column stats:
NONE
-              Map Join Operator
-                condition map:
-                     Left Outer Join0 to 1
-                keys:
-                  0 _col0 (type: string)
-                  1 _col0 (type: string)
-                outputColumnNames: _col0, _col1, _col3
-                Statistics: Num rows: 550 Data size: 10793 Basic stats: COMPLETE Column stats:
NONE
-                Filter Operator
-                  predicate: _col3 is null (type: boolean)
-                  Statistics: Num rows: 275 Data size: 5396 Basic stats: COMPLETE Column
stats: NONE
-                  Select Operator
-                    expressions: _col0 (type: string), _col1 (type: string)
-                    outputColumnNames: _col0, _col1
-                    Statistics: Num rows: 275 Data size: 5396 Basic stats: COMPLETE Column
stats: NONE
-                    Reduce Output Operator
-                      key expressions: _col0 (type: string)
-                      sort order: +
-                      Statistics: Num rows: 275 Data size: 5396 Basic stats: COMPLETE Column
stats: NONE
-                      value expressions: _col1 (type: string)
+              outputColumnNames: _col0, _col1, _col2, _col3
+              Statistics: Num rows: 500 Data size: 13812 Basic stats: COMPLETE Column stats:
NONE
+              File Output Operator
+                compressed: false
+                table:
+                    input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+                    output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+                    serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
+      Local Work:
+        Map Reduce Local Work
+
+  Stage: Stage-7
+    Conditional Operator
+
+  Stage: Stage-9
+    Map Reduce Local Work
+      Alias -> Map Local Tables:
+        $INTNAME1 
+          Fetch Operator
+            limit: -1
+      Alias -> Map Local Operator Tree:
+        $INTNAME1 
+          TableScan
+            HashTable Sink Operator
+              keys:
+                0 _col0 (type: string)
+                1 _col0 (type: string)
+
+  Stage: Stage-6
+    Map Reduce
+      Map Operator Tree:
+          TableScan
+            Map Join Operator
+              condition map:
+                   Left Outer Join0 to 1
+              keys:
+                0 _col0 (type: string)
+                1 _col0 (type: string)
+              outputColumnNames: _col0, _col1, _col2, _col3, _col5
+              Statistics: Num rows: 550 Data size: 15193 Basic stats: COMPLETE Column stats:
NONE
+              Filter Operator
+                predicate: (not CASE WHEN ((_col2 = 0)) THEN (false) WHEN (_col5 is not null)
THEN (true) WHEN (_col0 is null) THEN (null) WHEN ((_col3 < _col2)) THEN (true) ELSE (false)
END) (type: boolean)
+                Statistics: Num rows: 275 Data size: 7596 Basic stats: COMPLETE Column stats:
NONE
+                Select Operator
+                  expressions: _col0 (type: string), _col1 (type: string)
+                  outputColumnNames: _col0, _col1
+                  Statistics: Num rows: 275 Data size: 7596 Basic stats: COMPLETE Column
stats: NONE
+                  File Output Operator
+                    compressed: false
+                    table:
+                        input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+                        output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+                        serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
       Local Work:
         Map Reduce Local Work
+
+  Stage: Stage-3
+    Map Reduce
+      Map Operator Tree:
+          TableScan
+            Reduce Output Operator
+              key expressions: _col0 (type: string)
+              sort order: +
+              Statistics: Num rows: 275 Data size: 7596 Basic stats: COMPLETE Column stats:
NONE
+              value expressions: _col1 (type: string)
       Reduce Operator Tree:
         Select Operator
           expressions: KEY.reducesinkkey0 (type: string), VALUE._col0 (type: string)
           outputColumnNames: _col0, _col1
-          Statistics: Num rows: 275 Data size: 5396 Basic stats: COMPLETE Column stats: NONE
+          Statistics: Num rows: 275 Data size: 7596 Basic stats: COMPLETE Column stats: NONE
           File Output Operator
             compressed: false
-            Statistics: Num rows: 275 Data size: 5396 Basic stats: COMPLETE Column stats:
NONE
+            Statistics: Num rows: 275 Data size: 7596 Basic stats: COMPLETE Column stats:
NONE
             table:
                 input format: org.apache.hadoop.mapred.SequenceFileInputFormat
                 output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
                 serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
 
+  Stage: Stage-2
+    Map Reduce
+      Map Operator Tree:
+          TableScan
+            Reduce Output Operator
+              key expressions: _col0 (type: string)
+              sort order: +
+              Map-reduce partition columns: _col0 (type: string)
+              Statistics: Num rows: 500 Data size: 13812 Basic stats: COMPLETE Column stats:
NONE
+              value expressions: _col1 (type: string), _col2 (type: bigint), _col3 (type:
bigint)
+          TableScan
+            Reduce Output Operator
+              key expressions: _col0 (type: string)
+              sort order: +
+              Map-reduce partition columns: _col0 (type: string)
+              Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats:
NONE
+              value expressions: _col1 (type: boolean)
+      Reduce Operator Tree:
+        Join Operator
+          condition map:
+               Left Outer Join0 to 1
+          keys:
+            0 _col0 (type: string)
+            1 _col0 (type: string)
+          outputColumnNames: _col0, _col1, _col2, _col3, _col5
+          Statistics: Num rows: 550 Data size: 15193 Basic stats: COMPLETE Column stats:
NONE
+          Filter Operator
+            predicate: (not CASE WHEN ((_col2 = 0)) THEN (false) WHEN (_col5 is not null)
THEN (true) WHEN (_col0 is null) THEN (null) WHEN ((_col3 < _col2)) THEN (true) ELSE (false)
END) (type: boolean)
+            Statistics: Num rows: 275 Data size: 7596 Basic stats: COMPLETE Column stats:
NONE
+            Select Operator
+              expressions: _col0 (type: string), _col1 (type: string)
+              outputColumnNames: _col0, _col1
+              Statistics: Num rows: 275 Data size: 7596 Basic stats: COMPLETE Column stats:
NONE
+              File Output Operator
+                compressed: false
+                table:
+                    input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+                    output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+                    serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
+
+  Stage: Stage-5
+    Map Reduce
+      Map Operator Tree:
+          TableScan
+            alias: src
+            Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats:
NONE
+            Select Operator
+              expressions: key (type: string)
+              outputColumnNames: _col0
+              Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats:
NONE
+              Group By Operator
+                keys: _col0 (type: string), true (type: boolean)
+                mode: hash
+                outputColumnNames: _col0, _col1
+                Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats:
NONE
+                Reduce Output Operator
+                  key expressions: _col0 (type: string), _col1 (type: boolean)
+                  sort order: ++
+                  Map-reduce partition columns: _col0 (type: string), _col1 (type: boolean)
+                  Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column
stats: NONE
+      Reduce Operator Tree:
+        Group By Operator
+          keys: KEY._col0 (type: string), KEY._col1 (type: boolean)
+          mode: mergepartial
+          outputColumnNames: _col0, _col1
+          Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
+          File Output Operator
+            compressed: false
+            table:
+                input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+                output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+                serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
+
   Stage: Stage-0
     Fetch Operator
       limit: -1
       Processor Tree:
         ListSink
 
-Warning: Map Join MAPJOIN[32][bigTable=?] in task 'Stage-3:MAPRED' is a cross product
+Warning: Map Join MAPJOIN[35][bigTable=?] in task 'Stage-8:MAPRED' is a cross product
 PREHOOK: query: select *
 from src
 where not key in

http://git-wip-us.apache.org/repos/asf/hive/blob/382dc208/ql/src/test/results/clientpositive/vector_mapjoin_reduce.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/vector_mapjoin_reduce.q.out b/ql/src/test/results/clientpositive/vector_mapjoin_reduce.q.out
index 160b088..b2b9a3b 100644
--- a/ql/src/test/results/clientpositive/vector_mapjoin_reduce.q.out
+++ b/ql/src/test/results/clientpositive/vector_mapjoin_reduce.q.out
@@ -23,19 +23,20 @@ where li.l_linenumber = 1 and
  li.l_orderkey in (select l_orderkey from lineitem where l_shipmode = 'AIR')
 POSTHOOK: type: QUERY
 STAGE DEPENDENCIES:
-  Stage-4 is a root stage
-  Stage-7 depends on stages: Stage-4, Stage-8 , consists of Stage-9, Stage-10, Stage-2
-  Stage-9 has a backup stage: Stage-2
-  Stage-5 depends on stages: Stage-9
-  Stage-10 has a backup stage: Stage-2
+  Stage-1 is a root stage
+  Stage-12 depends on stages: Stage-1
+  Stage-9 depends on stages: Stage-12
+  Stage-8 depends on stages: Stage-5, Stage-9 , consists of Stage-10, Stage-11, Stage-3
+  Stage-10 has a backup stage: Stage-3
   Stage-6 depends on stages: Stage-10
-  Stage-2
-  Stage-11 is a root stage
-  Stage-8 depends on stages: Stage-11
-  Stage-0 depends on stages: Stage-5, Stage-6, Stage-2
+  Stage-11 has a backup stage: Stage-3
+  Stage-7 depends on stages: Stage-11
+  Stage-3
+  Stage-5 is a root stage
+  Stage-0 depends on stages: Stage-6, Stage-7, Stage-3
 
 STAGE PLANS:
-  Stage: Stage-4
+  Stage: Stage-1
     Map Reduce
       Map Operator Tree:
           TableScan
@@ -67,10 +68,54 @@ STAGE PLANS:
                 output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
                 serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
 
-  Stage: Stage-7
-    Conditional Operator
+  Stage: Stage-12
+    Map Reduce Local Work
+      Alias -> Map Local Tables:
+        $hdt$_1:li 
+          Fetch Operator
+            limit: -1
+      Alias -> Map Local Operator Tree:
+        $hdt$_1:li 
+          TableScan
+            alias: li
+            Statistics: Num rows: 100 Data size: 11999 Basic stats: COMPLETE Column stats:
NONE
+            Filter Operator
+              predicate: ((l_linenumber = 1) and l_partkey is not null) (type: boolean)
+              Statistics: Num rows: 50 Data size: 5999 Basic stats: COMPLETE Column stats:
NONE
+              Select Operator
+                expressions: l_orderkey (type: int), l_partkey (type: int), l_suppkey (type:
int)
+                outputColumnNames: _col0, _col1, _col2
+                Statistics: Num rows: 50 Data size: 5999 Basic stats: COMPLETE Column stats:
NONE
+                HashTable Sink Operator
+                  keys:
+                    0 _col0 (type: int)
+                    1 _col1 (type: int)
 
   Stage: Stage-9
+    Map Reduce
+      Map Operator Tree:
+          TableScan
+            Map Join Operator
+              condition map:
+                   Inner Join 0 to 1
+              keys:
+                0 _col0 (type: int)
+                1 _col1 (type: int)
+              outputColumnNames: _col0, _col1, _col3
+              Statistics: Num rows: 55 Data size: 6598 Basic stats: COMPLETE Column stats:
NONE
+              File Output Operator
+                compressed: false
+                table:
+                    input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+                    output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+                    serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
+      Local Work:
+        Map Reduce Local Work
+
+  Stage: Stage-8
+    Conditional Operator
+
+  Stage: Stage-10
     Map Reduce Local Work
       Alias -> Map Local Tables:
         $INTNAME1 
@@ -84,7 +129,7 @@ STAGE PLANS:
                 0 _col1 (type: int)
                 1 _col0 (type: int)
 
-  Stage: Stage-5
+  Stage: Stage-6
     Map Reduce
       Map Operator Tree:
           TableScan
@@ -94,10 +139,10 @@ STAGE PLANS:
               keys:
                 0 _col1 (type: int)
                 1 _col0 (type: int)
-              outputColumnNames: _col2, _col4
+              outputColumnNames: _col0, _col3
               Statistics: Num rows: 60 Data size: 7257 Basic stats: COMPLETE Column stats:
NONE
               Select Operator
-                expressions: _col4 (type: int), _col2 (type: int)
+                expressions: _col0 (type: int), _col3 (type: int)
                 outputColumnNames: _col0, _col1
                 Statistics: Num rows: 60 Data size: 7257 Basic stats: COMPLETE Column stats:
NONE
                 File Output Operator
@@ -110,7 +155,7 @@ STAGE PLANS:
       Local Work:
         Map Reduce Local Work
 
-  Stage: Stage-10
+  Stage: Stage-11
     Map Reduce Local Work
       Alias -> Map Local Tables:
         $INTNAME 
@@ -124,7 +169,7 @@ STAGE PLANS:
                 0 _col1 (type: int)
                 1 _col0 (type: int)
 
-  Stage: Stage-6
+  Stage: Stage-7
     Map Reduce
       Map Operator Tree:
           TableScan
@@ -134,10 +179,10 @@ STAGE PLANS:
               keys:
                 0 _col1 (type: int)
                 1 _col0 (type: int)
-              outputColumnNames: _col2, _col4
+              outputColumnNames: _col0, _col3
               Statistics: Num rows: 60 Data size: 7257 Basic stats: COMPLETE Column stats:
NONE
               Select Operator
-                expressions: _col4 (type: int), _col2 (type: int)
+                expressions: _col0 (type: int), _col3 (type: int)
                 outputColumnNames: _col0, _col1
                 Statistics: Num rows: 60 Data size: 7257 Basic stats: COMPLETE Column stats:
NONE
                 File Output Operator
@@ -150,7 +195,7 @@ STAGE PLANS:
       Local Work:
         Map Reduce Local Work
 
-  Stage: Stage-2
+  Stage: Stage-3
     Map Reduce
       Map Operator Tree:
           TableScan
@@ -159,13 +204,13 @@ STAGE PLANS:
               sort order: +
               Map-reduce partition columns: _col1 (type: int)
               Statistics: Num rows: 55 Data size: 6598 Basic stats: COMPLETE Column stats:
NONE
-              value expressions: _col2 (type: int)
+              value expressions: _col0 (type: int), _col3 (type: int)
           TableScan
             Reduce Output Operator
               key expressions: _col0 (type: int)
               sort order: +
               Map-reduce partition columns: _col0 (type: int)
-              Statistics: Num rows: 50 Data size: 5999 Basic stats: COMPLETE Column stats:
NONE
+              Statistics: Num rows: 25 Data size: 2999 Basic stats: COMPLETE Column stats:
NONE
       Reduce Operator Tree:
         Join Operator
           condition map:
@@ -173,10 +218,10 @@ STAGE PLANS:
           keys:
             0 _col1 (type: int)
             1 _col0 (type: int)
-          outputColumnNames: _col2, _col4
+          outputColumnNames: _col0, _col3
           Statistics: Num rows: 60 Data size: 7257 Basic stats: COMPLETE Column stats: NONE
           Select Operator
-            expressions: _col4 (type: int), _col2 (type: int)
+            expressions: _col0 (type: int), _col3 (type: int)
             outputColumnNames: _col0, _col1
             Statistics: Num rows: 60 Data size: 7257 Basic stats: COMPLETE Column stats:
NONE
             File Output Operator
@@ -187,63 +232,41 @@ STAGE PLANS:
                   output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
                   serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
 
-  Stage: Stage-11
-    Map Reduce Local Work
-      Alias -> Map Local Tables:
-        $hdt$_2:lineitem 
-          Fetch Operator
-            limit: -1
-      Alias -> Map Local Operator Tree:
-        $hdt$_2:lineitem 
+  Stage: Stage-5
+    Map Reduce
+      Map Operator Tree:
           TableScan
             alias: lineitem
             Statistics: Num rows: 100 Data size: 11999 Basic stats: COMPLETE Column stats:
NONE
             Filter Operator
-              predicate: ((l_shipmode = 'AIR') and l_orderkey is not null) (type: boolean)
+              predicate: (l_shipmode = 'AIR') (type: boolean)
               Statistics: Num rows: 50 Data size: 5999 Basic stats: COMPLETE Column stats:
NONE
               Select Operator
                 expressions: l_orderkey (type: int)
-                outputColumnNames: _col0
+                outputColumnNames: l_orderkey
                 Statistics: Num rows: 50 Data size: 5999 Basic stats: COMPLETE Column stats:
NONE
                 Group By Operator
-                  keys: _col0 (type: int)
+                  keys: l_orderkey (type: int)
                   mode: hash
                   outputColumnNames: _col0
                   Statistics: Num rows: 50 Data size: 5999 Basic stats: COMPLETE Column stats:
NONE
-                  HashTable Sink Operator
-                    keys:
-                      0 _col0 (type: int)
-                      1 _col0 (type: int)
-
-  Stage: Stage-8
-    Map Reduce
-      Map Operator Tree:
-          TableScan
-            alias: li
-            Statistics: Num rows: 100 Data size: 11999 Basic stats: COMPLETE Column stats:
NONE
-            Filter Operator
-              predicate: ((l_linenumber = 1) and l_partkey is not null and l_orderkey is
not null) (type: boolean)
-              Statistics: Num rows: 50 Data size: 5999 Basic stats: COMPLETE Column stats:
NONE
-              Select Operator
-                expressions: l_orderkey (type: int), l_partkey (type: int), l_suppkey (type:
int)
-                outputColumnNames: _col0, _col1, _col2
-                Statistics: Num rows: 50 Data size: 5999 Basic stats: COMPLETE Column stats:
NONE
-                Map Join Operator
-                  condition map:
-                       Left Semi Join 0 to 1
-                  keys:
-                    0 _col0 (type: int)
-                    1 _col0 (type: int)
-                  outputColumnNames: _col1, _col2
-                  Statistics: Num rows: 55 Data size: 6598 Basic stats: COMPLETE Column stats:
NONE
-                  File Output Operator
-                    compressed: false
-                    table:
-                        input format: org.apache.hadoop.mapred.SequenceFileInputFormat
-                        output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
-                        serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
-      Local Work:
-        Map Reduce Local Work
+                  Reduce Output Operator
+                    key expressions: _col0 (type: int)
+                    sort order: +
+                    Map-reduce partition columns: _col0 (type: int)
+                    Statistics: Num rows: 50 Data size: 5999 Basic stats: COMPLETE Column
stats: NONE
+      Reduce Operator Tree:
+        Group By Operator
+          keys: KEY._col0 (type: int)
+          mode: mergepartial
+          outputColumnNames: _col0
+          Statistics: Num rows: 25 Data size: 2999 Basic stats: COMPLETE Column stats: NONE
+          File Output Operator
+            compressed: false
+            table:
+                input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+                output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+                serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
 
   Stage: Stage-0
     Fetch Operator
@@ -290,19 +313,24 @@ where li.l_linenumber = 1 and
  li.l_orderkey in (select l_orderkey from lineitem where l_shipmode = 'AIR' and l_linenumber
= li.l_linenumber)
 POSTHOOK: type: QUERY
 STAGE DEPENDENCIES:
-  Stage-4 is a root stage
-  Stage-7 depends on stages: Stage-4, Stage-8 , consists of Stage-9, Stage-10, Stage-2
-  Stage-9 has a backup stage: Stage-2
-  Stage-5 depends on stages: Stage-9
-  Stage-10 has a backup stage: Stage-2
-  Stage-6 depends on stages: Stage-10
-  Stage-2
-  Stage-11 is a root stage
-  Stage-8 depends on stages: Stage-11
-  Stage-0 depends on stages: Stage-5, Stage-6, Stage-2
+  Stage-1 is a root stage
+  Stage-19 depends on stages: Stage-1
+  Stage-14 depends on stages: Stage-19
+  Stage-13 depends on stages: Stage-6, Stage-14 , consists of Stage-17, Stage-18, Stage-3
+  Stage-17 has a backup stage: Stage-3
+  Stage-11 depends on stages: Stage-17
+  Stage-18 has a backup stage: Stage-3
+  Stage-12 depends on stages: Stage-18
+  Stage-3
+  Stage-7 is a root stage
+  Stage-21 depends on stages: Stage-7
+  Stage-9 depends on stages: Stage-21
+  Stage-20 depends on stages: Stage-9
+  Stage-6 depends on stages: Stage-20
+  Stage-0 depends on stages: Stage-11, Stage-12, Stage-3
 
 STAGE PLANS:
-  Stage: Stage-4
+  Stage: Stage-1
     Map Reduce
       Map Operator Tree:
           TableScan
@@ -334,10 +362,54 @@ STAGE PLANS:
                 output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
                 serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
 
-  Stage: Stage-7
+  Stage: Stage-19
+    Map Reduce Local Work
+      Alias -> Map Local Tables:
+        $hdt$_1:li 
+          Fetch Operator
+            limit: -1
+      Alias -> Map Local Operator Tree:
+        $hdt$_1:li 
+          TableScan
+            alias: li
+            Statistics: Num rows: 100 Data size: 11999 Basic stats: COMPLETE Column stats:
NONE
+            Filter Operator
+              predicate: ((l_linenumber = 1) and l_partkey is not null) (type: boolean)
+              Statistics: Num rows: 50 Data size: 5999 Basic stats: COMPLETE Column stats:
NONE
+              Select Operator
+                expressions: l_orderkey (type: int), l_partkey (type: int), l_suppkey (type:
int), 1 (type: int)
+                outputColumnNames: _col0, _col1, _col2, _col3
+                Statistics: Num rows: 50 Data size: 5999 Basic stats: COMPLETE Column stats:
NONE
+                HashTable Sink Operator
+                  keys:
+                    0 _col0 (type: int)
+                    1 _col1 (type: int)
+
+  Stage: Stage-14
+    Map Reduce
+      Map Operator Tree:
+          TableScan
+            Map Join Operator
+              condition map:
+                   Inner Join 0 to 1
+              keys:
+                0 _col0 (type: int)
+                1 _col1 (type: int)
+              outputColumnNames: _col0, _col1, _col3, _col4
+              Statistics: Num rows: 55 Data size: 6598 Basic stats: COMPLETE Column stats:
NONE
+              File Output Operator
+                compressed: false
+                table:
+                    input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+                    output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+                    serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
+      Local Work:
+        Map Reduce Local Work
+
+  Stage: Stage-13
     Conditional Operator
 
-  Stage: Stage-9
+  Stage: Stage-17
     Map Reduce Local Work
       Alias -> Map Local Tables:
         $INTNAME1 
@@ -348,10 +420,10 @@ STAGE PLANS:
           TableScan
             HashTable Sink Operator
               keys:
-                0 _col1 (type: int)
-                1 _col0 (type: int)
+                0 _col1 (type: int), _col4 (type: int)
+                1 _col0 (type: int), _col1 (type: int)
 
-  Stage: Stage-5
+  Stage: Stage-11
     Map Reduce
       Map Operator Tree:
           TableScan
@@ -359,12 +431,12 @@ STAGE PLANS:
               condition map:
                    Inner Join 0 to 1
               keys:
-                0 _col1 (type: int)
-                1 _col0 (type: int)
-              outputColumnNames: _col2, _col4
+                0 _col1 (type: int), _col4 (type: int)
+                1 _col0 (type: int), _col1 (type: int)
+              outputColumnNames: _col0, _col3
               Statistics: Num rows: 60 Data size: 7257 Basic stats: COMPLETE Column stats:
NONE
               Select Operator
-                expressions: _col4 (type: int), _col2 (type: int)
+                expressions: _col0 (type: int), _col3 (type: int)
                 outputColumnNames: _col0, _col1
                 Statistics: Num rows: 60 Data size: 7257 Basic stats: COMPLETE Column stats:
NONE
                 File Output Operator
@@ -377,7 +449,7 @@ STAGE PLANS:
       Local Work:
         Map Reduce Local Work
 
-  Stage: Stage-10
+  Stage: Stage-18
     Map Reduce Local Work
       Alias -> Map Local Tables:
         $INTNAME 
@@ -388,10 +460,10 @@ STAGE PLANS:
           TableScan
             HashTable Sink Operator
               keys:
-                0 _col1 (type: int)
-                1 _col0 (type: int)
+                0 _col1 (type: int), _col4 (type: int)
+                1 _col0 (type: int), _col1 (type: int)
 
-  Stage: Stage-6
+  Stage: Stage-12
     Map Reduce
       Map Operator Tree:
           TableScan
@@ -399,12 +471,12 @@ STAGE PLANS:
               condition map:
                    Inner Join 0 to 1
               keys:
-                0 _col1 (type: int)
-                1 _col0 (type: int)
-              outputColumnNames: _col2, _col4
+                0 _col1 (type: int), _col4 (type: int)
+                1 _col0 (type: int), _col1 (type: int)
+              outputColumnNames: _col0, _col3
               Statistics: Num rows: 60 Data size: 7257 Basic stats: COMPLETE Column stats:
NONE
               Select Operator
-                expressions: _col4 (type: int), _col2 (type: int)
+                expressions: _col0 (type: int), _col3 (type: int)
                 outputColumnNames: _col0, _col1
                 Statistics: Num rows: 60 Data size: 7257 Basic stats: COMPLETE Column stats:
NONE
                 File Output Operator
@@ -417,33 +489,33 @@ STAGE PLANS:
       Local Work:
         Map Reduce Local Work
 
-  Stage: Stage-2
+  Stage: Stage-3
     Map Reduce
       Map Operator Tree:
           TableScan
             Reduce Output Operator
-              key expressions: _col1 (type: int)
-              sort order: +
-              Map-reduce partition columns: _col1 (type: int)
+              key expressions: _col1 (type: int), _col4 (type: int)
+              sort order: ++
+              Map-reduce partition columns: _col1 (type: int), _col4 (type: int)
               Statistics: Num rows: 55 Data size: 6598 Basic stats: COMPLETE Column stats:
NONE
-              value expressions: _col2 (type: int)
+              value expressions: _col0 (type: int), _col3 (type: int)
           TableScan
             Reduce Output Operator
-              key expressions: _col0 (type: int)
-              sort order: +
-              Map-reduce partition columns: _col0 (type: int)
-              Statistics: Num rows: 50 Data size: 5999 Basic stats: COMPLETE Column stats:
NONE
+              key expressions: _col0 (type: int), _col1 (type: int)
+              sort order: ++
+              Map-reduce partition columns: _col0 (type: int), _col1 (type: int)
+              Statistics: Num rows: 30 Data size: 3629 Basic stats: COMPLETE Column stats:
NONE
       Reduce Operator Tree:
         Join Operator
           condition map:
                Inner Join 0 to 1
           keys:
-            0 _col1 (type: int)
-            1 _col0 (type: int)
-          outputColumnNames: _col2, _col4
+            0 _col1 (type: int), _col4 (type: int)
+            1 _col0 (type: int), _col1 (type: int)
+          outputColumnNames: _col0, _col3
           Statistics: Num rows: 60 Data size: 7257 Basic stats: COMPLETE Column stats: NONE
           Select Operator
-            expressions: _col4 (type: int), _col2 (type: int)
+            expressions: _col0 (type: int), _col3 (type: int)
             outputColumnNames: _col0, _col1
             Statistics: Num rows: 60 Data size: 7257 Basic stats: COMPLETE Column stats:
NONE
             File Output Operator
@@ -454,63 +526,157 @@ STAGE PLANS:
                   output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
                   serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
 
-  Stage: Stage-11
+  Stage: Stage-7
+    Map Reduce
+      Map Operator Tree:
+          TableScan
+            alias: lineitem
+            Statistics: Num rows: 100 Data size: 11999 Basic stats: COMPLETE Column stats:
NONE
+            Filter Operator
+              predicate: l_partkey is not null (type: boolean)
+              Statistics: Num rows: 100 Data size: 11999 Basic stats: COMPLETE Column stats:
NONE
+              Group By Operator
+                keys: l_partkey (type: int)
+                mode: hash
+                outputColumnNames: _col0
+                Statistics: Num rows: 100 Data size: 11999 Basic stats: COMPLETE Column stats:
NONE
+                Reduce Output Operator
+                  key expressions: _col0 (type: int)
+                  sort order: +
+                  Map-reduce partition columns: _col0 (type: int)
+                  Statistics: Num rows: 100 Data size: 11999 Basic stats: COMPLETE Column
stats: NONE
+      Reduce Operator Tree:
+        Group By Operator
+          keys: KEY._col0 (type: int)
+          mode: mergepartial
+          outputColumnNames: _col0
+          Statistics: Num rows: 50 Data size: 5999 Basic stats: COMPLETE Column stats: NONE
+          File Output Operator
+            compressed: false
+            table:
+                input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+                output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+                serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
+
+  Stage: Stage-21
     Map Reduce Local Work
       Alias -> Map Local Tables:
-        $hdt$_2:lineitem 
+        $hdt$_2:$hdt$_3:$hdt$_4:li 
           Fetch Operator
             limit: -1
       Alias -> Map Local Operator Tree:
-        $hdt$_2:lineitem 
+        $hdt$_2:$hdt$_3:$hdt$_4:li 
           TableScan
-            alias: lineitem
+            alias: li
             Statistics: Num rows: 100 Data size: 11999 Basic stats: COMPLETE Column stats:
NONE
             Filter Operator
-              predicate: ((l_shipmode = 'AIR') and (l_linenumber = 1) and l_orderkey is not
null) (type: boolean)
-              Statistics: Num rows: 25 Data size: 2999 Basic stats: COMPLETE Column stats:
NONE
+              predicate: l_partkey is not null (type: boolean)
+              Statistics: Num rows: 100 Data size: 11999 Basic stats: COMPLETE Column stats:
NONE
               Select Operator
-                expressions: l_orderkey (type: int), 1 (type: int)
+                expressions: l_partkey (type: int), l_linenumber (type: int)
                 outputColumnNames: _col0, _col1
-                Statistics: Num rows: 25 Data size: 2999 Basic stats: COMPLETE Column stats:
NONE
-                Group By Operator
-                  keys: _col0 (type: int), _col1 (type: int)
-                  mode: hash
-                  outputColumnNames: _col0, _col1
-                  Statistics: Num rows: 25 Data size: 2999 Basic stats: COMPLETE Column stats:
NONE
-                  HashTable Sink Operator
-                    keys:
-                      0 _col0 (type: int), 1 (type: int)
-                      1 _col0 (type: int), _col1 (type: int)
+                Statistics: Num rows: 100 Data size: 11999 Basic stats: COMPLETE Column stats:
NONE
+                HashTable Sink Operator
+                  keys:
+                    0 _col0 (type: int)
+                    1 _col0 (type: int)
 
-  Stage: Stage-8
+  Stage: Stage-9
     Map Reduce
       Map Operator Tree:
           TableScan
-            alias: li
+            Map Join Operator
+              condition map:
+                   Inner Join 0 to 1
+              keys:
+                0 _col0 (type: int)
+                1 _col0 (type: int)
+              outputColumnNames: _col2
+              Statistics: Num rows: 110 Data size: 13198 Basic stats: COMPLETE Column stats:
NONE
+              Group By Operator
+                keys: _col2 (type: int)
+                mode: hash
+                outputColumnNames: _col0
+                Statistics: Num rows: 110 Data size: 13198 Basic stats: COMPLETE Column stats:
NONE
+                Reduce Output Operator
+                  key expressions: _col0 (type: int)
+                  sort order: +
+                  Map-reduce partition columns: _col0 (type: int)
+                  Statistics: Num rows: 110 Data size: 13198 Basic stats: COMPLETE Column
stats: NONE
+      Local Work:
+        Map Reduce Local Work
+      Reduce Operator Tree:
+        Group By Operator
+          keys: KEY._col0 (type: int)
+          mode: mergepartial
+          outputColumnNames: _col0
+          Statistics: Num rows: 55 Data size: 6599 Basic stats: COMPLETE Column stats: NONE
+          File Output Operator
+            compressed: false
+            table:
+                input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+                output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+                serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
+
+  Stage: Stage-20
+    Map Reduce Local Work
+      Alias -> Map Local Tables:
+        $hdt$_2:$hdt$_2:lineitem 
+          Fetch Operator
+            limit: -1
+      Alias -> Map Local Operator Tree:
+        $hdt$_2:$hdt$_2:lineitem 
+          TableScan
+            alias: lineitem
             Statistics: Num rows: 100 Data size: 11999 Basic stats: COMPLETE Column stats:
NONE
             Filter Operator
-              predicate: ((l_linenumber = 1) and l_partkey is not null and l_orderkey is
not null) (type: boolean)
+              predicate: (l_shipmode = 'AIR') (type: boolean)
               Statistics: Num rows: 50 Data size: 5999 Basic stats: COMPLETE Column stats:
NONE
               Select Operator
-                expressions: l_orderkey (type: int), l_partkey (type: int), l_suppkey (type:
int)
-                outputColumnNames: _col0, _col1, _col2
+                expressions: l_orderkey (type: int), l_linenumber (type: int)
+                outputColumnNames: _col0, _col1
                 Statistics: Num rows: 50 Data size: 5999 Basic stats: COMPLETE Column stats:
NONE
-                Map Join Operator
-                  condition map:
-                       Left Semi Join 0 to 1
+                HashTable Sink Operator
                   keys:
-                    0 _col0 (type: int), 1 (type: int)
-                    1 _col0 (type: int), _col1 (type: int)
-                  outputColumnNames: _col1, _col2
-                  Statistics: Num rows: 55 Data size: 6598 Basic stats: COMPLETE Column stats:
NONE
-                  File Output Operator
-                    compressed: false
-                    table:
-                        input format: org.apache.hadoop.mapred.SequenceFileInputFormat
-                        output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
-                        serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
+                    0 _col1 (type: int)
+                    1 _col0 (type: int)
+
+  Stage: Stage-6
+    Map Reduce
+      Map Operator Tree:
+          TableScan
+            Map Join Operator
+              condition map:
+                   Inner Join 0 to 1
+              keys:
+                0 _col1 (type: int)
+                1 _col0 (type: int)
+              outputColumnNames: _col0, _col3
+              Statistics: Num rows: 60 Data size: 7258 Basic stats: COMPLETE Column stats:
NONE
+              Group By Operator
+                keys: _col0 (type: int), _col3 (type: int)
+                mode: hash
+                outputColumnNames: _col0, _col1
+                Statistics: Num rows: 60 Data size: 7258 Basic stats: COMPLETE Column stats:
NONE
+                Reduce Output Operator
+                  key expressions: _col0 (type: int), _col1 (type: int)
+                  sort order: ++
+                  Map-reduce partition columns: _col0 (type: int), _col1 (type: int)
+                  Statistics: Num rows: 60 Data size: 7258 Basic stats: COMPLETE Column stats:
NONE
       Local Work:
         Map Reduce Local Work
+      Reduce Operator Tree:
+        Group By Operator
+          keys: KEY._col0 (type: int), KEY._col1 (type: int)
+          mode: mergepartial
+          outputColumnNames: _col0, _col1
+          Statistics: Num rows: 30 Data size: 3629 Basic stats: COMPLETE Column stats: NONE
+          File Output Operator
+            compressed: false
+            table:
+                input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+                output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+                serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
 
   Stage: Stage-0
     Fetch Operator


Mime
View raw message