hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From hashut...@apache.org
Subject svn commit: r1439946 [9/9] - in /hive/trunk: common/src/java/org/apache/hadoop/hive/conf/ conf/ ql/src/java/org/apache/hadoop/hive/ql/ ql/src/java/org/apache/hadoop/hive/ql/exec/ ql/src/java/org/apache/hadoop/hive/ql/optimizer/ ql/src/java/org/apache/h...
Date Tue, 29 Jan 2013 15:33:55 GMT
Modified: hive/trunk/ql/src/test/results/clientpositive/sort_merge_join_desc_2.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/sort_merge_join_desc_2.q.out?rev=1439946&r1=1439945&r2=1439946&view=diff
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/sort_merge_join_desc_2.q.out (original)
+++ hive/trunk/ql/src/test/results/clientpositive/sort_merge_join_desc_2.q.out Tue Jan 29
15:33:53 2013
@@ -67,7 +67,6 @@ ABSTRACT SYNTAX TREE:
 
 STAGE DEPENDENCIES:
   Stage-1 is a root stage
-  Stage-2 depends on stages: Stage-1
   Stage-0 is a root stage
 
 STAGE PLANS:
@@ -93,35 +92,24 @@ STAGE PLANS:
                   1 [Column[key], Column[value]]
                 outputColumnNames: _col0
                 Position of Big Table: 0
-                File Output Operator
-                  compressed: false
-                  GlobalTableId: 0
-                  table:
-                      input format: org.apache.hadoop.mapred.SequenceFileInputFormat
-                      output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
-
-  Stage: Stage-2
-    Map Reduce
-      Alias -> Map Operator Tree:
-#### A masked pattern was here ####
-          Select Operator
-            expressions:
-                  expr: _col0
-                  type: string
-            outputColumnNames: _col0
-            Select Operator
-              Group By Operator
-                aggregations:
-                      expr: count()
-                bucketGroup: false
-                mode: hash
-                outputColumnNames: _col0
-                Reduce Output Operator
-                  sort order: 
-                  tag: -1
-                  value expressions:
+                Select Operator
+                  expressions:
                         expr: _col0
-                        type: bigint
+                        type: string
+                  outputColumnNames: _col0
+                  Select Operator
+                    Group By Operator
+                      aggregations:
+                            expr: count()
+                      bucketGroup: false
+                      mode: hash
+                      outputColumnNames: _col0
+                      Reduce Output Operator
+                        sort order: 
+                        tag: -1
+                        value expressions:
+                              expr: _col0
+                              type: bigint
       Reduce Operator Tree:
         Group By Operator
           aggregations:

Modified: hive/trunk/ql/src/test/results/clientpositive/sort_merge_join_desc_3.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/sort_merge_join_desc_3.q.out?rev=1439946&r1=1439945&r2=1439946&view=diff
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/sort_merge_join_desc_3.q.out (original)
+++ hive/trunk/ql/src/test/results/clientpositive/sort_merge_join_desc_3.q.out Tue Jan 29
15:33:53 2013
@@ -67,7 +67,6 @@ ABSTRACT SYNTAX TREE:
 
 STAGE DEPENDENCIES:
   Stage-1 is a root stage
-  Stage-2 depends on stages: Stage-1
   Stage-0 is a root stage
 
 STAGE PLANS:
@@ -93,35 +92,24 @@ STAGE PLANS:
                   1 [Column[key], Column[value]]
                 outputColumnNames: _col0
                 Position of Big Table: 0
-                File Output Operator
-                  compressed: false
-                  GlobalTableId: 0
-                  table:
-                      input format: org.apache.hadoop.mapred.SequenceFileInputFormat
-                      output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
-
-  Stage: Stage-2
-    Map Reduce
-      Alias -> Map Operator Tree:
-#### A masked pattern was here ####
-          Select Operator
-            expressions:
-                  expr: _col0
-                  type: string
-            outputColumnNames: _col0
-            Select Operator
-              Group By Operator
-                aggregations:
-                      expr: count()
-                bucketGroup: false
-                mode: hash
-                outputColumnNames: _col0
-                Reduce Output Operator
-                  sort order: 
-                  tag: -1
-                  value expressions:
+                Select Operator
+                  expressions:
                         expr: _col0
-                        type: bigint
+                        type: string
+                  outputColumnNames: _col0
+                  Select Operator
+                    Group By Operator
+                      aggregations:
+                            expr: count()
+                      bucketGroup: false
+                      mode: hash
+                      outputColumnNames: _col0
+                      Reduce Output Operator
+                        sort order: 
+                        tag: -1
+                        value expressions:
+                              expr: _col0
+                              type: bigint
       Reduce Operator Tree:
         Group By Operator
           aggregations:

Modified: hive/trunk/ql/src/test/results/clientpositive/sort_merge_join_desc_4.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/sort_merge_join_desc_4.q.out?rev=1439946&r1=1439945&r2=1439946&view=diff
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/sort_merge_join_desc_4.q.out (original)
+++ hive/trunk/ql/src/test/results/clientpositive/sort_merge_join_desc_4.q.out Tue Jan 29
15:33:53 2013
@@ -64,13 +64,12 @@ ABSTRACT SYNTAX TREE:
   (TOK_QUERY (TOK_FROM (TOK_JOIN (TOK_TABREF (TOK_TABNAME table_desc1) a) (TOK_TABREF (TOK_TABNAME
table_desc2) b) (and (= (. (TOK_TABLE_OR_COL a) key) (. (TOK_TABLE_OR_COL b) key)) (= (. (TOK_TABLE_OR_COL
a) value) (. (TOK_TABLE_OR_COL b) value))))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE))
(TOK_SELECT (TOK_HINTLIST (TOK_HINT TOK_MAPJOIN (TOK_HINTARGLIST b))) (TOK_SELEXPR (TOK_FUNCTIONSTAR
count))) (TOK_WHERE (< (. (TOK_TABLE_OR_COL a) key) 10))))
 
 STAGE DEPENDENCIES:
-  Stage-4 is a root stage
-  Stage-1 depends on stages: Stage-4
-  Stage-2 depends on stages: Stage-1
+  Stage-3 is a root stage
+  Stage-1 depends on stages: Stage-3
   Stage-0 is a root stage
 
 STAGE PLANS:
-  Stage: Stage-4
+  Stage: Stage-3
     Map Reduce Local Work
       Alias -> Map Local Tables:
         b 
@@ -116,37 +115,26 @@ STAGE PLANS:
                   1 [Column[key], Column[value]]
                 outputColumnNames: _col0
                 Position of Big Table: 0
-                File Output Operator
-                  compressed: false
-                  GlobalTableId: 0
-                  table:
-                      input format: org.apache.hadoop.mapred.SequenceFileInputFormat
-                      output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+                Select Operator
+                  expressions:
+                        expr: _col0
+                        type: string
+                  outputColumnNames: _col0
+                  Select Operator
+                    Group By Operator
+                      aggregations:
+                            expr: count()
+                      bucketGroup: false
+                      mode: hash
+                      outputColumnNames: _col0
+                      Reduce Output Operator
+                        sort order: 
+                        tag: -1
+                        value expressions:
+                              expr: _col0
+                              type: bigint
       Local Work:
         Map Reduce Local Work
-
-  Stage: Stage-2
-    Map Reduce
-      Alias -> Map Operator Tree:
-#### A masked pattern was here ####
-          Select Operator
-            expressions:
-                  expr: _col0
-                  type: string
-            outputColumnNames: _col0
-            Select Operator
-              Group By Operator
-                aggregations:
-                      expr: count()
-                bucketGroup: false
-                mode: hash
-                outputColumnNames: _col0
-                Reduce Output Operator
-                  sort order: 
-                  tag: -1
-                  value expressions:
-                        expr: _col0
-                        type: bigint
       Reduce Operator Tree:
         Group By Operator
           aggregations:

Modified: hive/trunk/ql/src/test/results/clientpositive/sort_merge_join_desc_5.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/sort_merge_join_desc_5.q.out?rev=1439946&r1=1439945&r2=1439946&view=diff
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/sort_merge_join_desc_5.q.out (original)
+++ hive/trunk/ql/src/test/results/clientpositive/sort_merge_join_desc_5.q.out Tue Jan 29
15:33:53 2013
@@ -71,7 +71,6 @@ ABSTRACT SYNTAX TREE:
 
 STAGE DEPENDENCIES:
   Stage-1 is a root stage
-  Stage-2 depends on stages: Stage-1
   Stage-0 is a root stage
 
 STAGE PLANS:
@@ -93,21 +92,20 @@ STAGE PLANS:
                 0 [Column[key]]
                 1 [Column[key]]
               Position of Big Table: 0
-              File Output Operator
-                compressed: false
-                GlobalTableId: 0
-#### A masked pattern was here ####
-                NumFilesPerFileSink: 1
-                table:
-                    input format: org.apache.hadoop.mapred.SequenceFileInputFormat
-                    output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
-                    properties:
-                      columns 
-                      columns.types 
-                      escape.delim \
-                TotalFiles: 1
-                GatherStats: false
-                MultiFileSpray: false
+              Select Operator
+                Select Operator
+                  Group By Operator
+                    aggregations:
+                          expr: count()
+                    bucketGroup: false
+                    mode: hash
+                    outputColumnNames: _col0
+                    Reduce Output Operator
+                      sort order: 
+                      tag: -1
+                      value expressions:
+                            expr: _col0
+                            type: bigint
       Needs Tagging: false
       Path -> Alias:
 #### A masked pattern was here ####
@@ -160,47 +158,6 @@ STAGE PLANS:
               serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
               name: default.srcbucket_mapjoin_part_1
             name: default.srcbucket_mapjoin_part_1
-      Truncated Path -> Alias:
-        /srcbucket_mapjoin_part_1/part=1 [a]
-
-  Stage: Stage-2
-    Map Reduce
-      Alias -> Map Operator Tree:
-#### A masked pattern was here ####
-          Select Operator
-            Select Operator
-              Group By Operator
-                aggregations:
-                      expr: count()
-                bucketGroup: false
-                mode: hash
-                outputColumnNames: _col0
-                Reduce Output Operator
-                  sort order: 
-                  tag: -1
-                  value expressions:
-                        expr: _col0
-                        type: bigint
-      Needs Tagging: false
-      Path -> Alias:
-#### A masked pattern was here ####
-      Path -> Partition:
-#### A masked pattern was here ####
-          Partition
-            base file name: -mr-10002
-            input format: org.apache.hadoop.mapred.SequenceFileInputFormat
-            output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
-            properties:
-              columns 
-              columns.types 
-              escape.delim \
-          
-              input format: org.apache.hadoop.mapred.SequenceFileInputFormat
-              output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
-              properties:
-                columns 
-                columns.types 
-                escape.delim \
       Reduce Operator Tree:
         Group By Operator
           aggregations:
@@ -231,7 +188,7 @@ STAGE PLANS:
               GatherStats: false
               MultiFileSpray: false
       Truncated Path -> Alias:
-#### A masked pattern was here ####
+        /srcbucket_mapjoin_part_1/part=1 [a]
 
   Stage: Stage-0
     Fetch Operator

Modified: hive/trunk/ql/src/test/results/clientpositive/sort_merge_join_desc_6.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/sort_merge_join_desc_6.q.out?rev=1439946&r1=1439945&r2=1439946&view=diff
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/sort_merge_join_desc_6.q.out (original)
+++ hive/trunk/ql/src/test/results/clientpositive/sort_merge_join_desc_6.q.out Tue Jan 29
15:33:53 2013
@@ -70,13 +70,12 @@ ABSTRACT SYNTAX TREE:
   (TOK_QUERY (TOK_FROM (TOK_JOIN (TOK_TABREF (TOK_TABNAME srcbucket_mapjoin_part_1) a) (TOK_TABREF
(TOK_TABNAME srcbucket_mapjoin_part_2) b) (AND (AND (= (. (TOK_TABLE_OR_COL a) key) (. (TOK_TABLE_OR_COL
b) key)) (= (. (TOK_TABLE_OR_COL a) part) '1')) (= (. (TOK_TABLE_OR_COL b) part) '1')))) (TOK_INSERT
(TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_HINTLIST (TOK_HINT TOK_MAPJOIN (TOK_HINTARGLIST
b))) (TOK_SELEXPR (TOK_FUNCTIONSTAR count)))))
 
 STAGE DEPENDENCIES:
-  Stage-4 is a root stage
-  Stage-1 depends on stages: Stage-4
-  Stage-2 depends on stages: Stage-1
+  Stage-3 is a root stage
+  Stage-1 depends on stages: Stage-3
   Stage-0 is a root stage
 
 STAGE PLANS:
-  Stage: Stage-4
+  Stage: Stage-3
     Map Reduce Local Work
       Alias -> Map Local Tables:
         b 
@@ -122,21 +121,20 @@ STAGE PLANS:
                 0 [Column[key]]
                 1 [Column[key]]
               Position of Big Table: 0
-              File Output Operator
-                compressed: false
-                GlobalTableId: 0
-#### A masked pattern was here ####
-                NumFilesPerFileSink: 1
-                table:
-                    input format: org.apache.hadoop.mapred.SequenceFileInputFormat
-                    output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
-                    properties:
-                      columns 
-                      columns.types 
-                      escape.delim \
-                TotalFiles: 1
-                GatherStats: false
-                MultiFileSpray: false
+              Select Operator
+                Select Operator
+                  Group By Operator
+                    aggregations:
+                          expr: count()
+                    bucketGroup: false
+                    mode: hash
+                    outputColumnNames: _col0
+                    Reduce Output Operator
+                      sort order: 
+                      tag: -1
+                      value expressions:
+                            expr: _col0
+                            type: bigint
       Local Work:
         Map Reduce Local Work
       Needs Tagging: false
@@ -191,47 +189,6 @@ STAGE PLANS:
               serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
               name: default.srcbucket_mapjoin_part_1
             name: default.srcbucket_mapjoin_part_1
-      Truncated Path -> Alias:
-        /srcbucket_mapjoin_part_1/part=1 [a]
-
-  Stage: Stage-2
-    Map Reduce
-      Alias -> Map Operator Tree:
-#### A masked pattern was here ####
-          Select Operator
-            Select Operator
-              Group By Operator
-                aggregations:
-                      expr: count()
-                bucketGroup: false
-                mode: hash
-                outputColumnNames: _col0
-                Reduce Output Operator
-                  sort order: 
-                  tag: -1
-                  value expressions:
-                        expr: _col0
-                        type: bigint
-      Needs Tagging: false
-      Path -> Alias:
-#### A masked pattern was here ####
-      Path -> Partition:
-#### A masked pattern was here ####
-          Partition
-            base file name: -mr-10002
-            input format: org.apache.hadoop.mapred.SequenceFileInputFormat
-            output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
-            properties:
-              columns 
-              columns.types 
-              escape.delim \
-          
-              input format: org.apache.hadoop.mapred.SequenceFileInputFormat
-              output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
-              properties:
-                columns 
-                columns.types 
-                escape.delim \
       Reduce Operator Tree:
         Group By Operator
           aggregations:
@@ -262,7 +219,7 @@ STAGE PLANS:
               GatherStats: false
               MultiFileSpray: false
       Truncated Path -> Alias:
-#### A masked pattern was here ####
+        /srcbucket_mapjoin_part_1/part=1 [a]
 
   Stage: Stage-0
     Fetch Operator

Modified: hive/trunk/ql/src/test/results/clientpositive/sort_merge_join_desc_7.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/sort_merge_join_desc_7.q.out?rev=1439946&r1=1439945&r2=1439946&view=diff
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/sort_merge_join_desc_7.q.out (original)
+++ hive/trunk/ql/src/test/results/clientpositive/sort_merge_join_desc_7.q.out Tue Jan 29
15:33:53 2013
@@ -134,13 +134,12 @@ ABSTRACT SYNTAX TREE:
   (TOK_QUERY (TOK_FROM (TOK_JOIN (TOK_TABREF (TOK_TABNAME srcbucket_mapjoin_part_1) a) (TOK_TABREF
(TOK_TABNAME srcbucket_mapjoin_part_2) b) (AND (AND (= (. (TOK_TABLE_OR_COL a) key) (. (TOK_TABLE_OR_COL
b) key)) (TOK_FUNCTION TOK_ISNOTNULL (. (TOK_TABLE_OR_COL a) part))) (TOK_FUNCTION TOK_ISNOTNULL
(. (TOK_TABLE_OR_COL b) part))))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT
(TOK_HINTLIST (TOK_HINT TOK_MAPJOIN (TOK_HINTARGLIST b))) (TOK_SELEXPR (TOK_FUNCTIONSTAR count)))))
 
 STAGE DEPENDENCIES:
-  Stage-4 is a root stage
-  Stage-1 depends on stages: Stage-4
-  Stage-2 depends on stages: Stage-1
+  Stage-3 is a root stage
+  Stage-1 depends on stages: Stage-3
   Stage-0 is a root stage
 
 STAGE PLANS:
-  Stage: Stage-4
+  Stage: Stage-3
     Map Reduce Local Work
       Alias -> Map Local Tables:
         b 
@@ -179,21 +178,20 @@ STAGE PLANS:
                 0 [Column[key]]
                 1 [Column[key]]
               Position of Big Table: 0
-              File Output Operator
-                compressed: false
-                GlobalTableId: 0
-#### A masked pattern was here ####
-                NumFilesPerFileSink: 1
-                table:
-                    input format: org.apache.hadoop.mapred.SequenceFileInputFormat
-                    output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
-                    properties:
-                      columns 
-                      columns.types 
-                      escape.delim \
-                TotalFiles: 1
-                GatherStats: false
-                MultiFileSpray: false
+              Select Operator
+                Select Operator
+                  Group By Operator
+                    aggregations:
+                          expr: count()
+                    bucketGroup: false
+                    mode: hash
+                    outputColumnNames: _col0
+                    Reduce Output Operator
+                      sort order: 
+                      tag: -1
+                      value expressions:
+                            expr: _col0
+                            type: bigint
       Local Work:
         Map Reduce Local Work
       Needs Tagging: false
@@ -294,48 +292,6 @@ STAGE PLANS:
               serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
               name: default.srcbucket_mapjoin_part_1
             name: default.srcbucket_mapjoin_part_1
-      Truncated Path -> Alias:
-        /srcbucket_mapjoin_part_1/part=1 [a]
-        /srcbucket_mapjoin_part_1/part=2 [a]
-
-  Stage: Stage-2
-    Map Reduce
-      Alias -> Map Operator Tree:
-#### A masked pattern was here ####
-          Select Operator
-            Select Operator
-              Group By Operator
-                aggregations:
-                      expr: count()
-                bucketGroup: false
-                mode: hash
-                outputColumnNames: _col0
-                Reduce Output Operator
-                  sort order: 
-                  tag: -1
-                  value expressions:
-                        expr: _col0
-                        type: bigint
-      Needs Tagging: false
-      Path -> Alias:
-#### A masked pattern was here ####
-      Path -> Partition:
-#### A masked pattern was here ####
-          Partition
-            base file name: -mr-10002
-            input format: org.apache.hadoop.mapred.SequenceFileInputFormat
-            output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
-            properties:
-              columns 
-              columns.types 
-              escape.delim \
-          
-              input format: org.apache.hadoop.mapred.SequenceFileInputFormat
-              output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
-              properties:
-                columns 
-                columns.types 
-                escape.delim \
       Reduce Operator Tree:
         Group By Operator
           aggregations:
@@ -366,7 +322,8 @@ STAGE PLANS:
               GatherStats: false
               MultiFileSpray: false
       Truncated Path -> Alias:
-#### A masked pattern was here ####
+        /srcbucket_mapjoin_part_1/part=1 [a]
+        /srcbucket_mapjoin_part_1/part=2 [a]
 
   Stage: Stage-0
     Fetch Operator

Modified: hive/trunk/ql/src/test/results/clientpositive/union22.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/union22.q.out?rev=1439946&r1=1439945&r2=1439946&view=diff
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/union22.q.out (original)
+++ hive/trunk/ql/src/test/results/clientpositive/union22.q.out Tue Jan 29 15:33:53 2013
@@ -42,26 +42,30 @@ POSTHOOK: Lineage: dst_union22_delta PAR
 POSTHOOK: Lineage: dst_union22_delta PARTITION(ds=1).k3 SIMPLE [(src)src.FieldSchema(name:key,
type:string, comment:default), ]
 POSTHOOK: Lineage: dst_union22_delta PARTITION(ds=1).k4 SIMPLE [(src)src.FieldSchema(name:value,
type:string, comment:default), ]
 POSTHOOK: Lineage: dst_union22_delta PARTITION(ds=1).k5 SIMPLE [(src)src.FieldSchema(name:value,
type:string, comment:default), ]
-PREHOOK: query: explain extended
+PREHOOK: query: -- Since the inputs are small, it should be automatically converted to mapjoin
+
+explain extended
 insert overwrite table dst_union22 partition (ds='2')
 select * from
 (
 select k1 as k1, k2 as k2, k3 as k3, k4 as k4 from dst_union22_delta where ds = '1' and k0
<= 50
 union all
-select /*+ MAPJOIN(b) */ a.k1 as k1, a.k2 as k2, b.k3 as k3, b.k4 as k4
+select a.k1 as k1, a.k2 as k2, b.k3 as k3, b.k4 as k4
 from dst_union22 a left outer join (select * from dst_union22_delta where ds = '1' and k0
> 50) b on
 a.k1 = b.k1 and a.ds='1'
 where a.k1 > 20
 )
 subq
 PREHOOK: type: QUERY
-POSTHOOK: query: explain extended
+POSTHOOK: query: -- Since the inputs are small, it should be automatically converted to mapjoin
+
+explain extended
 insert overwrite table dst_union22 partition (ds='2')
 select * from
 (
 select k1 as k1, k2 as k2, k3 as k3, k4 as k4 from dst_union22_delta where ds = '1' and k0
<= 50
 union all
-select /*+ MAPJOIN(b) */ a.k1 as k1, a.k2 as k2, b.k3 as k3, b.k4 as k4
+select a.k1 as k1, a.k2 as k2, b.k3 as k3, b.k4 as k4
 from dst_union22 a left outer join (select * from dst_union22_delta where ds = '1' and k0
> 50) b on
 a.k1 = b.k1 and a.ds='1'
 where a.k1 > 20
@@ -79,18 +83,22 @@ POSTHOOK: Lineage: dst_union22_delta PAR
 POSTHOOK: Lineage: dst_union22_delta PARTITION(ds=1).k4 SIMPLE [(src)src.FieldSchema(name:value,
type:string, comment:default), ]
 POSTHOOK: Lineage: dst_union22_delta PARTITION(ds=1).k5 SIMPLE [(src)src.FieldSchema(name:value,
type:string, comment:default), ]
 ABSTRACT SYNTAX TREE:
-  (TOK_QUERY (TOK_FROM (TOK_SUBQUERY (TOK_UNION (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME
dst_union22_delta))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR
(TOK_TABLE_OR_COL k1) k1) (TOK_SELEXPR (TOK_TABLE_OR_COL k2) k2) (TOK_SELEXPR (TOK_TABLE_OR_COL
k3) k3) (TOK_SELEXPR (TOK_TABLE_OR_COL k4) k4)) (TOK_WHERE (and (= (TOK_TABLE_OR_COL ds) '1')
(<= (TOK_TABLE_OR_COL k0) 50))))) (TOK_QUERY (TOK_FROM (TOK_LEFTOUTERJOIN (TOK_TABREF (TOK_TABNAME
dst_union22) a) (TOK_SUBQUERY (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME dst_union22_delta)))
(TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR TOK_ALLCOLREF))
(TOK_WHERE (and (= (TOK_TABLE_OR_COL ds) '1') (> (TOK_TABLE_OR_COL k0) 50))))) b) (and
(= (. (TOK_TABLE_OR_COL a) k1) (. (TOK_TABLE_OR_COL b) k1)) (= (. (TOK_TABLE_OR_COL a) ds)
'1')))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_HINTLIST (TOK_HINT
TOK_MAPJOIN (TOK_HINTARGLIST b))) (TOK_SE
 LEXPR (. (TOK_TABLE_OR_COL a) k1) k1) (TOK_SELEXPR (. (TOK_TABLE_OR_COL a) k2) k2) (TOK_SELEXPR
(. (TOK_TABLE_OR_COL b) k3) k3) (TOK_SELEXPR (. (TOK_TABLE_OR_COL b) k4) k4)) (TOK_WHERE (>
(. (TOK_TABLE_OR_COL a) k1) 20))))) subq)) (TOK_INSERT (TOK_DESTINATION (TOK_TAB (TOK_TABNAME
dst_union22) (TOK_PARTSPEC (TOK_PARTVAL ds '2')))) (TOK_SELECT (TOK_SELEXPR TOK_ALLCOLREF))))
+  (TOK_QUERY (TOK_FROM (TOK_SUBQUERY (TOK_UNION (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME
dst_union22_delta))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR
(TOK_TABLE_OR_COL k1) k1) (TOK_SELEXPR (TOK_TABLE_OR_COL k2) k2) (TOK_SELEXPR (TOK_TABLE_OR_COL
k3) k3) (TOK_SELEXPR (TOK_TABLE_OR_COL k4) k4)) (TOK_WHERE (and (= (TOK_TABLE_OR_COL ds) '1')
(<= (TOK_TABLE_OR_COL k0) 50))))) (TOK_QUERY (TOK_FROM (TOK_LEFTOUTERJOIN (TOK_TABREF (TOK_TABNAME
dst_union22) a) (TOK_SUBQUERY (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME dst_union22_delta)))
(TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR TOK_ALLCOLREF))
(TOK_WHERE (and (= (TOK_TABLE_OR_COL ds) '1') (> (TOK_TABLE_OR_COL k0) 50))))) b) (and
(= (. (TOK_TABLE_OR_COL a) k1) (. (TOK_TABLE_OR_COL b) k1)) (= (. (TOK_TABLE_OR_COL a) ds)
'1')))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (. (TOK_TABLE_OR_COL
a) k1) k1) (TOK_SELEXPR (. (TOK
 _TABLE_OR_COL a) k2) k2) (TOK_SELEXPR (. (TOK_TABLE_OR_COL b) k3) k3) (TOK_SELEXPR (. (TOK_TABLE_OR_COL
b) k4) k4)) (TOK_WHERE (> (. (TOK_TABLE_OR_COL a) k1) 20))))) subq)) (TOK_INSERT (TOK_DESTINATION
(TOK_TAB (TOK_TABNAME dst_union22) (TOK_PARTSPEC (TOK_PARTVAL ds '2')))) (TOK_SELECT (TOK_SELEXPR
TOK_ALLCOLREF))))
 
 STAGE DEPENDENCIES:
-  Stage-7 is a root stage
-  Stage-1 depends on stages: Stage-7
-  Stage-2 depends on stages: Stage-1
-  Stage-3 depends on stages: Stage-2
-  Stage-0 depends on stages: Stage-3
-  Stage-4 depends on stages: Stage-0
+  Stage-7 is a root stage , consists of Stage-8, Stage-1
+  Stage-8 has a backup stage: Stage-1
+  Stage-6 depends on stages: Stage-8
+  Stage-2 depends on stages: Stage-1, Stage-6
+  Stage-0 depends on stages: Stage-2
+  Stage-3 depends on stages: Stage-0
+  Stage-1
 
 STAGE PLANS:
   Stage: Stage-7
+    Conditional Operator
+
+  Stage: Stage-8
     Map Reduce Local Work
       Alias -> Map Local Tables:
         null-subquery2:subq-subquery2:b:dst_union22_delta 
@@ -130,7 +138,7 @@ STAGE PLANS:
                     1 [Column[_col1]]
                   Position of Big Table: 0
 
-  Stage: Stage-1
+  Stage: Stage-6
     Map Reduce
       Alias -> Map Operator Tree:
         null-subquery2:subq-subquery2:a 
@@ -159,24 +167,35 @@ STAGE PLANS:
                   1 [Column[_col1]]
                 outputColumnNames: _col0, _col1, _col10, _col11
                 Position of Big Table: 0
-                File Output Operator
-                  compressed: false
-                  GlobalTableId: 0
-#### A masked pattern was here ####
-                  NumFilesPerFileSink: 1
-                  table:
-                      input format: org.apache.hadoop.mapred.SequenceFileInputFormat
-                      output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
-                      properties:
-                        columns _col0,_col1,_col10,_col11
-                        columns.types string,string,string,string
-                        escape.delim \
-                  TotalFiles: 1
-                  GatherStats: false
-                  MultiFileSpray: false
+                Select Operator
+                  expressions:
+                        expr: _col0
+                        type: string
+                        expr: _col1
+                        type: string
+                        expr: _col10
+                        type: string
+                        expr: _col11
+                        type: string
+                  outputColumnNames: _col0, _col1, _col2, _col3
+                  File Output Operator
+                    compressed: false
+                    GlobalTableId: 0
+#### A masked pattern was here ####
+                    NumFilesPerFileSink: 1
+                    table:
+                        input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+                        output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+                        properties:
+                          columns _col0,_col1,_col2,_col3
+                          columns.types string,string,string,string
+                          escape.delim \
+                    TotalFiles: 1
+                    GatherStats: false
+                    MultiFileSpray: false
       Local Work:
         Map Reduce Local Work
-      Needs Tagging: false
+      Needs Tagging: true
       Path -> Alias:
 #### A masked pattern was here ####
       Path -> Partition:
@@ -225,74 +244,55 @@ STAGE PLANS:
               serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
               name: default.dst_union22
             name: default.dst_union22
-      Truncated Path -> Alias:
-        /dst_union22/ds=1 [null-subquery2:subq-subquery2:a]
-
-  Stage: Stage-2
-    Map Reduce
-      Alias -> Map Operator Tree:
-#### A masked pattern was here ####
-          Select Operator
-            expressions:
-                  expr: _col0
-                  type: string
-                  expr: _col1
-                  type: string
-                  expr: _col10
-                  type: string
-                  expr: _col11
-                  type: string
-            outputColumnNames: _col0, _col1, _col10, _col11
-            Select Operator
-              expressions:
-                    expr: _col0
-                    type: string
-                    expr: _col1
-                    type: string
-                    expr: _col10
-                    type: string
-                    expr: _col11
-                    type: string
-              outputColumnNames: _col0, _col1, _col2, _col3
-              File Output Operator
-                compressed: false
-                GlobalTableId: 0
-#### A masked pattern was here ####
-                NumFilesPerFileSink: 1
-                table:
-                    input format: org.apache.hadoop.mapred.SequenceFileInputFormat
-                    output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
-                    properties:
-                      columns _col0,_col1,_col2,_col3
-                      columns.types string,string,string,string
-                      escape.delim \
-                TotalFiles: 1
-                GatherStats: false
-                MultiFileSpray: false
-      Needs Tagging: false
-      Path -> Alias:
-#### A masked pattern was here ####
-      Path -> Partition:
 #### A masked pattern was here ####
           Partition
-            base file name: -mr-10002
-            input format: org.apache.hadoop.mapred.SequenceFileInputFormat
-            output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+            base file name: ds=1
+            input format: org.apache.hadoop.mapred.TextInputFormat
+            output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+            partition values:
+              ds 1
             properties:
-              columns _col0,_col1,_col10,_col11
-              columns.types string,string,string,string
-              escape.delim \
+              bucket_count -1
+              columns k0,k1,k2,k3,k4,k5
+              columns.types string:string:string:string:string:string
+#### A masked pattern was here ####
+              name default.dst_union22_delta
+              numFiles 1
+              numRows 500
+              partition_columns ds
+              rawDataSize 16936
+              serialization.ddl struct dst_union22_delta { string k0, string k1, string k2,
string k3, string k4, string k5}
+              serialization.format 1
+              serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+              totalSize 17436
+#### A masked pattern was here ####
+            serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
           
-              input format: org.apache.hadoop.mapred.SequenceFileInputFormat
-              output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+              input format: org.apache.hadoop.mapred.TextInputFormat
+              output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
               properties:
-                columns _col0,_col1,_col10,_col11
-                columns.types string,string,string,string
-                escape.delim \
-      Truncated Path -> Alias:
+                bucket_count -1
+                columns k0,k1,k2,k3,k4,k5
+                columns.types string:string:string:string:string:string
 #### A masked pattern was here ####
+                name default.dst_union22_delta
+                numFiles 1
+                numPartitions 1
+                numRows 500
+                partition_columns ds
+                rawDataSize 16936
+                serialization.ddl struct dst_union22_delta { string k0, string k1, string
k2, string k3, string k4, string k5}
+                serialization.format 1
+                serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+                totalSize 17436
+#### A masked pattern was here ####
+              serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+              name: default.dst_union22_delta
+            name: default.dst_union22_delta
+      Truncated Path -> Alias:
+        /dst_union22/ds=1 [null-subquery2:subq-subquery2:a]
 
-  Stage: Stage-3
+  Stage: Stage-2
     Map Reduce
       Alias -> Map Operator Tree:
 #### A masked pattern was here ####
@@ -410,7 +410,7 @@ STAGE PLANS:
       Path -> Partition:
 #### A masked pattern was here ####
           Partition
-            base file name: -mr-10003
+            base file name: -mr-10002
             input format: org.apache.hadoop.mapred.SequenceFileInputFormat
             output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
             properties:
@@ -503,17 +503,215 @@ STAGE PLANS:
               name: default.dst_union22
 #### A masked pattern was here ####
 
-  Stage: Stage-4
+  Stage: Stage-3
     Stats-Aggr Operator
 #### A masked pattern was here ####
 
+  Stage: Stage-1
+    Map Reduce
+      Alias -> Map Operator Tree:
+        null-subquery2:subq-subquery2:a 
+          TableScan
+            alias: a
+            GatherStats: false
+            Filter Operator
+              isSamplingPred: false
+              predicate:
+                  expr: (k1 > 20.0)
+                  type: boolean
+              Reduce Output Operator
+                key expressions:
+                      expr: k1
+                      type: string
+                sort order: +
+                Map-reduce partition columns:
+                      expr: k1
+                      type: string
+                tag: 0
+                value expressions:
+                      expr: k1
+                      type: string
+                      expr: k2
+                      type: string
+                      expr: ds
+                      type: string
+        null-subquery2:subq-subquery2:b:dst_union22_delta 
+          TableScan
+            alias: dst_union22_delta
+            GatherStats: false
+            Filter Operator
+              isSamplingPred: false
+              predicate:
+                  expr: ((k0 > 50.0) and (k1 > 20.0))
+                  type: boolean
+              Select Operator
+                expressions:
+                      expr: k1
+                      type: string
+                      expr: k3
+                      type: string
+                      expr: k4
+                      type: string
+                outputColumnNames: _col1, _col3, _col4
+                Reduce Output Operator
+                  key expressions:
+                        expr: _col1
+                        type: string
+                  sort order: +
+                  Map-reduce partition columns:
+                        expr: _col1
+                        type: string
+                  tag: 1
+                  value expressions:
+                        expr: _col3
+                        type: string
+                        expr: _col4
+                        type: string
+      Needs Tagging: true
+      Path -> Alias:
+#### A masked pattern was here ####
+      Path -> Partition:
+#### A masked pattern was here ####
+          Partition
+            base file name: ds=1
+            input format: org.apache.hadoop.mapred.TextInputFormat
+            output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+            partition values:
+              ds 1
+            properties:
+              bucket_count -1
+              columns k1,k2,k3,k4
+              columns.types string:string:string:string
+#### A masked pattern was here ####
+              name default.dst_union22
+              numFiles 1
+              numRows 500
+              partition_columns ds
+              rawDataSize 11124
+              serialization.ddl struct dst_union22 { string k1, string k2, string k3, string
k4}
+              serialization.format 1
+              serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+              totalSize 11624
+#### A masked pattern was here ####
+            serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+          
+              input format: org.apache.hadoop.mapred.TextInputFormat
+              output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+              properties:
+                bucket_count -1
+                columns k1,k2,k3,k4
+                columns.types string:string:string:string
+#### A masked pattern was here ####
+                name default.dst_union22
+                numFiles 1
+                numPartitions 1
+                numRows 500
+                partition_columns ds
+                rawDataSize 11124
+                serialization.ddl struct dst_union22 { string k1, string k2, string k3, string
k4}
+                serialization.format 1
+                serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+                totalSize 11624
+#### A masked pattern was here ####
+              serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+              name: default.dst_union22
+            name: default.dst_union22
+#### A masked pattern was here ####
+          Partition
+            base file name: ds=1
+            input format: org.apache.hadoop.mapred.TextInputFormat
+            output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+            partition values:
+              ds 1
+            properties:
+              bucket_count -1
+              columns k0,k1,k2,k3,k4,k5
+              columns.types string:string:string:string:string:string
+#### A masked pattern was here ####
+              name default.dst_union22_delta
+              numFiles 1
+              numRows 500
+              partition_columns ds
+              rawDataSize 16936
+              serialization.ddl struct dst_union22_delta { string k0, string k1, string k2,
string k3, string k4, string k5}
+              serialization.format 1
+              serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+              totalSize 17436
+#### A masked pattern was here ####
+            serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+          
+              input format: org.apache.hadoop.mapred.TextInputFormat
+              output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+              properties:
+                bucket_count -1
+                columns k0,k1,k2,k3,k4,k5
+                columns.types string:string:string:string:string:string
+#### A masked pattern was here ####
+                name default.dst_union22_delta
+                numFiles 1
+                numPartitions 1
+                numRows 500
+                partition_columns ds
+                rawDataSize 16936
+                serialization.ddl struct dst_union22_delta { string k0, string k1, string
k2, string k3, string k4, string k5}
+                serialization.format 1
+                serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+                totalSize 17436
+#### A masked pattern was here ####
+              serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+              name: default.dst_union22_delta
+            name: default.dst_union22_delta
+      Reduce Operator Tree:
+        Join Operator
+          condition map:
+               Left Outer Join0 to 1
+          condition expressions:
+            0 {VALUE._col0} {VALUE._col1}
+            1 {VALUE._col3} {VALUE._col4}
+          filter mappings:
+            0 [1, 1]
+          filter predicates:
+            0 {(VALUE._col4 = '1')}
+            1 
+          handleSkewJoin: false
+          outputColumnNames: _col0, _col1, _col10, _col11
+          Select Operator
+            expressions:
+                  expr: _col0
+                  type: string
+                  expr: _col1
+                  type: string
+                  expr: _col10
+                  type: string
+                  expr: _col11
+                  type: string
+            outputColumnNames: _col0, _col1, _col2, _col3
+            File Output Operator
+              compressed: false
+              GlobalTableId: 0
+#### A masked pattern was here ####
+              NumFilesPerFileSink: 1
+              table:
+                  input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+                  output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+                  properties:
+                    columns _col0,_col1,_col2,_col3
+                    columns.types string,string,string,string
+                    escape.delim \
+              TotalFiles: 1
+              GatherStats: false
+              MultiFileSpray: false
+      Truncated Path -> Alias:
+        /dst_union22/ds=1 [null-subquery2:subq-subquery2:a]
+        /dst_union22_delta/ds=1 [null-subquery2:subq-subquery2:b:dst_union22_delta]
+
 
 PREHOOK: query: insert overwrite table dst_union22 partition (ds='2')
 select * from
 (
 select k1 as k1, k2 as k2, k3 as k3, k4 as k4 from dst_union22_delta where ds = '1' and k0
<= 50
 union all
-select /*+ MAPJOIN(b) */ a.k1 as k1, a.k2 as k2, b.k3 as k3, b.k4 as k4
+select a.k1 as k1, a.k2 as k2, b.k3 as k3, b.k4 as k4
 from dst_union22 a left outer join (select * from dst_union22_delta where ds = '1' and k0
> 50) b on
 a.k1 = b.k1 and a.ds='1'
 where a.k1 > 20
@@ -530,7 +728,7 @@ select * from
 (
 select k1 as k1, k2 as k2, k3 as k3, k4 as k4 from dst_union22_delta where ds = '1' and k0
<= 50
 union all
-select /*+ MAPJOIN(b) */ a.k1 as k1, a.k2 as k2, b.k3 as k3, b.k4 as k4
+select a.k1 as k1, a.k2 as k2, b.k3 as k3, b.k4 as k4
 from dst_union22 a left outer join (select * from dst_union22_delta where ds = '1' and k0
> 50) b on
 a.k1 = b.k1 and a.ds='1'
 where a.k1 > 20



Mime
View raw message