hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From na...@apache.org
Subject svn commit: r891936 - in /hadoop/hive/trunk: CHANGES.txt ql/src/java/org/apache/hadoop/hive/ql/optimizer/MapJoinProcessor.java ql/src/test/queries/clientpositive/join_map_ppr.q ql/src/test/results/clientpositive/join_map_ppr.q.out
Date Thu, 17 Dec 2009 21:53:30 GMT
Author: namit
Date: Thu Dec 17 21:53:29 2009
New Revision: 891936

URL: http://svn.apache.org/viewvc?rev=891936&view=rev
Log:
HIVE-993 Remove duplicate conversion for MapJoin
(He Yongqiang via namit)


Modified:
    hadoop/hive/trunk/CHANGES.txt
    hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/MapJoinProcessor.java
    hadoop/hive/trunk/ql/src/test/queries/clientpositive/join_map_ppr.q
    hadoop/hive/trunk/ql/src/test/results/clientpositive/join_map_ppr.q.out

Modified: hadoop/hive/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/CHANGES.txt?rev=891936&r1=891935&r2=891936&view=diff
==============================================================================
--- hadoop/hive/trunk/CHANGES.txt (original)
+++ hadoop/hive/trunk/CHANGES.txt Thu Dec 17 21:53:29 2009
@@ -324,6 +324,9 @@
     HIVE-991 union with 200 kids fail
     (namit via He Yongqiang)
 
+    HIVE-993 Remove duplicate conversion for MapJoin
+    (He Yongqiang via namit)
+
 Release 0.4.0 -  Unreleased
 
   INCOMPATIBLE CHANGES

Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/MapJoinProcessor.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/MapJoinProcessor.java?rev=891936&r1=891935&r2=891936&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/MapJoinProcessor.java
(original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/MapJoinProcessor.java
Thu Dec 17 21:53:29 2009
@@ -145,8 +145,6 @@
       pos++;
     }
 
-    int keyLength = 0;
-    
     //get the join keys from old parent ReduceSink operators
     for (pos = 0; pos < newParentOps.size(); pos++) {
       ReduceSinkOperator oldPar = (ReduceSinkOperator)oldReduceSinkParentOps.get(pos);
@@ -191,27 +189,6 @@
       valueExprMap.put(new Byte((byte)pos), values);      
     }
 
-    // implicit type conversion hierarchy
-    for (int k = 0; k < keyLength; k++) {
-      // Find the common class for type conversion
-      TypeInfo commonType = keyExprMap.get(new Byte((byte)0)).get(k).getTypeInfo();
-      for (int i=1; i < newParentOps.size(); i++) {
-        TypeInfo a = commonType;
-        TypeInfo b = keyExprMap.get(new Byte((byte)i)).get(k).getTypeInfo(); 
-        commonType = FunctionRegistry.getCommonClassForComparison(a, b);
-        if (commonType == null) {
-          throw new SemanticException("Cannot do equality join on different types: " + a.getTypeName()
+ " and " + b.getTypeName());
-        }
-      }
-      
-      // Add implicit type conversion if necessary
-      for (int i=0; i < newParentOps.size(); i++) {
-        if (!commonType.equals(keyExprMap.get(new Byte((byte)i)).get(k).getTypeInfo())) {
-          keyExprMap.get(new Byte((byte)i)).set(k, TypeCheckProcFactory.DefaultExprProcessor.getFuncExprNodeDesc(commonType.getTypeName(),
keyExprMap.get(new Byte((byte)i)).get(k)));
-        }
-      }
-    }
-    
     org.apache.hadoop.hive.ql.plan.joinCond[] joinCondns = op.getConf().getConds();
 
     Operator[] newPar = new Operator[newParentOps.size()];

Modified: hadoop/hive/trunk/ql/src/test/queries/clientpositive/join_map_ppr.q
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/queries/clientpositive/join_map_ppr.q?rev=891936&r1=891935&r2=891936&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/test/queries/clientpositive/join_map_ppr.q (original)
+++ hadoop/hive/trunk/ql/src/test/queries/clientpositive/join_map_ppr.q Thu Dec 17 21:53:29
2009
@@ -15,5 +15,27 @@
 
 select * from dest_j1 x order by x.key;
 
+CREATE TABLE src_copy(key int, value string);
+CREATE TABLE src1_copy(key string, value string);
+INSERT OVERWRITE TABLE src_copy select key, value from src;
+INSERT OVERWRITE TABLE src1_copy select key, value from src1;
+
+EXPLAIN EXTENDED
+INSERT OVERWRITE TABLE dest_j1
+SELECT /*+ MAPJOIN(x,y) */ x.key, z.value, y.value
+FROM src1_copy x JOIN src_copy y ON (x.key = y.key) 
+JOIN srcpart z ON (x.key = z.key)
+WHERE z.ds='2008-04-08' and z.hr=11;
+
+INSERT OVERWRITE TABLE dest_j1
+SELECT /*+ MAPJOIN(x,y) */ x.key, z.value, y.value
+FROM src1_copy x JOIN src_copy y ON (x.key = y.key) 
+JOIN srcpart z ON (x.key = z.key)
+WHERE z.ds='2008-04-08' and z.hr=11;
+
+select * from dest_j1 x order by x.key;
+
+drop table src_copy;
+drop table src1_copy;
 drop table dest_j1;
 

Modified: hadoop/hive/trunk/ql/src/test/results/clientpositive/join_map_ppr.q.out
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/results/clientpositive/join_map_ppr.q.out?rev=891936&r1=891935&r2=891936&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/test/results/clientpositive/join_map_ppr.q.out (original)
+++ hadoop/hive/trunk/ql/src/test/results/clientpositive/join_map_ppr.q.out Thu Dec 17 21:53:29
2009
@@ -81,7 +81,7 @@
                       File Output Operator
                         compressed: false
                         GlobalTableId: 1
-                        directory: file:/Users/heyongqiang/Documents/workspace/Hive-Test/build/ql/tmp/489986805/10002
+                        directory: file:/Users/heyongqiang/Documents/workspace/Hive-Test/build/ql/tmp/199036060/10002
                         table:
                             input format: org.apache.hadoop.mapred.TextInputFormat
                             output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
@@ -96,7 +96,7 @@
                               file.inputformat org.apache.hadoop.mapred.TextInputFormat
                               file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
                               location file:/Users/heyongqiang/Documents/workspace/Hive-Test/build/ql/test/data/warehouse/dest_j1
-                              transient_lastDdlTime 1258011142
+                              transient_lastDdlTime 1260991796
                             serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
                             name: dest_j1
       Local Work:
@@ -156,7 +156,7 @@
                         File Output Operator
                           compressed: false
                           GlobalTableId: 1
-                          directory: file:/Users/heyongqiang/Documents/workspace/Hive-Test/build/ql/tmp/489986805/10002
+                          directory: file:/Users/heyongqiang/Documents/workspace/Hive-Test/build/ql/tmp/199036060/10002
                           table:
                               input format: org.apache.hadoop.mapred.TextInputFormat
                               output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
@@ -171,7 +171,7 @@
                                 file.inputformat org.apache.hadoop.mapred.TextInputFormat
                                 file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
                                 location file:/Users/heyongqiang/Documents/workspace/Hive-Test/build/ql/test/data/warehouse/dest_j1
-                                transient_lastDdlTime 1258011142
+                                transient_lastDdlTime 1260991796
                               serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
                               name: dest_j1
             x 
@@ -221,7 +221,7 @@
                         File Output Operator
                           compressed: false
                           GlobalTableId: 1
-                          directory: file:/Users/heyongqiang/Documents/workspace/Hive-Test/build/ql/tmp/489986805/10002
+                          directory: file:/Users/heyongqiang/Documents/workspace/Hive-Test/build/ql/tmp/199036060/10002
                           table:
                               input format: org.apache.hadoop.mapred.TextInputFormat
                               output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
@@ -236,7 +236,7 @@
                                 file.inputformat org.apache.hadoop.mapred.TextInputFormat
                                 file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
                                 location file:/Users/heyongqiang/Documents/workspace/Hive-Test/build/ql/test/data/warehouse/dest_j1
-                                transient_lastDdlTime 1258011142
+                                transient_lastDdlTime 1260991796
                               serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
                               name: dest_j1
       Needs Tagging: false
@@ -261,8 +261,8 @@
               serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
               file.inputformat org.apache.hadoop.mapred.TextInputFormat
               file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
-              location file:/Users/heyongqiang/Documents/workspace/Hive-Test/build/ql/test/data/warehouse/srcpart/ds=2008-04-08/hr=11
-              transient_lastDdlTime 1258011140
+              location file:/Users/heyongqiang/Documents/workspace/Hive-Test/build/ql/test/data/warehouse/srcpart
+              transient_lastDdlTime 1260991793
             serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
           
               input format: org.apache.hadoop.mapred.TextInputFormat
@@ -279,7 +279,7 @@
                 file.inputformat org.apache.hadoop.mapred.TextInputFormat
                 file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
                 location file:/Users/heyongqiang/Documents/workspace/Hive-Test/build/ql/test/data/warehouse/srcpart
-                transient_lastDdlTime 1258011139
+                transient_lastDdlTime 1260991793
               serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
               name: srcpart
             name: srcpart
@@ -290,11 +290,11 @@
           Move Operator
             files:
                 hdfs directory: true
-                source: file:/Users/heyongqiang/Documents/workspace/Hive-Test/build/ql/tmp/489986805/10002
-                destination: file:/Users/heyongqiang/Documents/workspace/Hive-Test/build/ql/tmp/379610489/10000
+                source: file:/Users/heyongqiang/Documents/workspace/Hive-Test/build/ql/tmp/199036060/10002
+                destination: file:/Users/heyongqiang/Documents/workspace/Hive-Test/build/ql/tmp/209835084/10000
           Map Reduce
             Alias -> Map Operator Tree:
-              file:/Users/heyongqiang/Documents/workspace/Hive-Test/build/ql/tmp/489986805/10002

+              file:/Users/heyongqiang/Documents/workspace/Hive-Test/build/ql/tmp/199036060/10002

                   Reduce Output Operator
                     sort order: 
                     Map-reduce partition columns:
@@ -310,9 +310,9 @@
                           type: string
             Needs Tagging: false
             Path -> Alias:
-              file:/Users/heyongqiang/Documents/workspace/Hive-Test/build/ql/tmp/489986805/10002
[file:/Users/heyongqiang/Documents/workspace/Hive-Test/build/ql/tmp/489986805/10002]
+              file:/Users/heyongqiang/Documents/workspace/Hive-Test/build/ql/tmp/199036060/10002
[file:/Users/heyongqiang/Documents/workspace/Hive-Test/build/ql/tmp/199036060/10002]
             Path -> Partition:
-              file:/Users/heyongqiang/Documents/workspace/Hive-Test/build/ql/tmp/489986805/10002

+              file:/Users/heyongqiang/Documents/workspace/Hive-Test/build/ql/tmp/199036060/10002

                 Partition
                   input format: org.apache.hadoop.mapred.TextInputFormat
                   output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
@@ -327,7 +327,7 @@
                     file.inputformat org.apache.hadoop.mapred.TextInputFormat
                     file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
                     location file:/Users/heyongqiang/Documents/workspace/Hive-Test/build/ql/test/data/warehouse/dest_j1
-                    transient_lastDdlTime 1258011142
+                    transient_lastDdlTime 1260991796
                   serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
                 
                     input format: org.apache.hadoop.mapred.TextInputFormat
@@ -343,7 +343,7 @@
                       file.inputformat org.apache.hadoop.mapred.TextInputFormat
                       file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
                       location file:/Users/heyongqiang/Documents/workspace/Hive-Test/build/ql/test/data/warehouse/dest_j1
-                      transient_lastDdlTime 1258011142
+                      transient_lastDdlTime 1260991796
                     serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
                     name: dest_j1
                   name: dest_j1
@@ -352,7 +352,7 @@
                 File Output Operator
                   compressed: false
                   GlobalTableId: 0
-                  directory: file:/Users/heyongqiang/Documents/workspace/Hive-Test/build/ql/tmp/379610489/10000
+                  directory: file:/Users/heyongqiang/Documents/workspace/Hive-Test/build/ql/tmp/209835084/10000
                   table:
                       input format: org.apache.hadoop.mapred.TextInputFormat
                       output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
@@ -367,7 +367,7 @@
                         file.inputformat org.apache.hadoop.mapred.TextInputFormat
                         location file:/Users/heyongqiang/Documents/workspace/Hive-Test/build/ql/test/data/warehouse/dest_j1
                         file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
-                        transient_lastDdlTime 1258011142
+                        transient_lastDdlTime 1260991796
                       serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
                       name: dest_j1
 
@@ -375,7 +375,7 @@
     Move Operator
       tables:
           replace: true
-          source: file:/Users/heyongqiang/Documents/workspace/Hive-Test/build/ql/tmp/379610489/10000
+          source: file:/Users/heyongqiang/Documents/workspace/Hive-Test/build/ql/tmp/209835084/10000
           table:
               input format: org.apache.hadoop.mapred.TextInputFormat
               output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
@@ -390,10 +390,10 @@
                 file.inputformat org.apache.hadoop.mapred.TextInputFormat
                 file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
                 location file:/Users/heyongqiang/Documents/workspace/Hive-Test/build/ql/test/data/warehouse/dest_j1
-                transient_lastDdlTime 1258011142
+                transient_lastDdlTime 1260991796
               serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
               name: dest_j1
-          tmp directory: file:/Users/heyongqiang/Documents/workspace/Hive-Test/build/ql/tmp/379610489/10001
+          tmp directory: file:/Users/heyongqiang/Documents/workspace/Hive-Test/build/ql/tmp/209835084/10001
 
 
 PREHOOK: query: INSERT OVERWRITE TABLE dest_j1
@@ -419,11 +419,11 @@
 PREHOOK: query: select * from dest_j1 x order by x.key
 PREHOOK: type: QUERY
 PREHOOK: Input: default@dest_j1
-PREHOOK: Output: file:/Users/heyongqiang/Documents/workspace/Hive-Test/build/ql/tmp/18999706/10000
+PREHOOK: Output: file:/Users/heyongqiang/Documents/workspace/Hive-Test/build/ql/tmp/1227206168/10000
 POSTHOOK: query: select * from dest_j1 x order by x.key
 POSTHOOK: type: QUERY
 POSTHOOK: Input: default@dest_j1
-POSTHOOK: Output: file:/Users/heyongqiang/Documents/workspace/Hive-Test/build/ql/tmp/18999706/10000
+POSTHOOK: Output: file:/Users/heyongqiang/Documents/workspace/Hive-Test/build/ql/tmp/1227206168/10000
 128	val_128	val_128
 128	val_128	val_128
 128	val_128	val_128
@@ -531,6 +531,570 @@
 98	val_98	val_98
 98	val_98	val_98
 98	val_98	val_98
+PREHOOK: query: CREATE TABLE src_copy(key int, value string)
+PREHOOK: type: CREATETABLE
+POSTHOOK: query: CREATE TABLE src_copy(key int, value string)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: default@src_copy
+PREHOOK: query: CREATE TABLE src1_copy(key string, value string)
+PREHOOK: type: CREATETABLE
+POSTHOOK: query: CREATE TABLE src1_copy(key string, value string)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: default@src1_copy
+PREHOOK: query: INSERT OVERWRITE TABLE src_copy select key, value from src
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Output: default@src_copy
+POSTHOOK: query: INSERT OVERWRITE TABLE src_copy select key, value from src
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Output: default@src_copy
+PREHOOK: query: INSERT OVERWRITE TABLE src1_copy select key, value from src1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src1
+PREHOOK: Output: default@src1_copy
+POSTHOOK: query: INSERT OVERWRITE TABLE src1_copy select key, value from src1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src1
+POSTHOOK: Output: default@src1_copy
+PREHOOK: query: EXPLAIN EXTENDED
+INSERT OVERWRITE TABLE dest_j1
+SELECT /*+ MAPJOIN(x,y) */ x.key, z.value, y.value
+FROM src1_copy x JOIN src_copy y ON (x.key = y.key) 
+JOIN srcpart z ON (x.key = z.key)
+WHERE z.ds='2008-04-08' and z.hr=11
+PREHOOK: type: QUERY
+POSTHOOK: query: EXPLAIN EXTENDED
+INSERT OVERWRITE TABLE dest_j1
+SELECT /*+ MAPJOIN(x,y) */ x.key, z.value, y.value
+FROM src1_copy x JOIN src_copy y ON (x.key = y.key) 
+JOIN srcpart z ON (x.key = z.key)
+WHERE z.ds='2008-04-08' and z.hr=11
+POSTHOOK: type: QUERY
+ABSTRACT SYNTAX TREE:
+  (TOK_QUERY (TOK_FROM (TOK_JOIN (TOK_JOIN (TOK_TABREF src1_copy x) (TOK_TABREF src_copy
y) (= (. (TOK_TABLE_OR_COL x) key) (. (TOK_TABLE_OR_COL y) key))) (TOK_TABREF srcpart z) (=
(. (TOK_TABLE_OR_COL x) key) (. (TOK_TABLE_OR_COL z) key)))) (TOK_INSERT (TOK_DESTINATION
(TOK_TAB dest_j1)) (TOK_SELECT (TOK_HINTLIST (TOK_HINT TOK_MAPJOIN (TOK_HINTARGLIST x y)))
(TOK_SELEXPR (. (TOK_TABLE_OR_COL x) key)) (TOK_SELEXPR (. (TOK_TABLE_OR_COL z) value)) (TOK_SELEXPR
(. (TOK_TABLE_OR_COL y) value))) (TOK_WHERE (and (= (. (TOK_TABLE_OR_COL z) ds) '2008-04-08')
(= (. (TOK_TABLE_OR_COL z) hr) 11)))))
+
+STAGE DEPENDENCIES:
+  Stage-1 is a root stage
+  Stage-4 depends on stages: Stage-1
+  Stage-0 depends on stages: Stage-4
+
+STAGE PLANS:
+  Stage: Stage-1
+    Map Reduce
+      Alias -> Map Operator Tree:
+        z 
+          TableScan
+            alias: z
+            Filter Operator
+              isSamplingPred: false
+              predicate:
+                  expr: ((ds = '2008-04-08') and (hr = 11))
+                  type: boolean
+              Common Join Operator
+                condition map:
+                     Inner Join 0 to 1
+                     Inner Join 0 to 2
+                condition expressions:
+                  0 {key}
+                  1 {value}
+                  2 {value} {ds} {hr}
+                keys:
+                  0 [class org.apache.hadoop.hive.ql.udf.generic.GenericUDFBridge(Column[key]()]
+                  1 [class org.apache.hadoop.hive.ql.udf.generic.GenericUDFBridge(Column[key]()]
+                  2 [class org.apache.hadoop.hive.ql.udf.generic.GenericUDFBridge(Column[key]()]
+                outputColumnNames: _col0, _col3, _col5, _col6, _col7
+                Position of Big Table: 2
+                Select Operator
+                  expressions:
+                        expr: _col0
+                        type: string
+                        expr: _col3
+                        type: string
+                        expr: _col5
+                        type: string
+                        expr: _col6
+                        type: string
+                        expr: _col7
+                        type: string
+                  outputColumnNames: _col0, _col3, _col5, _col6, _col7
+                  Filter Operator
+                    isSamplingPred: false
+                    predicate:
+                        expr: ((_col6 = '2008-04-08') and (_col7 = 11))
+                        type: boolean
+                    Select Operator
+                      expressions:
+                            expr: _col0
+                            type: string
+                            expr: _col5
+                            type: string
+                            expr: _col3
+                            type: string
+                      outputColumnNames: _col0, _col1, _col2
+                      File Output Operator
+                        compressed: false
+                        GlobalTableId: 1
+                        directory: file:/Users/heyongqiang/Documents/workspace/Hive-Test/build/ql/tmp/1471315856/10002
+                        table:
+                            input format: org.apache.hadoop.mapred.TextInputFormat
+                            output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+                            properties:
+                              name dest_j1
+                              columns.types string:string:string
+                              serialization.ddl struct dest_j1 { string key, string value,
string val2}
+                              serialization.format 1
+                              columns key,value,val2
+                              bucket_count -1
+                              serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+                              file.inputformat org.apache.hadoop.mapred.TextInputFormat
+                              file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+                              location file:/Users/heyongqiang/Documents/workspace/Hive-Test/build/ql/test/data/warehouse/dest_j1
+                              transient_lastDdlTime 1260991796
+                            serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+                            name: dest_j1
+      Local Work:
+        Map Reduce Local Work
+          Alias -> Map Local Tables:
+            y 
+              Fetch Operator
+                limit: -1
+            x 
+              Fetch Operator
+                limit: -1
+          Alias -> Map Local Operator Tree:
+            y 
+              TableScan
+                alias: y
+                Common Join Operator
+                  condition map:
+                       Inner Join 0 to 1
+                       Inner Join 0 to 2
+                  condition expressions:
+                    0 {key}
+                    1 {value}
+                    2 {value} {ds} {hr}
+                  keys:
+                    0 [class org.apache.hadoop.hive.ql.udf.generic.GenericUDFBridge(Column[key]()]
+                    1 [class org.apache.hadoop.hive.ql.udf.generic.GenericUDFBridge(Column[key]()]
+                    2 [class org.apache.hadoop.hive.ql.udf.generic.GenericUDFBridge(Column[key]()]
+                  outputColumnNames: _col0, _col3, _col5, _col6, _col7
+                  Position of Big Table: 2
+                  Select Operator
+                    expressions:
+                          expr: _col0
+                          type: string
+                          expr: _col3
+                          type: string
+                          expr: _col5
+                          type: string
+                          expr: _col6
+                          type: string
+                          expr: _col7
+                          type: string
+                    outputColumnNames: _col0, _col3, _col5, _col6, _col7
+                    Filter Operator
+                      isSamplingPred: false
+                      predicate:
+                          expr: ((_col6 = '2008-04-08') and (_col7 = 11))
+                          type: boolean
+                      Select Operator
+                        expressions:
+                              expr: _col0
+                              type: string
+                              expr: _col5
+                              type: string
+                              expr: _col3
+                              type: string
+                        outputColumnNames: _col0, _col1, _col2
+                        File Output Operator
+                          compressed: false
+                          GlobalTableId: 1
+                          directory: file:/Users/heyongqiang/Documents/workspace/Hive-Test/build/ql/tmp/1471315856/10002
+                          table:
+                              input format: org.apache.hadoop.mapred.TextInputFormat
+                              output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+                              properties:
+                                name dest_j1
+                                columns.types string:string:string
+                                serialization.ddl struct dest_j1 { string key, string value,
string val2}
+                                serialization.format 1
+                                columns key,value,val2
+                                bucket_count -1
+                                serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+                                file.inputformat org.apache.hadoop.mapred.TextInputFormat
+                                file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+                                location file:/Users/heyongqiang/Documents/workspace/Hive-Test/build/ql/test/data/warehouse/dest_j1
+                                transient_lastDdlTime 1260991796
+                              serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+                              name: dest_j1
+            x 
+              TableScan
+                alias: x
+                Common Join Operator
+                  condition map:
+                       Inner Join 0 to 1
+                       Inner Join 0 to 2
+                  condition expressions:
+                    0 {key}
+                    1 {value}
+                    2 {value} {ds} {hr}
+                  keys:
+                    0 [class org.apache.hadoop.hive.ql.udf.generic.GenericUDFBridge(Column[key]()]
+                    1 [class org.apache.hadoop.hive.ql.udf.generic.GenericUDFBridge(Column[key]()]
+                    2 [class org.apache.hadoop.hive.ql.udf.generic.GenericUDFBridge(Column[key]()]
+                  outputColumnNames: _col0, _col3, _col5, _col6, _col7
+                  Position of Big Table: 2
+                  Select Operator
+                    expressions:
+                          expr: _col0
+                          type: string
+                          expr: _col3
+                          type: string
+                          expr: _col5
+                          type: string
+                          expr: _col6
+                          type: string
+                          expr: _col7
+                          type: string
+                    outputColumnNames: _col0, _col3, _col5, _col6, _col7
+                    Filter Operator
+                      isSamplingPred: false
+                      predicate:
+                          expr: ((_col6 = '2008-04-08') and (_col7 = 11))
+                          type: boolean
+                      Select Operator
+                        expressions:
+                              expr: _col0
+                              type: string
+                              expr: _col5
+                              type: string
+                              expr: _col3
+                              type: string
+                        outputColumnNames: _col0, _col1, _col2
+                        File Output Operator
+                          compressed: false
+                          GlobalTableId: 1
+                          directory: file:/Users/heyongqiang/Documents/workspace/Hive-Test/build/ql/tmp/1471315856/10002
+                          table:
+                              input format: org.apache.hadoop.mapred.TextInputFormat
+                              output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+                              properties:
+                                name dest_j1
+                                columns.types string:string:string
+                                serialization.ddl struct dest_j1 { string key, string value,
string val2}
+                                serialization.format 1
+                                columns key,value,val2
+                                bucket_count -1
+                                serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+                                file.inputformat org.apache.hadoop.mapred.TextInputFormat
+                                file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+                                location file:/Users/heyongqiang/Documents/workspace/Hive-Test/build/ql/test/data/warehouse/dest_j1
+                                transient_lastDdlTime 1260991796
+                              serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+                              name: dest_j1
+      Needs Tagging: false
+      Path -> Alias:
+        file:/Users/heyongqiang/Documents/workspace/Hive-Test/build/ql/test/data/warehouse/srcpart/ds=2008-04-08/hr=11
[z]
+      Path -> Partition:
+        file:/Users/heyongqiang/Documents/workspace/Hive-Test/build/ql/test/data/warehouse/srcpart/ds=2008-04-08/hr=11

+          Partition
+            input format: org.apache.hadoop.mapred.TextInputFormat
+            output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+            partition values:
+              ds 2008-04-08
+              hr 11
+            properties:
+              name srcpart
+              columns.types string:string
+              serialization.ddl struct srcpart { string key, string value}
+              serialization.format 1
+              columns key,value
+              partition_columns ds/hr
+              bucket_count -1
+              serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+              file.inputformat org.apache.hadoop.mapred.TextInputFormat
+              file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+              location file:/Users/heyongqiang/Documents/workspace/Hive-Test/build/ql/test/data/warehouse/srcpart
+              transient_lastDdlTime 1260991793
+            serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+          
+              input format: org.apache.hadoop.mapred.TextInputFormat
+              output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+              properties:
+                name srcpart
+                columns.types string:string
+                serialization.ddl struct srcpart { string key, string value}
+                serialization.format 1
+                columns key,value
+                partition_columns ds/hr
+                bucket_count -1
+                serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+                file.inputformat org.apache.hadoop.mapred.TextInputFormat
+                file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+                location file:/Users/heyongqiang/Documents/workspace/Hive-Test/build/ql/test/data/warehouse/srcpart
+                transient_lastDdlTime 1260991793
+              serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+              name: srcpart
+            name: srcpart
+
+  Stage: Stage-4
+    Conditional Operator
+      list of dependent Tasks:
+          Move Operator
+            files:
+                hdfs directory: true
+                source: file:/Users/heyongqiang/Documents/workspace/Hive-Test/build/ql/tmp/1471315856/10002
+                destination: file:/Users/heyongqiang/Documents/workspace/Hive-Test/build/ql/tmp/612668114/10000
+          Map Reduce
+            Alias -> Map Operator Tree:
+              file:/Users/heyongqiang/Documents/workspace/Hive-Test/build/ql/tmp/1471315856/10002

+                  Reduce Output Operator
+                    sort order: 
+                    Map-reduce partition columns:
+                          expr: rand()
+                          type: double
+                    tag: -1
+                    value expressions:
+                          expr: key
+                          type: string
+                          expr: value
+                          type: string
+                          expr: val2
+                          type: string
+            Needs Tagging: false
+            Path -> Alias:
+              file:/Users/heyongqiang/Documents/workspace/Hive-Test/build/ql/tmp/1471315856/10002
[file:/Users/heyongqiang/Documents/workspace/Hive-Test/build/ql/tmp/1471315856/10002]
+            Path -> Partition:
+              file:/Users/heyongqiang/Documents/workspace/Hive-Test/build/ql/tmp/1471315856/10002

+                Partition
+                  input format: org.apache.hadoop.mapred.TextInputFormat
+                  output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+                  properties:
+                    name dest_j1
+                    columns.types string:string:string
+                    serialization.ddl struct dest_j1 { string key, string value, string val2}
+                    serialization.format 1
+                    columns key,value,val2
+                    bucket_count -1
+                    serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+                    file.inputformat org.apache.hadoop.mapred.TextInputFormat
+                    file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+                    location file:/Users/heyongqiang/Documents/workspace/Hive-Test/build/ql/test/data/warehouse/dest_j1
+                    transient_lastDdlTime 1260991796
+                  serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+                
+                    input format: org.apache.hadoop.mapred.TextInputFormat
+                    output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+                    properties:
+                      name dest_j1
+                      columns.types string:string:string
+                      serialization.ddl struct dest_j1 { string key, string value, string
val2}
+                      serialization.format 1
+                      columns key,value,val2
+                      bucket_count -1
+                      serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+                      file.inputformat org.apache.hadoop.mapred.TextInputFormat
+                      file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+                      location file:/Users/heyongqiang/Documents/workspace/Hive-Test/build/ql/test/data/warehouse/dest_j1
+                      transient_lastDdlTime 1260991796
+                    serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+                    name: dest_j1
+                  name: dest_j1
+            Reduce Operator Tree:
+              Extract
+                File Output Operator
+                  compressed: false
+                  GlobalTableId: 0
+                  directory: file:/Users/heyongqiang/Documents/workspace/Hive-Test/build/ql/tmp/612668114/10000
+                  table:
+                      input format: org.apache.hadoop.mapred.TextInputFormat
+                      output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+                      properties:
+                        name dest_j1
+                        columns.types string:string:string
+                        serialization.ddl struct dest_j1 { string key, string value, string
val2}
+                        serialization.format 1
+                        columns key,value,val2
+                        bucket_count -1
+                        serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+                        file.inputformat org.apache.hadoop.mapred.TextInputFormat
+                        location file:/Users/heyongqiang/Documents/workspace/Hive-Test/build/ql/test/data/warehouse/dest_j1
+                        file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+                        transient_lastDdlTime 1260991796
+                      serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+                      name: dest_j1
+
+  Stage: Stage-0
+    Move Operator
+      tables:
+          replace: true
+          source: file:/Users/heyongqiang/Documents/workspace/Hive-Test/build/ql/tmp/612668114/10000
+          table:
+              input format: org.apache.hadoop.mapred.TextInputFormat
+              output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+              properties:
+                name dest_j1
+                columns.types string:string:string
+                serialization.ddl struct dest_j1 { string key, string value, string val2}
+                serialization.format 1
+                columns key,value,val2
+                bucket_count -1
+                serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+                file.inputformat org.apache.hadoop.mapred.TextInputFormat
+                file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+                location file:/Users/heyongqiang/Documents/workspace/Hive-Test/build/ql/test/data/warehouse/dest_j1
+                transient_lastDdlTime 1260991796
+              serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+              name: dest_j1
+          tmp directory: file:/Users/heyongqiang/Documents/workspace/Hive-Test/build/ql/tmp/612668114/10001
+
+
+PREHOOK: query: INSERT OVERWRITE TABLE dest_j1
+SELECT /*+ MAPJOIN(x,y) */ x.key, z.value, y.value
+FROM src1_copy x JOIN src_copy y ON (x.key = y.key) 
+JOIN srcpart z ON (x.key = z.key)
+WHERE z.ds='2008-04-08' and z.hr=11
+PREHOOK: type: QUERY
+PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11
+PREHOOK: Input: default@src_copy
+PREHOOK: Input: default@src1_copy
+PREHOOK: Output: default@dest_j1
+POSTHOOK: query: INSERT OVERWRITE TABLE dest_j1
+SELECT /*+ MAPJOIN(x,y) */ x.key, z.value, y.value
+FROM src1_copy x JOIN src_copy y ON (x.key = y.key) 
+JOIN srcpart z ON (x.key = z.key)
+WHERE z.ds='2008-04-08' and z.hr=11
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11
+POSTHOOK: Input: default@src_copy
+POSTHOOK: Input: default@src1_copy
+POSTHOOK: Output: default@dest_j1
+PREHOOK: query: select * from dest_j1 x order by x.key
+PREHOOK: type: QUERY
+PREHOOK: Input: default@dest_j1
+PREHOOK: Output: file:/Users/heyongqiang/Documents/workspace/Hive-Test/build/ql/tmp/1242228089/10000
+POSTHOOK: query: select * from dest_j1 x order by x.key
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@dest_j1
+POSTHOOK: Output: file:/Users/heyongqiang/Documents/workspace/Hive-Test/build/ql/tmp/1242228089/10000
+128	val_128	val_128
+128	val_128	val_128
+128	val_128	val_128
+128	val_128	val_128
+128	val_128	val_128
+128	val_128	val_128
+128	val_128	val_128
+128	val_128	val_128
+128	val_128	val_128
+146	val_146	val_146
+146	val_146	val_146
+146	val_146	val_146
+146	val_146	val_146
+150	val_150	val_150
+213	val_213	val_213
+213	val_213	val_213
+213	val_213	val_213
+213	val_213	val_213
+224	val_224	val_224
+224	val_224	val_224
+224	val_224	val_224
+224	val_224	val_224
+238	val_238	val_238
+238	val_238	val_238
+238	val_238	val_238
+238	val_238	val_238
+255	val_255	val_255
+255	val_255	val_255
+255	val_255	val_255
+255	val_255	val_255
+273	val_273	val_273
+273	val_273	val_273
+273	val_273	val_273
+273	val_273	val_273
+273	val_273	val_273
+273	val_273	val_273
+273	val_273	val_273
+273	val_273	val_273
+273	val_273	val_273
+278	val_278	val_278
+278	val_278	val_278
+278	val_278	val_278
+278	val_278	val_278
+311	val_311	val_311
+311	val_311	val_311
+311	val_311	val_311
+311	val_311	val_311
+311	val_311	val_311
+311	val_311	val_311
+311	val_311	val_311
+311	val_311	val_311
+311	val_311	val_311
+369	val_369	val_369
+369	val_369	val_369
+369	val_369	val_369
+369	val_369	val_369
+369	val_369	val_369
+369	val_369	val_369
+369	val_369	val_369
+369	val_369	val_369
+369	val_369	val_369
+401	val_401	val_401
+401	val_401	val_401
+401	val_401	val_401
+401	val_401	val_401
+401	val_401	val_401
+401	val_401	val_401
+401	val_401	val_401
+401	val_401	val_401
+401	val_401	val_401
+401	val_401	val_401
+401	val_401	val_401
+401	val_401	val_401
+401	val_401	val_401
+401	val_401	val_401
+401	val_401	val_401
+401	val_401	val_401
+401	val_401	val_401
+401	val_401	val_401
+401	val_401	val_401
+401	val_401	val_401
+401	val_401	val_401
+401	val_401	val_401
+401	val_401	val_401
+401	val_401	val_401
+401	val_401	val_401
+406	val_406	val_406
+406	val_406	val_406
+406	val_406	val_406
+406	val_406	val_406
+406	val_406	val_406
+406	val_406	val_406
+406	val_406	val_406
+406	val_406	val_406
+406	val_406	val_406
+406	val_406	val_406
+406	val_406	val_406
+406	val_406	val_406
+406	val_406	val_406
+406	val_406	val_406
+406	val_406	val_406
+406	val_406	val_406
+66	val_66	val_66
+98	val_98	val_98
+98	val_98	val_98
+98	val_98	val_98
+98	val_98	val_98
+PREHOOK: query: drop table src_copy
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: drop table src_copy
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Output: default@src_copy
+PREHOOK: query: drop table src1_copy
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: drop table src1_copy
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Output: default@src1_copy
 PREHOOK: query: drop table dest_j1
 PREHOOK: type: DROPTABLE
 POSTHOOK: query: drop table dest_j1



Mime
View raw message