hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From na...@apache.org
Subject svn commit: r1236479 - in /hive/trunk/ql/src: java/org/apache/hadoop/hive/ql/optimizer/GenMRRedSink3.java test/queries/clientpositive/union_lateralview.q test/results/clientpositive/union_lateralview.q.out
Date Fri, 27 Jan 2012 01:05:23 GMT
Author: namit
Date: Fri Jan 27 01:05:22 2012
New Revision: 1236479

URL: http://svn.apache.org/viewvc?rev=1236479&view=rev
Log:
HIVE-2754 NPE in union with lateral view
(Yongqiang He via namit)


Added:
    hive/trunk/ql/src/test/queries/clientpositive/union_lateralview.q
    hive/trunk/ql/src/test/results/clientpositive/union_lateralview.q.out
Modified:
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRRedSink3.java

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRRedSink3.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRRedSink3.java?rev=1236479&r1=1236478&r2=1236479&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRRedSink3.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRRedSink3.java Fri Jan
27 01:05:22 2012
@@ -63,7 +63,17 @@ public class GenMRRedSink3 implements No
     Map<Operator<? extends Serializable>, GenMapRedCtx> mapCurrCtx = ctx
         .getMapCurrCtx();
     GenMapRedCtx mapredCtx = mapCurrCtx.get(ctx.getCurrUnionOp());
-    Task<? extends Serializable> unionTask = mapredCtx.getCurrTask();
+    Task<? extends Serializable> unionTask = null;
+    if(mapredCtx != null) {
+      unionTask = mapredCtx.getCurrTask();
+    } else {
+      Operator<? extends Serializable> topOp = ctx.getCurrTopOp();
+      if(topOp == null) {
+        return null;
+      }
+      unionTask = ctx.getCurrTask();
+    }
+
     MapredWork plan = (MapredWork) unionTask.getWork();
     HashMap<Operator<? extends Serializable>, Task<? extends Serializable>>
opTaskMap = ctx
         .getOpTaskMap();

Added: hive/trunk/ql/src/test/queries/clientpositive/union_lateralview.q
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientpositive/union_lateralview.q?rev=1236479&view=auto
==============================================================================
--- hive/trunk/ql/src/test/queries/clientpositive/union_lateralview.q (added)
+++ hive/trunk/ql/src/test/queries/clientpositive/union_lateralview.q Fri Jan 27 01:05:22
2012
@@ -0,0 +1,43 @@
+
+create table test_union_lateral_view(key int, arr_ele int, value string);
+
+EXPLAIN 
+INSERT OVERWRITE TABLE test_union_lateral_view
+SELECT b.key, d.arr_ele, d.value
+FROM (
+ SELECT c.arr_ele as arr_ele, a.key as key, a.value as value
+ FROM (
+   SELECT key, value, array(1,2,3) as arr
+   FROM src
+
+   UNION ALL
+   
+   SELECT key, value, array(1,2,3) as arr
+   FROM srcpart
+   WHERE ds = '2008-04-08' and hr='12'
+ ) a LATERAL VIEW EXPLODE(arr) c AS arr_ele
+) d
+LEFT OUTER JOIN src b
+ON d.key = b.key
+;
+
+INSERT OVERWRITE TABLE test_union_lateral_view
+SELECT b.key, d.arr_ele, d.value
+FROM (
+ SELECT c.arr_ele as arr_ele, a.key as key, a.value as value
+ FROM (
+   SELECT key, value, array(1,2,3) as arr
+   FROM src
+
+   UNION ALL
+   
+   SELECT key, value, array(1,2,3) as arr
+   FROM srcpart
+   WHERE ds = '2008-04-08' and hr='12'
+ ) a LATERAL VIEW EXPLODE(arr) c AS arr_ele
+) d
+LEFT OUTER JOIN src b
+ON d.key = b.key
+;
+
+select key, arr_ele, value from test_union_lateral_view order by key, arr_ele limit 20;

Added: hive/trunk/ql/src/test/results/clientpositive/union_lateralview.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/union_lateralview.q.out?rev=1236479&view=auto
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/union_lateralview.q.out (added)
+++ hive/trunk/ql/src/test/results/clientpositive/union_lateralview.q.out Fri Jan 27 01:05:22
2012
@@ -0,0 +1,342 @@
+PREHOOK: query: create table test_union_lateral_view(key int, arr_ele int, value string)
+PREHOOK: type: CREATETABLE
+POSTHOOK: query: create table test_union_lateral_view(key int, arr_ele int, value string)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: default@test_union_lateral_view
+PREHOOK: query: EXPLAIN 
+INSERT OVERWRITE TABLE test_union_lateral_view
+SELECT b.key, d.arr_ele, d.value
+FROM (
+ SELECT c.arr_ele as arr_ele, a.key as key, a.value as value
+ FROM (
+   SELECT key, value, array(1,2,3) as arr
+   FROM src
+
+   UNION ALL
+   
+   SELECT key, value, array(1,2,3) as arr
+   FROM srcpart
+   WHERE ds = '2008-04-08' and hr='12'
+ ) a LATERAL VIEW EXPLODE(arr) c AS arr_ele
+) d
+LEFT OUTER JOIN src b
+ON d.key = b.key
+PREHOOK: type: QUERY
+POSTHOOK: query: EXPLAIN 
+INSERT OVERWRITE TABLE test_union_lateral_view
+SELECT b.key, d.arr_ele, d.value
+FROM (
+ SELECT c.arr_ele as arr_ele, a.key as key, a.value as value
+ FROM (
+   SELECT key, value, array(1,2,3) as arr
+   FROM src
+
+   UNION ALL
+   
+   SELECT key, value, array(1,2,3) as arr
+   FROM srcpart
+   WHERE ds = '2008-04-08' and hr='12'
+ ) a LATERAL VIEW EXPLODE(arr) c AS arr_ele
+) d
+LEFT OUTER JOIN src b
+ON d.key = b.key
+POSTHOOK: type: QUERY
+ABSTRACT SYNTAX TREE:
+  (TOK_QUERY (TOK_FROM (TOK_LEFTOUTERJOIN (TOK_SUBQUERY (TOK_QUERY (TOK_FROM (TOK_LATERAL_VIEW
(TOK_SELECT (TOK_SELEXPR (TOK_FUNCTION EXPLODE (TOK_TABLE_OR_COL arr)) arr_ele (TOK_TABALIAS
c))) (TOK_SUBQUERY (TOK_UNION (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME src))) (TOK_INSERT
(TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_TABLE_OR_COL key))
(TOK_SELEXPR (TOK_TABLE_OR_COL value)) (TOK_SELEXPR (TOK_FUNCTION array 1 2 3) arr)))) (TOK_QUERY
(TOK_FROM (TOK_TABREF (TOK_TABNAME srcpart))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE))
(TOK_SELECT (TOK_SELEXPR (TOK_TABLE_OR_COL key)) (TOK_SELEXPR (TOK_TABLE_OR_COL value)) (TOK_SELEXPR
(TOK_FUNCTION array 1 2 3) arr)) (TOK_WHERE (and (= (TOK_TABLE_OR_COL ds) '2008-04-08') (=
(TOK_TABLE_OR_COL hr) '12')))))) a))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE))
(TOK_SELECT (TOK_SELEXPR (. (TOK_TABLE_OR_COL c) arr_ele) arr_ele) (TOK_SELEXPR (. (TOK_TABLE_OR_COL
a) key) key) (TOK_SELEXPR (. (TOK_T
 ABLE_OR_COL a) value) value)))) d) (TOK_TABREF (TOK_TABNAME src) b) (= (. (TOK_TABLE_OR_COL
d) key) (. (TOK_TABLE_OR_COL b) key)))) (TOK_INSERT (TOK_DESTINATION (TOK_TAB (TOK_TABNAME
test_union_lateral_view))) (TOK_SELECT (TOK_SELEXPR (. (TOK_TABLE_OR_COL b) key)) (TOK_SELEXPR
(. (TOK_TABLE_OR_COL d) arr_ele)) (TOK_SELEXPR (. (TOK_TABLE_OR_COL d) value)))))
+
+STAGE DEPENDENCIES:
+  Stage-1 is a root stage
+  Stage-0 depends on stages: Stage-1
+  Stage-2 depends on stages: Stage-0
+
+STAGE PLANS:
+  Stage: Stage-1
+    Map Reduce
+      Alias -> Map Operator Tree:
+        b 
+          TableScan
+            alias: b
+            Reduce Output Operator
+              key expressions:
+                    expr: key
+                    type: string
+              sort order: +
+              Map-reduce partition columns:
+                    expr: key
+                    type: string
+              tag: 1
+              value expressions:
+                    expr: key
+                    type: string
+        d-subquery1:a-subquery1:src 
+          TableScan
+            alias: src
+            Select Operator
+              expressions:
+                    expr: key
+                    type: string
+                    expr: value
+                    type: string
+                    expr: array(1,2,3)
+                    type: array<int>
+              outputColumnNames: _col0, _col1, _col2
+              Union
+                Lateral View Forward
+                  Select Operator
+                    SELECT * : (no compute)
+                    Lateral View Join Operator
+                      outputColumnNames: _col0, _col1, _col2, _col3
+                      Select Operator
+                        expressions:
+                              expr: _col3
+                              type: int
+                              expr: _col0
+                              type: string
+                              expr: _col1
+                              type: string
+                        outputColumnNames: _col0, _col1, _col2
+                        Reduce Output Operator
+                          key expressions:
+                                expr: _col1
+                                type: string
+                          sort order: +
+                          Map-reduce partition columns:
+                                expr: _col1
+                                type: string
+                          tag: 0
+                          value expressions:
+                                expr: _col0
+                                type: int
+                                expr: _col2
+                                type: string
+                  Select Operator
+                    expressions:
+                          expr: _col2
+                          type: array<int>
+                    outputColumnNames: _col0
+                    UDTF Operator
+                      function name: explode
+                      Lateral View Join Operator
+                        outputColumnNames: _col0, _col1, _col2, _col3
+                        Select Operator
+                          expressions:
+                                expr: _col3
+                                type: int
+                                expr: _col0
+                                type: string
+                                expr: _col1
+                                type: string
+                          outputColumnNames: _col0, _col1, _col2
+                          Reduce Output Operator
+                            key expressions:
+                                  expr: _col1
+                                  type: string
+                            sort order: +
+                            Map-reduce partition columns:
+                                  expr: _col1
+                                  type: string
+                            tag: 0
+                            value expressions:
+                                  expr: _col0
+                                  type: int
+                                  expr: _col2
+                                  type: string
+        d-subquery2:a-subquery2:srcpart 
+          TableScan
+            alias: srcpart
+            Select Operator
+              expressions:
+                    expr: key
+                    type: string
+                    expr: value
+                    type: string
+                    expr: array(1,2,3)
+                    type: array<int>
+              outputColumnNames: _col0, _col1, _col2
+              Union
+                Lateral View Forward
+                  Select Operator
+                    SELECT * : (no compute)
+                    Lateral View Join Operator
+                      outputColumnNames: _col0, _col1, _col2, _col3
+                      Select Operator
+                        expressions:
+                              expr: _col3
+                              type: int
+                              expr: _col0
+                              type: string
+                              expr: _col1
+                              type: string
+                        outputColumnNames: _col0, _col1, _col2
+                        Reduce Output Operator
+                          key expressions:
+                                expr: _col1
+                                type: string
+                          sort order: +
+                          Map-reduce partition columns:
+                                expr: _col1
+                                type: string
+                          tag: 0
+                          value expressions:
+                                expr: _col0
+                                type: int
+                                expr: _col2
+                                type: string
+                  Select Operator
+                    expressions:
+                          expr: _col2
+                          type: array<int>
+                    outputColumnNames: _col0
+                    UDTF Operator
+                      function name: explode
+                      Lateral View Join Operator
+                        outputColumnNames: _col0, _col1, _col2, _col3
+                        Select Operator
+                          expressions:
+                                expr: _col3
+                                type: int
+                                expr: _col0
+                                type: string
+                                expr: _col1
+                                type: string
+                          outputColumnNames: _col0, _col1, _col2
+                          Reduce Output Operator
+                            key expressions:
+                                  expr: _col1
+                                  type: string
+                            sort order: +
+                            Map-reduce partition columns:
+                                  expr: _col1
+                                  type: string
+                            tag: 0
+                            value expressions:
+                                  expr: _col0
+                                  type: int
+                                  expr: _col2
+                                  type: string
+      Reduce Operator Tree:
+        Join Operator
+          condition map:
+               Left Outer Join0 to 1
+          condition expressions:
+            0 {VALUE._col0} {VALUE._col2}
+            1 {VALUE._col0}
+          handleSkewJoin: false
+          outputColumnNames: _col0, _col2, _col3
+          Select Operator
+            expressions:
+                  expr: _col3
+                  type: string
+                  expr: _col0
+                  type: int
+                  expr: _col2
+                  type: string
+            outputColumnNames: _col0, _col1, _col2
+            Select Operator
+              expressions:
+                    expr: UDFToInteger(_col0)
+                    type: int
+                    expr: _col1
+                    type: int
+                    expr: _col2
+                    type: string
+              outputColumnNames: _col0, _col1, _col2
+              File Output Operator
+                compressed: false
+                GlobalTableId: 1
+                table:
+                    input format: org.apache.hadoop.mapred.TextInputFormat
+                    output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+                    serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+                    name: default.test_union_lateral_view
+
+  Stage: Stage-0
+    Move Operator
+      tables:
+          replace: true
+          table:
+              input format: org.apache.hadoop.mapred.TextInputFormat
+              output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+              serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+              name: default.test_union_lateral_view
+
+  Stage: Stage-2
+    Stats-Aggr Operator
+
+
+PREHOOK: query: INSERT OVERWRITE TABLE test_union_lateral_view
+SELECT b.key, d.arr_ele, d.value
+FROM (
+ SELECT c.arr_ele as arr_ele, a.key as key, a.value as value
+ FROM (
+   SELECT key, value, array(1,2,3) as arr
+   FROM src
+
+   UNION ALL
+   
+   SELECT key, value, array(1,2,3) as arr
+   FROM srcpart
+   WHERE ds = '2008-04-08' and hr='12'
+ ) a LATERAL VIEW EXPLODE(arr) c AS arr_ele
+) d
+LEFT OUTER JOIN src b
+ON d.key = b.key
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12
+PREHOOK: Output: default@test_union_lateral_view
+POSTHOOK: query: INSERT OVERWRITE TABLE test_union_lateral_view
+SELECT b.key, d.arr_ele, d.value
+FROM (
+ SELECT c.arr_ele as arr_ele, a.key as key, a.value as value
+ FROM (
+   SELECT key, value, array(1,2,3) as arr
+   FROM src
+
+   UNION ALL
+   
+   SELECT key, value, array(1,2,3) as arr
+   FROM srcpart
+   WHERE ds = '2008-04-08' and hr='12'
+ ) a LATERAL VIEW EXPLODE(arr) c AS arr_ele
+) d
+LEFT OUTER JOIN src b
+ON d.key = b.key
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12
+POSTHOOK: Output: default@test_union_lateral_view
+POSTHOOK: Lineage: test_union_lateral_view.arr_ele EXPRESSION []
+POSTHOOK: Lineage: test_union_lateral_view.key EXPRESSION [(src)b.FieldSchema(name:key, type:string,
comment:default), ]
+POSTHOOK: Lineage: test_union_lateral_view.value EXPRESSION [(srcpart)srcpart.FieldSchema(name:key,
type:string, comment:default), (src)src.FieldSchema(name:key, type:string, comment:default),
]
+PREHOOK: query: select key, arr_ele, value from test_union_lateral_view order by key, arr_ele
limit 20
+PREHOOK: type: QUERY
+PREHOOK: Input: default@test_union_lateral_view
+#### A masked pattern was here ####
+POSTHOOK: query: select key, arr_ele, value from test_union_lateral_view order by key, arr_ele
limit 20
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@test_union_lateral_view
+#### A masked pattern was here ####
+POSTHOOK: Lineage: test_union_lateral_view.arr_ele EXPRESSION []
+POSTHOOK: Lineage: test_union_lateral_view.key EXPRESSION [(src)b.FieldSchema(name:key, type:string,
comment:default), ]
+POSTHOOK: Lineage: test_union_lateral_view.value EXPRESSION [(srcpart)srcpart.FieldSchema(name:key,
type:string, comment:default), (src)src.FieldSchema(name:key, type:string, comment:default),
]
+0	1	val_0
+0	1	val_0
+0	1	val_0
+0	1	val_0
+0	1	val_0
+0	1	val_0
+0	1	val_0
+0	1	val_0
+0	1	val_0
+0	1	val_0
+0	1	val_0
+0	1	val_0
+0	1	val_0
+0	1	val_0
+0	1	val_0
+0	1	val_0
+0	1	val_0
+0	1	val_0
+0	2	val_0
+0	2	val_0



Mime
View raw message