hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From hashut...@apache.org
Subject svn commit: r1447131 - in /hive/trunk/ql/src: java/org/apache/hadoop/hive/ql/exec/MapOperator.java test/queries/clientpositive/join_vc.q test/results/clientpositive/join_vc.q.out
Date Mon, 18 Feb 2013 06:25:27 GMT
Author: hashutosh
Date: Mon Feb 18 06:25:26 2013
New Revision: 1447131

URL: http://svn.apache.org/r1447131
Log:
HIVE-4033 : NPE at runtime while selecting virtual column after joining three tables on different
keys (Ashutosh Chauhan)

Added:
    hive/trunk/ql/src/test/queries/clientpositive/join_vc.q
    hive/trunk/ql/src/test/results/clientpositive/join_vc.q.out
Modified:
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/MapOperator.java

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/MapOperator.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/MapOperator.java?rev=1447131&r1=1447130&r2=1447131&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/MapOperator.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/MapOperator.java Mon Feb 18 06:25:26
2013
@@ -85,7 +85,6 @@ public class MapOperator extends Operato
   // convert from partition to table schema
   private transient Converter partTblObjectInspectorConverter;
   private transient boolean isPartitioned;
-  private transient boolean hasVC;
   private Map<MapInputPath, MapOpCtx> opCtxMap;
   private final Set<MapInputPath> listInputPaths = new HashSet<MapInputPath>();
 
@@ -353,7 +352,6 @@ public class MapOperator extends Operato
       if (tsDesc != null) {
         this.vcs = tsDesc.getVirtualCols();
         if (vcs != null && vcs.size() > 0) {
-          this.hasVC = true;
           List<String> vcNames = new ArrayList<String>(vcs.size());
           this.vcValues = new Writable[vcs.size()];
           List<ObjectInspector> vcsObjectInspectors = new ArrayList<ObjectInspector>(vcs.size());
@@ -617,7 +615,7 @@ public class MapOperator extends Operato
 
     Object row = null;
     try {
-      if (this.hasVC) {
+      if (null != this.rowWithPartAndVC) {
         this.rowWithPartAndVC[0] =
             partTblObjectInspectorConverter.convert(deserializer.deserialize(value));
         int vcPos = isPartitioned ? 2 : 1;
@@ -649,7 +647,7 @@ public class MapOperator extends Operato
     // The row has been converted to comply with table schema, irrespective of partition
schema.
     // So, use tblOI (and not partOI) for forwarding
     try {
-      if (this.hasVC) {
+      if (null != this.rowWithPartAndVC) {
         forward(this.rowWithPartAndVC, this.tblRowObjectInspector);
       } else if (!isPartitioned) {
         forward(row, tblRowObjectInspector);
@@ -660,7 +658,7 @@ public class MapOperator extends Operato
       // Serialize the row and output the error message.
       String rowString;
       try {
-        if (this.hasVC) {
+        if (null != rowWithPartAndVC) {
           rowString = SerDeUtils.getJSONString(rowWithPartAndVC, tblRowObjectInspector);
         } else if (!isPartitioned) {
           rowString = SerDeUtils.getJSONString(row, tblRowObjectInspector);

Added: hive/trunk/ql/src/test/queries/clientpositive/join_vc.q
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientpositive/join_vc.q?rev=1447131&view=auto
==============================================================================
--- hive/trunk/ql/src/test/queries/clientpositive/join_vc.q (added)
+++ hive/trunk/ql/src/test/queries/clientpositive/join_vc.q Mon Feb 18 06:25:26 2013
@@ -0,0 +1,5 @@
+-- see HIVE-4033 earlier a flag named hasVC was not initialized correctly in MapOperator.java,
resulting in NPE for following query. order by and limit in the query is not relevant, problem
would be evident even without those. They are there to keep .q.out file small and sorted.
+
+explain select t3.BLOCK__OFFSET__INSIDE__FILE,t3.key,t3.value from src t1 join src t2 on
t1.key = t2.key join src t3 on t2.value = t3.value order by t3.value limit 3;
+
+select t3.BLOCK__OFFSET__INSIDE__FILE,t3.key,t3.value from src t1 join src t2 on t1.key =
t2.key join src t3 on t2.value = t3.value order by t3.value limit 3;

Added: hive/trunk/ql/src/test/results/clientpositive/join_vc.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/join_vc.q.out?rev=1447131&view=auto
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/join_vc.q.out (added)
+++ hive/trunk/ql/src/test/results/clientpositive/join_vc.q.out Mon Feb 18 06:25:26 2013
@@ -0,0 +1,164 @@
+PREHOOK: query: -- see HIVE-4033 earlier a flag named hasVC was not initialized correctly
in MapOperator.java, resulting in NPE for following query. order by and limit in the query
is not relevant, problem would be evident even without those. They are there to keep .q.out
file small and sorted.
+
+explain select t3.BLOCK__OFFSET__INSIDE__FILE,t3.key,t3.value from src t1 join src t2 on
t1.key = t2.key join src t3 on t2.value = t3.value order by t3.value limit 3
+PREHOOK: type: QUERY
+POSTHOOK: query: -- see HIVE-4033 earlier a flag named hasVC was not initialized correctly
in MapOperator.java, resulting in NPE for following query. order by and limit in the query
is not relevant, problem would be evident even without those. They are there to keep .q.out
file small and sorted.
+
+explain select t3.BLOCK__OFFSET__INSIDE__FILE,t3.key,t3.value from src t1 join src t2 on
t1.key = t2.key join src t3 on t2.value = t3.value order by t3.value limit 3
+POSTHOOK: type: QUERY
+ABSTRACT SYNTAX TREE:
+  (TOK_QUERY (TOK_FROM (TOK_JOIN (TOK_JOIN (TOK_TABREF (TOK_TABNAME src) t1) (TOK_TABREF
(TOK_TABNAME src) t2) (= (. (TOK_TABLE_OR_COL t1) key) (. (TOK_TABLE_OR_COL t2) key))) (TOK_TABREF
(TOK_TABNAME src) t3) (= (. (TOK_TABLE_OR_COL t2) value) (. (TOK_TABLE_OR_COL t3) value))))
(TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (. (TOK_TABLE_OR_COL
t3) BLOCK__OFFSET__INSIDE__FILE)) (TOK_SELEXPR (. (TOK_TABLE_OR_COL t3) key)) (TOK_SELEXPR
(. (TOK_TABLE_OR_COL t3) value))) (TOK_ORDERBY (TOK_TABSORTCOLNAMEASC (. (TOK_TABLE_OR_COL
t3) value))) (TOK_LIMIT 3)))
+
+STAGE DEPENDENCIES:
+  Stage-3 is a root stage
+  Stage-1 depends on stages: Stage-3
+  Stage-2 depends on stages: Stage-1
+  Stage-0 is a root stage
+
+STAGE PLANS:
+  Stage: Stage-3
+    Map Reduce
+      Alias -> Map Operator Tree:
+        t1 
+          TableScan
+            alias: t1
+            Reduce Output Operator
+              key expressions:
+                    expr: key
+                    type: string
+              sort order: +
+              Map-reduce partition columns:
+                    expr: key
+                    type: string
+              tag: 0
+        t2 
+          TableScan
+            alias: t2
+            Reduce Output Operator
+              key expressions:
+                    expr: key
+                    type: string
+              sort order: +
+              Map-reduce partition columns:
+                    expr: key
+                    type: string
+              tag: 1
+              value expressions:
+                    expr: value
+                    type: string
+      Reduce Operator Tree:
+        Join Operator
+          condition map:
+               Inner Join 0 to 1
+          condition expressions:
+            0 
+            1 {VALUE._col1}
+          handleSkewJoin: false
+          outputColumnNames: _col5
+          File Output Operator
+            compressed: false
+            GlobalTableId: 0
+            table:
+                input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+                output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+
+  Stage: Stage-1
+    Map Reduce
+      Alias -> Map Operator Tree:
+        $INTNAME 
+            Reduce Output Operator
+              key expressions:
+                    expr: _col5
+                    type: string
+              sort order: +
+              Map-reduce partition columns:
+                    expr: _col5
+                    type: string
+              tag: 0
+        t3 
+          TableScan
+            alias: t3
+            Reduce Output Operator
+              key expressions:
+                    expr: value
+                    type: string
+              sort order: +
+              Map-reduce partition columns:
+                    expr: value
+                    type: string
+              tag: 1
+              value expressions:
+                    expr: key
+                    type: string
+                    expr: value
+                    type: string
+                    expr: BLOCK__OFFSET__INSIDE__FILE
+                    type: bigint
+      Reduce Operator Tree:
+        Join Operator
+          condition map:
+               Inner Join 0 to 1
+          condition expressions:
+            0 
+            1 {VALUE._col0} {VALUE._col1} {VALUE._col2}
+          handleSkewJoin: false
+          outputColumnNames: _col8, _col9, _col10
+          Select Operator
+            expressions:
+                  expr: _col10
+                  type: bigint
+                  expr: _col8
+                  type: string
+                  expr: _col9
+                  type: string
+            outputColumnNames: _col0, _col1, _col2
+            File Output Operator
+              compressed: false
+              GlobalTableId: 0
+              table:
+                  input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+                  output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+
+  Stage: Stage-2
+    Map Reduce
+      Alias -> Map Operator Tree:
+#### A masked pattern was here ####
+            Reduce Output Operator
+              key expressions:
+                    expr: _col2
+                    type: string
+              sort order: +
+              tag: -1
+              value expressions:
+                    expr: _col0
+                    type: bigint
+                    expr: _col1
+                    type: string
+                    expr: _col2
+                    type: string
+      Reduce Operator Tree:
+        Extract
+          Limit
+            File Output Operator
+              compressed: false
+              GlobalTableId: 0
+              table:
+                  input format: org.apache.hadoop.mapred.TextInputFormat
+                  output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+
+  Stage: Stage-0
+    Fetch Operator
+      limit: 3
+
+
+PREHOOK: query: select t3.BLOCK__OFFSET__INSIDE__FILE,t3.key,t3.value from src t1 join src
t2 on t1.key = t2.key join src t3 on t2.value = t3.value order by t3.value limit 3
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+#### A masked pattern was here ####
+POSTHOOK: query: select t3.BLOCK__OFFSET__INSIDE__FILE,t3.key,t3.value from src t1 join src
t2 on t1.key = t2.key join src t3 on t2.value = t3.value order by t3.value limit 3
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+#### A masked pattern was here ####
+2088	0	val_0
+2632	0	val_0
+968	0	val_0



Mime
View raw message