hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ekoif...@apache.org
Subject hive git commit: HIVE-10776 - Schema on insert for bucketed tables throwing NullPointerException
Date Fri, 22 May 2015 22:15:01 GMT
Repository: hive
Updated Branches:
  refs/heads/branch-1.2 7b89fad81 -> 613b559f2


HIVE-10776 - Schema on insert for bucketed tables throwing NullPointerException


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/613b559f
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/613b559f
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/613b559f

Branch: refs/heads/branch-1.2
Commit: 613b559f2e1b027c64f265120c202f6af4138930
Parents: 7b89fad
Author: Eugene Koifman <ekoifman@hortonworks.com>
Authored: Fri May 22 15:14:54 2015 -0700
Committer: Eugene Koifman <ekoifman@hortonworks.com>
Committed: Fri May 22 15:14:54 2015 -0700

----------------------------------------------------------------------
 .../hadoop/hive/ql/parse/SemanticAnalyzer.java  |  9 +-
 .../clientpositive/insert_into_with_schema2.q   | 23 +++++
 .../insert_into_with_schema2.q.out              | 98 ++++++++++++++++++++
 3 files changed, 126 insertions(+), 4 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/613b559f/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
index bf889fc..50c57fc 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
@@ -3907,7 +3907,7 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
    * @throws SemanticException
    */
   private void handleInsertStatementSpec(List<ExprNodeDesc> col_list, String dest,
-                                         RowResolver out_rwsch, RowResolver inputRR, QB qb,
+                                         RowResolver outputRR, RowResolver inputRR, QB qb,
                                          ASTNode selExprList) throws SemanticException {
     //(z,x)
     List<String> targetTableSchema = qb.getParseInfo().getDestSchemaForClause(dest);//specified
in the query
@@ -3929,7 +3929,7 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
     Map<String, ColumnInfo> targetCol2ColumnInfo = new HashMap<String, ColumnInfo>();
     int colListPos = 0;
     for(String targetCol : targetTableSchema) {
-      targetCol2ColumnInfo.put(targetCol, out_rwsch.getColumnInfos().get(colListPos));
+      targetCol2ColumnInfo.put(targetCol, outputRR.getColumnInfos().get(colListPos));
       targetCol2Projection.put(targetCol, col_list.get(colListPos++));
     }
     Table target = qb.getMetaData().getDestTableForAlias(dest);
@@ -3973,16 +3973,17 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
         t.setText("TOK_NULL");
         ExprNodeDesc exp = genExprNodeDesc(new ASTNode(t), inputRR, tcCtx);
         new_col_list.add(exp);
-        final String tableAlias = "";//is this OK? this column doesn't come from any table
+        final String tableAlias = null;//this column doesn't come from any table
         ColumnInfo colInfo = new ColumnInfo(getColumnInternalName(colListPos),
           exp.getWritableObjectInspector(), tableAlias, false);
         newSchema.add(colInfo);
+        outputRR.addMappingOnly(colInfo.getTabAlias(), colInfo.getInternalName(), colInfo);
       }
       colListPos++;
     }
     col_list.clear();
     col_list.addAll(new_col_list);
-    out_rwsch.setRowSchema(new RowSchema(newSchema));
+    outputRR.setRowSchema(new RowSchema(newSchema));
   }
   String recommendName(ExprNodeDesc exp, String colAlias) {
     if (!colAlias.startsWith(autogenColAliasPrfxLbl)) {

http://git-wip-us.apache.org/repos/asf/hive/blob/613b559f/ql/src/test/queries/clientpositive/insert_into_with_schema2.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/insert_into_with_schema2.q b/ql/src/test/queries/clientpositive/insert_into_with_schema2.q
new file mode 100644
index 0000000..b7c6b58
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/insert_into_with_schema2.q
@@ -0,0 +1,23 @@
+-- SORT_QUERY_RESULTS;
+
+set hive.enforce.bucketing=true;
+
+create table studenttab10k (age2 int);
+insert into studenttab10k values(1);
+
+create table student_acid (age int, grade int)
+ clustered by (age) into 1 buckets;
+
+insert into student_acid(age) select * from studenttab10k;
+
+select * from student_acid;
+
+insert into student_acid(grade, age) select 3 g, * from studenttab10k;
+
+select * from student_acid;
+
+insert into student_acid(grade, age) values(20, 2);
+
+insert into student_acid(age) values(22);
+
+select * from student_acid;

http://git-wip-us.apache.org/repos/asf/hive/blob/613b559f/ql/src/test/results/clientpositive/insert_into_with_schema2.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/insert_into_with_schema2.q.out b/ql/src/test/results/clientpositive/insert_into_with_schema2.q.out
new file mode 100644
index 0000000..a55a82f
--- /dev/null
+++ b/ql/src/test/results/clientpositive/insert_into_with_schema2.q.out
@@ -0,0 +1,98 @@
+PREHOOK: query: create table studenttab10k (age2 int)
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@studenttab10k
+POSTHOOK: query: create table studenttab10k (age2 int)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@studenttab10k
+PREHOOK: query: insert into studenttab10k values(1)
+PREHOOK: type: QUERY
+PREHOOK: Input: default@values__tmp__table__1
+PREHOOK: Output: default@studenttab10k
+POSTHOOK: query: insert into studenttab10k values(1)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@values__tmp__table__1
+POSTHOOK: Output: default@studenttab10k
+POSTHOOK: Lineage: studenttab10k.age2 EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col1,
type:string, comment:), ]
+PREHOOK: query: create table student_acid (age int, grade int)
+ clustered by (age) into 1 buckets
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@student_acid
+POSTHOOK: query: create table student_acid (age int, grade int)
+ clustered by (age) into 1 buckets
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@student_acid
+PREHOOK: query: insert into student_acid(age) select * from studenttab10k
+PREHOOK: type: QUERY
+PREHOOK: Input: default@studenttab10k
+PREHOOK: Output: default@student_acid
+POSTHOOK: query: insert into student_acid(age) select * from studenttab10k
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@studenttab10k
+POSTHOOK: Output: default@student_acid
+POSTHOOK: Lineage: student_acid.age SIMPLE [(studenttab10k)studenttab10k.FieldSchema(name:age2,
type:int, comment:null), ]
+POSTHOOK: Lineage: student_acid.grade EXPRESSION []
+PREHOOK: query: select * from student_acid
+PREHOOK: type: QUERY
+PREHOOK: Input: default@student_acid
+#### A masked pattern was here ####
+POSTHOOK: query: select * from student_acid
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@student_acid
+#### A masked pattern was here ####
+1	NULL
+PREHOOK: query: insert into student_acid(grade, age) select 3 g, * from studenttab10k
+PREHOOK: type: QUERY
+PREHOOK: Input: default@studenttab10k
+PREHOOK: Output: default@student_acid
+POSTHOOK: query: insert into student_acid(grade, age) select 3 g, * from studenttab10k
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@studenttab10k
+POSTHOOK: Output: default@student_acid
+POSTHOOK: Lineage: student_acid.age SIMPLE [(studenttab10k)studenttab10k.FieldSchema(name:age2,
type:int, comment:null), ]
+POSTHOOK: Lineage: student_acid.grade SIMPLE []
+PREHOOK: query: select * from student_acid
+PREHOOK: type: QUERY
+PREHOOK: Input: default@student_acid
+#### A masked pattern was here ####
+POSTHOOK: query: select * from student_acid
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@student_acid
+#### A masked pattern was here ####
+1	3
+1	NULL
+PREHOOK: query: insert into student_acid(grade, age) values(20, 2)
+PREHOOK: type: QUERY
+PREHOOK: Input: default@values__tmp__table__2
+PREHOOK: Output: default@student_acid
+POSTHOOK: query: insert into student_acid(grade, age) values(20, 2)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@values__tmp__table__2
+POSTHOOK: Output: default@student_acid
+POSTHOOK: Lineage: student_acid.age EXPRESSION [(values__tmp__table__2)values__tmp__table__2.FieldSchema(name:tmp_values_col2,
type:string, comment:), ]
+POSTHOOK: Lineage: student_acid.grade EXPRESSION [(values__tmp__table__2)values__tmp__table__2.FieldSchema(name:tmp_values_col1,
type:string, comment:), ]
+PREHOOK: query: insert into student_acid(age) values(22)
+PREHOOK: type: QUERY
+PREHOOK: Input: default@values__tmp__table__3
+PREHOOK: Output: default@student_acid
+POSTHOOK: query: insert into student_acid(age) values(22)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@values__tmp__table__3
+POSTHOOK: Output: default@student_acid
+POSTHOOK: Lineage: student_acid.age EXPRESSION [(values__tmp__table__3)values__tmp__table__3.FieldSchema(name:tmp_values_col1,
type:string, comment:), ]
+POSTHOOK: Lineage: student_acid.grade EXPRESSION []
+PREHOOK: query: select * from student_acid
+PREHOOK: type: QUERY
+PREHOOK: Input: default@student_acid
+#### A masked pattern was here ####
+POSTHOOK: query: select * from student_acid
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@student_acid
+#### A masked pattern was here ####
+1	3
+1	NULL
+2	20
+22	NULL


Mime
View raw message