Return-Path: X-Original-To: apmail-hive-commits-archive@www.apache.org Delivered-To: apmail-hive-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id A2ADB18A48 for ; Thu, 28 May 2015 17:50:53 +0000 (UTC) Received: (qmail 8311 invoked by uid 500); 28 May 2015 17:50:53 -0000 Delivered-To: apmail-hive-commits-archive@hive.apache.org Received: (qmail 8265 invoked by uid 500); 28 May 2015 17:50:53 -0000 Mailing-List: contact commits-help@hive.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: hive-dev@hive.apache.org Delivered-To: mailing list commits@hive.apache.org Received: (qmail 8248 invoked by uid 99); 28 May 2015 17:50:53 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Thu, 28 May 2015 17:50:53 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 56D8CE10A2; Thu, 28 May 2015 17:50:53 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: ekoifman@apache.org To: commits@hive.apache.org Message-Id: X-Mailer: ASF-Git Admin Mailer Subject: hive git commit: HIVE-10828 - Insert with schema and dynamic partitions NullPointerException (Eugene Koifman, reviewed by Ashutosh Chauhan) Date: Thu, 28 May 2015 17:50:53 +0000 (UTC) Repository: hive Updated Branches: refs/heads/master 1293f3d38 -> 49cc02873 HIVE-10828 - Insert with schema and dynamic partitions NullPointerException (Eugene Koifman, reviewed by Ashutosh Chauhan) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/49cc0287 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/49cc0287 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/49cc0287 Branch: refs/heads/master Commit: 49cc02873fb7b1d43d5e1dbb3ef3435c877f61b2 Parents: 1293f3d Author: Eugene Koifman Authored: Thu May 28 10:50:45 2015 -0700 Committer: Eugene Koifman Committed: Thu May 28 10:50:45 2015 -0700 ---------------------------------------------------------------------- .../hadoop/hive/ql/parse/SemanticAnalyzer.java | 14 ++++---- .../clientpositive/insert_into_with_schema2.q | 11 ++++++ .../insert_into_with_schema2.q.out | 37 ++++++++++++++++++++ 3 files changed, 55 insertions(+), 7 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/49cc0287/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java index 7f355e5..351c267 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java @@ -3864,7 +3864,7 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer { } selectStar = selectStar && exprList.getChildCount() == posn + 1; - handleInsertStatementSpec(col_list, dest, out_rwsch, inputRR, qb, selExprList); + out_rwsch = handleInsertStatementSpec(col_list, dest, out_rwsch, inputRR, qb, selExprList); ArrayList columnNames = new ArrayList(); Map colExprMap = new HashMap(); @@ -3909,14 +3909,14 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer { * @see #handleInsertStatementSpecPhase1(ASTNode, QBParseInfo, org.apache.hadoop.hive.ql.parse.SemanticAnalyzer.Phase1Ctx) * @throws SemanticException */ - private void handleInsertStatementSpec(List col_list, String dest, + private RowResolver handleInsertStatementSpec(List col_list, String dest, RowResolver outputRR, RowResolver inputRR, QB qb, ASTNode selExprList) throws SemanticException { //(z,x) List targetTableSchema = qb.getParseInfo().getDestSchemaForClause(dest);//specified in the query if(targetTableSchema == null) { //no insert schema was specified - return; + return outputRR; } if(targetTableSchema.size() != col_list.size()) { Table target = qb.getMetaData().getDestTableForAlias(dest); @@ -3959,6 +3959,7 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer { } } } + RowResolver newOutputRR = new RowResolver(); //now make the select produce , with //where missing columns are NULL-filled for(String f : targetTableColNames) { @@ -3967,7 +3968,7 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer { new_col_list.add(targetCol2Projection.get(f)); ColumnInfo ci = targetCol2ColumnInfo.get(f);//todo: is this OK? ci.setInternalName(getColumnInternalName(colListPos)); - newSchema.add(ci); + newOutputRR.put(ci.getTabAlias(), ci.getInternalName(), ci); } else { //add new 'synthetic' columns for projections not provided by Select @@ -3979,14 +3980,13 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer { final String tableAlias = null;//this column doesn't come from any table ColumnInfo colInfo = new ColumnInfo(getColumnInternalName(colListPos), exp.getWritableObjectInspector(), tableAlias, false); - newSchema.add(colInfo); - outputRR.addMappingOnly(colInfo.getTabAlias(), colInfo.getInternalName(), colInfo); + newOutputRR.put(colInfo.getTabAlias(), colInfo.getInternalName(), colInfo); } colListPos++; } col_list.clear(); col_list.addAll(new_col_list); - outputRR.setRowSchema(new RowSchema(newSchema)); + return newOutputRR; } String recommendName(ExprNodeDesc exp, String colAlias) { if (!colAlias.startsWith(autogenColAliasPrfxLbl)) { http://git-wip-us.apache.org/repos/asf/hive/blob/49cc0287/ql/src/test/queries/clientpositive/insert_into_with_schema2.q ---------------------------------------------------------------------- diff --git a/ql/src/test/queries/clientpositive/insert_into_with_schema2.q b/ql/src/test/queries/clientpositive/insert_into_with_schema2.q index b7c6b58..a5352ec 100644 --- a/ql/src/test/queries/clientpositive/insert_into_with_schema2.q +++ b/ql/src/test/queries/clientpositive/insert_into_with_schema2.q @@ -21,3 +21,14 @@ insert into student_acid(grade, age) values(20, 2); insert into student_acid(age) values(22); select * from student_acid; + +set hive.exec.dynamic.partition.mode=nonstrict; + +drop table if exists acid_partitioned; +create table acid_partitioned (a int, c string) + partitioned by (p int) + clustered by (a) into 1 buckets; + +insert into acid_partitioned partition (p) (a,p) values(1,2); + +select * from acid_partitioned; http://git-wip-us.apache.org/repos/asf/hive/blob/49cc0287/ql/src/test/results/clientpositive/insert_into_with_schema2.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/insert_into_with_schema2.q.out b/ql/src/test/results/clientpositive/insert_into_with_schema2.q.out index a55a82f..32e6e92 100644 --- a/ql/src/test/results/clientpositive/insert_into_with_schema2.q.out +++ b/ql/src/test/results/clientpositive/insert_into_with_schema2.q.out @@ -96,3 +96,40 @@ POSTHOOK: Input: default@student_acid 1 NULL 2 20 22 NULL +PREHOOK: query: drop table if exists acid_partitioned +PREHOOK: type: DROPTABLE +POSTHOOK: query: drop table if exists acid_partitioned +POSTHOOK: type: DROPTABLE +PREHOOK: query: create table acid_partitioned (a int, c string) + partitioned by (p int) + clustered by (a) into 1 buckets +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@acid_partitioned +POSTHOOK: query: create table acid_partitioned (a int, c string) + partitioned by (p int) + clustered by (a) into 1 buckets +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@acid_partitioned +PREHOOK: query: insert into acid_partitioned partition (p) (a,p) values(1,2) +PREHOOK: type: QUERY +PREHOOK: Input: default@values__tmp__table__4 +PREHOOK: Output: default@acid_partitioned +POSTHOOK: query: insert into acid_partitioned partition (p) (a,p) values(1,2) +POSTHOOK: type: QUERY +POSTHOOK: Input: default@values__tmp__table__4 +POSTHOOK: Output: default@acid_partitioned@p=2 +POSTHOOK: Lineage: acid_partitioned PARTITION(p=2).a EXPRESSION [(values__tmp__table__4)values__tmp__table__4.FieldSchema(name:tmp_values_col1, type:string, comment:), ] +POSTHOOK: Lineage: acid_partitioned PARTITION(p=2).c SIMPLE [] +PREHOOK: query: select * from acid_partitioned +PREHOOK: type: QUERY +PREHOOK: Input: default@acid_partitioned +PREHOOK: Input: default@acid_partitioned@p=2 +#### A masked pattern was here #### +POSTHOOK: query: select * from acid_partitioned +POSTHOOK: type: QUERY +POSTHOOK: Input: default@acid_partitioned +POSTHOOK: Input: default@acid_partitioned@p=2 +#### A masked pattern was here #### +1 NULL 2