From commits-return-3070-archive-asf-public=cust-asf.ponee.io@impala.apache.org Fri Feb 2 19:51:28 2018 Return-Path: X-Original-To: archive-asf-public@eu.ponee.io Delivered-To: archive-asf-public@eu.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by mx-eu-01.ponee.io (Postfix) with ESMTP id 16BAE18067A for ; Fri, 2 Feb 2018 19:51:28 +0100 (CET) Received: by cust-asf.ponee.io (Postfix) id 06BA3160C49; Fri, 2 Feb 2018 18:51:28 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id 1C759160C59 for ; Fri, 2 Feb 2018 19:51:26 +0100 (CET) Received: (qmail 52764 invoked by uid 500); 2 Feb 2018 18:51:26 -0000 Mailing-List: contact commits-help@impala.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@impala.apache.org Delivered-To: mailing list commits@impala.apache.org Received: (qmail 52650 invoked by uid 99); 2 Feb 2018 18:51:26 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 02 Feb 2018 18:51:26 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 8E826EE68A; Fri, 2 Feb 2018 18:51:25 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: philz@apache.org To: commits@impala.apache.org Date: Fri, 02 Feb 2018 18:51:28 -0000 Message-Id: <660abaf7fab94a8fad5f5f1dedc1f41e@git.apache.org> In-Reply-To: <4b43b408f27e4168be25c92d099add70@git.apache.org> References: <4b43b408f27e4168be25c92d099add70@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [04/19] impala git commit: IMPALA-6454: CTAS into Kudu fails with mixed-case partition or primary key column names. IMPALA-6454: CTAS into Kudu fails with mixed-case partition or primary key column names. CTAS into Kudu fails if the primary key and/or the partition column names are not specified in lower case.The problem is that we pass in the primary key column names directly from the parser instead we should be passing the post-analysis ColumnDefs as primary keys. So it is fixed by making changes to createCtasTarget() in KuduTable class to take a list of ColumnDef class associated with the primary keys from getPrimaryKeyColumnDefs() in CreateTableStmt class. ColumnDef class has column names stored in lower case that are used by createCtasTarget() to populate primaryKeyColumnNames_ in KuduTable class that resolves the issue. Testing ------- Verified against the newly added test case that reproduces the issue without the fix. Change-Id: Ica1c8ec1544339e9e80733a7a0c78594e0a727d2 Reviewed-on: http://gerrit.cloudera.org:8080/9147 Reviewed-by: Alex Behm Tested-by: Impala Public Jenkins Project: http://git-wip-us.apache.org/repos/asf/impala/repo Commit: http://git-wip-us.apache.org/repos/asf/impala/commit/530fa27c Tree: http://git-wip-us.apache.org/repos/asf/impala/tree/530fa27c Diff: http://git-wip-us.apache.org/repos/asf/impala/diff/530fa27c Branch: refs/heads/2.x Commit: 530fa27c5cb9fc327c0b2c1518dc7edc1eb5c4fa Parents: a326d51 Author: Pranay Authored: Fri Jan 26 18:36:59 2018 -0800 Committer: Impala Public Jenkins Committed: Fri Feb 2 01:10:15 2018 +0000 ---------------------------------------------------------------------- .../org/apache/impala/analysis/CreateTableAsSelectStmt.java | 2 +- .../java/org/apache/impala/analysis/CreateTableStmt.java | 2 +- fe/src/main/java/org/apache/impala/catalog/KuduTable.java | 8 +++++--- .../test/java/org/apache/impala/analysis/AnalyzeDDLTest.java | 7 +++++++ 4 files changed, 14 insertions(+), 5 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/impala/blob/530fa27c/fe/src/main/java/org/apache/impala/analysis/CreateTableAsSelectStmt.java ---------------------------------------------------------------------- diff --git a/fe/src/main/java/org/apache/impala/analysis/CreateTableAsSelectStmt.java b/fe/src/main/java/org/apache/impala/analysis/CreateTableAsSelectStmt.java index ba9a02a..e050d06 100644 --- a/fe/src/main/java/org/apache/impala/analysis/CreateTableAsSelectStmt.java +++ b/fe/src/main/java/org/apache/impala/analysis/CreateTableAsSelectStmt.java @@ -191,7 +191,7 @@ public class CreateTableAsSelectStmt extends StatementBase { Table tmpTable = null; if (KuduTable.isKuduTable(msTbl)) { tmpTable = KuduTable.createCtasTarget(db, msTbl, createStmt_.getColumnDefs(), - createStmt_.getTblPrimaryKeyColumnNames(), + createStmt_.getPrimaryKeyColumnDefs(), createStmt_.getKuduPartitionParams()); } else { // TODO: Creating a tmp table using load() is confusing. http://git-wip-us.apache.org/repos/asf/impala/blob/530fa27c/fe/src/main/java/org/apache/impala/analysis/CreateTableStmt.java ---------------------------------------------------------------------- diff --git a/fe/src/main/java/org/apache/impala/analysis/CreateTableStmt.java b/fe/src/main/java/org/apache/impala/analysis/CreateTableStmt.java index b4d59e8..2e5425a 100644 --- a/fe/src/main/java/org/apache/impala/analysis/CreateTableStmt.java +++ b/fe/src/main/java/org/apache/impala/analysis/CreateTableStmt.java @@ -94,7 +94,7 @@ public class CreateTableStmt extends StatementBase { getColumnDefs().clear(); getColumnDefs().addAll(colDefs); } - private List getPrimaryKeyColumnDefs() { + public List getPrimaryKeyColumnDefs() { return tableDef_.getPrimaryKeyColumnDefs(); } public boolean isExternal() { return tableDef_.isExternal(); } http://git-wip-us.apache.org/repos/asf/impala/blob/530fa27c/fe/src/main/java/org/apache/impala/catalog/KuduTable.java ---------------------------------------------------------------------- diff --git a/fe/src/main/java/org/apache/impala/catalog/KuduTable.java b/fe/src/main/java/org/apache/impala/catalog/KuduTable.java index e9e1617..8296ed0 100644 --- a/fe/src/main/java/org/apache/impala/catalog/KuduTable.java +++ b/fe/src/main/java/org/apache/impala/catalog/KuduTable.java @@ -107,7 +107,7 @@ public class KuduTable extends Table { // Comma separated list of Kudu master hosts with optional ports. private String kuduMasters_; - // Primary key column names. + // Primary key column names, the column names are all in lower case. private final List primaryKeyColumnNames_ = Lists.newArrayList(); // Partitioning schemes of this Kudu table. Both range and hash-based partitioning are @@ -318,13 +318,15 @@ public class KuduTable extends Table { */ public static KuduTable createCtasTarget(Db db, org.apache.hadoop.hive.metastore.api.Table msTbl, List columnDefs, - List primaryKeyColumnNames, List partitionParams) { + List primaryKeyColumnDefs, List partitionParams) { KuduTable tmpTable = new KuduTable(msTbl, db, msTbl.getTableName(), msTbl.getOwner()); int pos = 0; for (ColumnDef colDef: columnDefs) { tmpTable.addColumn(new Column(colDef.getColName(), colDef.getType(), pos++)); } - tmpTable.primaryKeyColumnNames_.addAll(primaryKeyColumnNames); + for (ColumnDef pkColDef: primaryKeyColumnDefs) { + tmpTable.primaryKeyColumnNames_.add(pkColDef.getColName()); + } tmpTable.partitionBy_.addAll(partitionParams); return tmpTable; } http://git-wip-us.apache.org/repos/asf/impala/blob/530fa27c/fe/src/test/java/org/apache/impala/analysis/AnalyzeDDLTest.java ---------------------------------------------------------------------- diff --git a/fe/src/test/java/org/apache/impala/analysis/AnalyzeDDLTest.java b/fe/src/test/java/org/apache/impala/analysis/AnalyzeDDLTest.java index 3083f1f..122b49d 100644 --- a/fe/src/test/java/org/apache/impala/analysis/AnalyzeDDLTest.java +++ b/fe/src/test/java/org/apache/impala/analysis/AnalyzeDDLTest.java @@ -1733,6 +1733,13 @@ public class AnalyzeDDLTest extends FrontendTestBase { " stored as kudu as select id, a from functional.complextypes_fileformat", "Expr 'a' in select list returns a complex type 'ARRAY'.\n" + "Only scalar types are allowed in the select list."); + + // IMPALA-6454: CTAS into Kudu tables with primary key specified in upper case. + AnalyzesOk("create table part_kudu_tbl primary key(INT_COL, SMALLINT_COL, ID)" + + " partition by hash(INT_COL, SMALLINT_COL, ID) PARTITIONS 2" + + " stored as kudu as SELECT INT_COL, SMALLINT_COL, ID, BIGINT_COL," + + " DATE_STRING_COL, STRING_COL, TIMESTAMP_COL, YEAR, MONTH FROM " + + " functional.alltypes"); } @Test