Return-Path: X-Original-To: apmail-hive-commits-archive@www.apache.org Delivered-To: apmail-hive-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id B9A21183A7 for ; Thu, 3 Dec 2015 04:35:47 +0000 (UTC) Received: (qmail 31972 invoked by uid 500); 3 Dec 2015 04:35:47 -0000 Delivered-To: apmail-hive-commits-archive@hive.apache.org Received: (qmail 31930 invoked by uid 500); 3 Dec 2015 04:35:47 -0000 Mailing-List: contact commits-help@hive.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: hive-dev@hive.apache.org Delivered-To: mailing list commits@hive.apache.org Received: (qmail 31919 invoked by uid 99); 3 Dec 2015 04:35:47 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Thu, 03 Dec 2015 04:35:47 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 40A88E67CA; Thu, 3 Dec 2015 04:35:47 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: prasanthj@apache.org To: commits@hive.apache.org Message-Id: <367cbcc8ecc24289b4919e8151ed84c9@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: hive git commit: HIVE-11312: ORC format: where clause with CHAR data type not returning any rows (Prasanth Jayachandran reviewed by Sergey Shelukhin) Date: Thu, 3 Dec 2015 04:35:47 +0000 (UTC) Repository: hive Updated Branches: refs/heads/branch-1 eaec01a61 -> 7db94f072 HIVE-11312: ORC format: where clause with CHAR data type not returning any rows (Prasanth Jayachandran reviewed by Sergey Shelukhin) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/7db94f07 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/7db94f07 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/7db94f07 Branch: refs/heads/branch-1 Commit: 7db94f072a51c6fc3f8fe22af5505f6bc43d63b2 Parents: eaec01a Author: Prasanth Jayachandran Authored: Wed Dec 2 22:35:34 2015 -0600 Committer: Prasanth Jayachandran Committed: Wed Dec 2 22:35:34 2015 -0600 ---------------------------------------------------------------------- .../hive/ql/parse/TypeCheckProcFactory.java | 15 +++ .../clientpositive/orc_ppd_str_conversion.q | 17 ++++ .../results/clientpositive/orc_ppd_char.q.out | 4 +- .../clientpositive/orc_ppd_str_conversion.q.out | 99 ++++++++++++++++++++ 4 files changed, 133 insertions(+), 2 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/7db94f07/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactory.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactory.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactory.java index 0e97530..99df1bb 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactory.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactory.java @@ -34,6 +34,7 @@ import java.util.Stack; import org.apache.commons.lang.StringUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hive.common.type.HiveChar; import org.apache.hadoop.hive.common.type.HiveDecimal; import org.apache.hadoop.hive.common.type.HiveIntervalDayTime; import org.apache.hadoop.hive.common.type.HiveIntervalYearMonth; @@ -76,6 +77,7 @@ import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.StructTypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory; +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils; import org.apache.hadoop.hive.serde2.typeinfo.VarcharTypeInfo; import org.apache.hadoop.io.NullWritable; import org.apache.hive.common.util.DateUtils; @@ -1043,6 +1045,19 @@ public class TypeCheckProcFactory { children.set(constIdx, new ExprNodeConstantDesc(value)); } } + + // if column type is char and constant type is string, then convert the constant to char + // type with padded spaces. + final PrimitiveTypeInfo colTypeInfo = TypeInfoFactory + .getPrimitiveTypeInfo(columnType); + if (constType.equalsIgnoreCase(serdeConstants.STRING_TYPE_NAME) && + colTypeInfo instanceof CharTypeInfo) { + final Object originalValue = ((ExprNodeConstantDesc) children.get(constIdx)).getValue(); + final String constValue = originalValue.toString(); + final int length = TypeInfoUtils.getCharacterLengthForType(colTypeInfo); + final HiveChar newValue = new HiveChar(constValue, length); + children.set(constIdx, new ExprNodeConstantDesc(colTypeInfo, newValue)); + } } desc = ExprNodeGenericFuncDesc.newInstance(genericUDF, funcText, children); http://git-wip-us.apache.org/repos/asf/hive/blob/7db94f07/ql/src/test/queries/clientpositive/orc_ppd_str_conversion.q ---------------------------------------------------------------------- diff --git a/ql/src/test/queries/clientpositive/orc_ppd_str_conversion.q b/ql/src/test/queries/clientpositive/orc_ppd_str_conversion.q new file mode 100644 index 0000000..768e3f7 --- /dev/null +++ b/ql/src/test/queries/clientpositive/orc_ppd_str_conversion.q @@ -0,0 +1,17 @@ +set hive.cbo.enable=false; + +create table orc_test( col1 varchar(15), col2 char(10)) stored as orc; +create table text_test( col1 varchar(15), col2 char(10)); + +insert into orc_test values ('val1', '1'); +insert overwrite table text_test select * from orc_test; + +explain select * from text_test where col2='1'; +select * from text_test where col2='1'; + +set hive.optimize.index.filter=false; +select * from orc_test where col2='1'; + +set hive.optimize.index.filter=true; +select * from orc_test where col2='1'; + http://git-wip-us.apache.org/repos/asf/hive/blob/7db94f07/ql/src/test/results/clientpositive/orc_ppd_char.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/orc_ppd_char.q.out b/ql/src/test/results/clientpositive/orc_ppd_char.q.out index 79838c1..8af2c5a 100644 --- a/ql/src/test/results/clientpositive/orc_ppd_char.q.out +++ b/ql/src/test/results/clientpositive/orc_ppd_char.q.out @@ -100,7 +100,7 @@ POSTHOOK: query: select sum(hash(*)) from newtypesorc where c="apple " POSTHOOK: type: QUERY POSTHOOK: Input: default@newtypesorc #### A masked pattern was here #### -NULL +-252951929000 PREHOOK: query: select sum(hash(*)) from newtypesorc where c="apple " PREHOOK: type: QUERY PREHOOK: Input: default@newtypesorc @@ -109,7 +109,7 @@ POSTHOOK: query: select sum(hash(*)) from newtypesorc where c="apple " POSTHOOK: type: QUERY POSTHOOK: Input: default@newtypesorc #### A masked pattern was here #### -NULL +-252951929000 PREHOOK: query: select sum(hash(*)) from newtypesorc where c in ("apple", "carrot") PREHOOK: type: QUERY PREHOOK: Input: default@newtypesorc http://git-wip-us.apache.org/repos/asf/hive/blob/7db94f07/ql/src/test/results/clientpositive/orc_ppd_str_conversion.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/orc_ppd_str_conversion.q.out b/ql/src/test/results/clientpositive/orc_ppd_str_conversion.q.out new file mode 100644 index 0000000..0e6dc59 --- /dev/null +++ b/ql/src/test/results/clientpositive/orc_ppd_str_conversion.q.out @@ -0,0 +1,99 @@ +PREHOOK: query: create table orc_test( col1 varchar(15), col2 char(10)) stored as orc +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@orc_test +POSTHOOK: query: create table orc_test( col1 varchar(15), col2 char(10)) stored as orc +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@orc_test +PREHOOK: query: create table text_test( col1 varchar(15), col2 char(10)) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@text_test +POSTHOOK: query: create table text_test( col1 varchar(15), col2 char(10)) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@text_test +PREHOOK: query: insert into orc_test values ('val1', '1') +PREHOOK: type: QUERY +PREHOOK: Input: default@values__tmp__table__1 +PREHOOK: Output: default@orc_test +POSTHOOK: query: insert into orc_test values ('val1', '1') +POSTHOOK: type: QUERY +POSTHOOK: Input: default@values__tmp__table__1 +POSTHOOK: Output: default@orc_test +POSTHOOK: Lineage: orc_test.col1 EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col1, type:string, comment:), ] +POSTHOOK: Lineage: orc_test.col2 EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col2, type:string, comment:), ] +PREHOOK: query: insert overwrite table text_test select * from orc_test +PREHOOK: type: QUERY +PREHOOK: Input: default@orc_test +PREHOOK: Output: default@text_test +POSTHOOK: query: insert overwrite table text_test select * from orc_test +POSTHOOK: type: QUERY +POSTHOOK: Input: default@orc_test +POSTHOOK: Output: default@text_test +POSTHOOK: Lineage: text_test.col1 SIMPLE [(orc_test)orc_test.FieldSchema(name:col1, type:varchar(15), comment:null), ] +POSTHOOK: Lineage: text_test.col2 SIMPLE [(orc_test)orc_test.FieldSchema(name:col2, type:char(10), comment:null), ] +PREHOOK: query: explain select * from text_test where col2='1' +PREHOOK: type: QUERY +POSTHOOK: query: explain select * from text_test where col2='1' +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Map Reduce + Map Operator Tree: + TableScan + alias: text_test + Statistics: Num rows: 1 Data size: 15 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (col2 = '1 ') (type: boolean) + Statistics: Num rows: 1 Data size: 15 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: col1 (type: varchar(15)), '1 ' (type: char(10)) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 15 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 15 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.TextInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + +PREHOOK: query: select * from text_test where col2='1' +PREHOOK: type: QUERY +PREHOOK: Input: default@text_test +#### A masked pattern was here #### +POSTHOOK: query: select * from text_test where col2='1' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@text_test +#### A masked pattern was here #### +val1 1 +PREHOOK: query: select * from orc_test where col2='1' +PREHOOK: type: QUERY +PREHOOK: Input: default@orc_test +#### A masked pattern was here #### +POSTHOOK: query: select * from orc_test where col2='1' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@orc_test +#### A masked pattern was here #### +val1 1 +PREHOOK: query: select * from orc_test where col2='1' +PREHOOK: type: QUERY +PREHOOK: Input: default@orc_test +#### A masked pattern was here #### +POSTHOOK: query: select * from orc_test where col2='1' +POSTHOOK: type: QUERY +POSTHOOK: Input: default@orc_test +#### A masked pattern was here #### +val1 1