Return-Path: X-Original-To: apmail-hive-commits-archive@www.apache.org Delivered-To: apmail-hive-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 70E0F10692 for ; Wed, 25 Sep 2013 04:25:55 +0000 (UTC) Received: (qmail 97923 invoked by uid 500); 25 Sep 2013 04:25:54 -0000 Delivered-To: apmail-hive-commits-archive@hive.apache.org Received: (qmail 97903 invoked by uid 500); 25 Sep 2013 04:25:54 -0000 Mailing-List: contact commits-help@hive.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: hive-dev@hive.apache.org Delivered-To: mailing list commits@hive.apache.org Received: (qmail 97894 invoked by uid 99); 25 Sep 2013 04:25:53 -0000 Received: from athena.apache.org (HELO athena.apache.org) (140.211.11.136) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 25 Sep 2013 04:25:53 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=5.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 25 Sep 2013 04:25:51 +0000 Received: from eris.apache.org (localhost [127.0.0.1]) by eris.apache.org (Postfix) with ESMTP id 603A423888A6; Wed, 25 Sep 2013 04:25:31 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r1526102 - in /hive/trunk: ql/src/java/org/apache/hadoop/hive/ql/udf/generic/ ql/src/test/queries/clientpositive/ ql/src/test/results/clientpositive/ serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/ serde/src/test/org... Date: Wed, 25 Sep 2013 04:25:30 -0000 To: commits@hive.apache.org From: hashutosh@apache.org X-Mailer: svnmailer-1.0.9 Message-Id: <20130925042531.603A423888A6@eris.apache.org> X-Virus-Checked: Checked by ClamAV on apache.org Author: hashutosh Date: Wed Sep 25 04:25:30 2013 New Revision: 1526102 URL: http://svn.apache.org/r1526102 Log: HIVE-5329 : Date and timestamp type converts invalid strings to 1970-01-01 (Jason Dere via Ashutosh Chauhan) Added: hive/trunk/ql/src/test/queries/clientpositive/type_conversions_1.q hive/trunk/ql/src/test/results/clientpositive/type_conversions_1.q.out Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFToDate.java hive/trunk/ql/src/test/queries/clientpositive/partition_date.q hive/trunk/ql/src/test/results/clientpositive/partition_date.q.out hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/JavaDateObjectInspector.java hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/JavaTimestampObjectInspector.java hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/PrimitiveObjectInspectorUtils.java hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableDateObjectInspector.java hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableTimestampObjectInspector.java hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/objectinspector/primitive/TestPrimitiveObjectInspectorUtils.java Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFToDate.java URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFToDate.java?rev=1526102&r1=1526101&r2=1526102&view=diff ============================================================================== --- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFToDate.java (original) +++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFToDate.java Wed Sep 25 04:25:30 2013 @@ -23,6 +23,7 @@ import org.apache.hadoop.hive.ql.exec.UD import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory; import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorConverter.DateConverter; import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils; @@ -51,11 +52,13 @@ public class GenericUDFToDate extends Ge } try { argumentOI = (PrimitiveObjectInspector) arguments[0]; + PrimitiveCategory pc = argumentOI.getPrimitiveCategory(); PrimitiveGrouping pg = - PrimitiveObjectInspectorUtils.getPrimitiveGrouping(argumentOI.getPrimitiveCategory()); + PrimitiveObjectInspectorUtils.getPrimitiveGrouping(pc); switch (pg) { case DATE_GROUP: case STRING_GROUP: + case VOID_GROUP: break; default: throw new UDFArgumentException( Modified: hive/trunk/ql/src/test/queries/clientpositive/partition_date.q URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientpositive/partition_date.q?rev=1526102&r1=1526101&r2=1526102&view=diff ============================================================================== --- hive/trunk/ql/src/test/queries/clientpositive/partition_date.q (original) +++ hive/trunk/ql/src/test/queries/clientpositive/partition_date.q Wed Sep 25 04:25:30 2013 @@ -12,7 +12,7 @@ insert overwrite table partition_date_1 select * from src limit 11; select distinct dt from partition_date_1; -select *, cast(dt as timestamp) from partition_date_1 where dt = '2000-01-01' and region = 2 order by key,value; +select * from partition_date_1 where dt = '2000-01-01' and region = 2 order by key,value; -- 15 select count(*) from partition_date_1 where dt = date '2000-01-01'; Added: hive/trunk/ql/src/test/queries/clientpositive/type_conversions_1.q URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientpositive/type_conversions_1.q?rev=1526102&view=auto ============================================================================== --- hive/trunk/ql/src/test/queries/clientpositive/type_conversions_1.q (added) +++ hive/trunk/ql/src/test/queries/clientpositive/type_conversions_1.q Wed Sep 25 04:25:30 2013 @@ -0,0 +1,24 @@ + +-- casting from null should yield null +select + cast(null as tinyint), + cast(null as smallint), + cast(null as int), + cast(null as bigint), + cast(null as float), + cast(null as double), + cast(null as decimal), + cast(null as date), + cast(null as timestamp), + cast(null as string), + cast(null as varchar(10)), + cast(null as boolean), + cast(null as binary) +from src limit 1; + +-- Invalid conversions, should all be null +select + cast('abcd' as date), + cast('abcd' as timestamp) +from src limit 1; + Modified: hive/trunk/ql/src/test/results/clientpositive/partition_date.q.out URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/partition_date.q.out?rev=1526102&r1=1526101&r2=1526102&view=diff ============================================================================== --- hive/trunk/ql/src/test/results/clientpositive/partition_date.q.out (original) +++ hive/trunk/ql/src/test/results/clientpositive/partition_date.q.out Wed Sep 25 04:25:30 2013 @@ -93,12 +93,12 @@ POSTHOOK: Lineage: partition_date_1 PART POSTHOOK: Lineage: partition_date_1 PARTITION(dt=2013-08-08,region=1).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] 2000-01-01 2013-08-08 -PREHOOK: query: select *, cast(dt as timestamp) from partition_date_1 where dt = '2000-01-01' and region = 2 order by key,value +PREHOOK: query: select * from partition_date_1 where dt = '2000-01-01' and region = 2 order by key,value PREHOOK: type: QUERY PREHOOK: Input: default@partition_date_1 PREHOOK: Input: default@partition_date_1@dt=2000-01-01/region=2 #### A masked pattern was here #### -POSTHOOK: query: select *, cast(dt as timestamp) from partition_date_1 where dt = '2000-01-01' and region = 2 order by key,value +POSTHOOK: query: select * from partition_date_1 where dt = '2000-01-01' and region = 2 order by key,value POSTHOOK: type: QUERY POSTHOOK: Input: default@partition_date_1 POSTHOOK: Input: default@partition_date_1@dt=2000-01-01/region=2 @@ -111,11 +111,11 @@ POSTHOOK: Lineage: partition_date_1 PART POSTHOOK: Lineage: partition_date_1 PARTITION(dt=2013-08-08,region=10).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] POSTHOOK: Lineage: partition_date_1 PARTITION(dt=2013-08-08,region=1).key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ] POSTHOOK: Lineage: partition_date_1 PARTITION(dt=2013-08-08,region=1).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ] -165 val_165 2000-01-01 2 1969-12-31 16:00:00 -238 val_238 2000-01-01 2 1969-12-31 16:00:00 -27 val_27 2000-01-01 2 1969-12-31 16:00:00 -311 val_311 2000-01-01 2 1969-12-31 16:00:00 -86 val_86 2000-01-01 2 1969-12-31 16:00:00 +165 val_165 2000-01-01 2 +238 val_238 2000-01-01 2 +27 val_27 2000-01-01 2 +311 val_311 2000-01-01 2 +86 val_86 2000-01-01 2 PREHOOK: query: -- 15 select count(*) from partition_date_1 where dt = date '2000-01-01' PREHOOK: type: QUERY Added: hive/trunk/ql/src/test/results/clientpositive/type_conversions_1.q.out URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/type_conversions_1.q.out?rev=1526102&view=auto ============================================================================== --- hive/trunk/ql/src/test/results/clientpositive/type_conversions_1.q.out (added) +++ hive/trunk/ql/src/test/results/clientpositive/type_conversions_1.q.out Wed Sep 25 04:25:30 2013 @@ -0,0 +1,56 @@ +PREHOOK: query: -- casting from null should yield null +select + cast(null as tinyint), + cast(null as smallint), + cast(null as int), + cast(null as bigint), + cast(null as float), + cast(null as double), + cast(null as decimal), + cast(null as date), + cast(null as timestamp), + cast(null as string), + cast(null as varchar(10)), + cast(null as boolean), + cast(null as binary) +from src limit 1 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +#### A masked pattern was here #### +POSTHOOK: query: -- casting from null should yield null +select + cast(null as tinyint), + cast(null as smallint), + cast(null as int), + cast(null as bigint), + cast(null as float), + cast(null as double), + cast(null as decimal), + cast(null as date), + cast(null as timestamp), + cast(null as string), + cast(null as varchar(10)), + cast(null as boolean), + cast(null as binary) +from src limit 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +#### A masked pattern was here #### +NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL +PREHOOK: query: -- Invalid conversions, should all be null +select + cast('abcd' as date), + cast('abcd' as timestamp) +from src limit 1 +PREHOOK: type: QUERY +PREHOOK: Input: default@src +#### A masked pattern was here #### +POSTHOOK: query: -- Invalid conversions, should all be null +select + cast('abcd' as date), + cast('abcd' as timestamp) +from src limit 1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@src +#### A masked pattern was here #### +NULL NULL Modified: hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/JavaDateObjectInspector.java URL: http://svn.apache.org/viewvc/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/JavaDateObjectInspector.java?rev=1526102&r1=1526101&r2=1526102&view=diff ============================================================================== --- hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/JavaDateObjectInspector.java (original) +++ hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/JavaDateObjectInspector.java Wed Sep 25 04:25:30 2013 @@ -46,11 +46,17 @@ public class JavaDateObjectInspector } public Object set(Object o, Date value) { + if (value == null) { + return null; + } ((Date) o).setTime(value.getTime()); return o; } public Object set(Object o, DateWritable d) { + if (d == null) { + return null; + } ((Date) o).setTime(d.get().getTime()); return o; } Modified: hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/JavaTimestampObjectInspector.java URL: http://svn.apache.org/viewvc/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/JavaTimestampObjectInspector.java?rev=1526102&r1=1526101&r2=1526102&view=diff ============================================================================== --- hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/JavaTimestampObjectInspector.java (original) +++ hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/JavaTimestampObjectInspector.java Wed Sep 25 04:25:30 2013 @@ -43,6 +43,9 @@ public class JavaTimestampObjectInspecto } public Object set(Object o, Timestamp value) { + if (value == null) { + return null; + } ((Timestamp) o).setTime(value.getTime()); return o; } @@ -53,6 +56,9 @@ public class JavaTimestampObjectInspecto } public Object set(Object o, TimestampWritable tw) { + if (tw == null) { + return null; + } Timestamp t = (Timestamp) o; t.setTime(tw.getTimestamp().getTime()); t.setNanos(tw.getTimestamp().getNanos()); Modified: hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/PrimitiveObjectInspectorUtils.java URL: http://svn.apache.org/viewvc/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/PrimitiveObjectInspectorUtils.java?rev=1526102&r1=1526101&r2=1526102&view=diff ============================================================================== --- hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/PrimitiveObjectInspectorUtils.java (original) +++ hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/PrimitiveObjectInspectorUtils.java Wed Sep 25 04:25:30 2013 @@ -1166,7 +1166,8 @@ public final class PrimitiveObjectInspec * Provide a general grouping for each primitive data type. */ public static enum PrimitiveGrouping { - NUMERIC_GROUP, STRING_GROUP, BOOLEAN_GROUP, DATE_GROUP, BINARY_GROUP, UNKNOWN_GROUP + NUMERIC_GROUP, STRING_GROUP, BOOLEAN_GROUP, DATE_GROUP, BINARY_GROUP, + VOID_GROUP, UNKNOWN_GROUP }; /** @@ -1196,6 +1197,8 @@ public final class PrimitiveObjectInspec return PrimitiveGrouping.DATE_GROUP; case BINARY: return PrimitiveGrouping.BINARY_GROUP; + case VOID: + return PrimitiveGrouping.VOID_GROUP; default: return PrimitiveGrouping.UNKNOWN_GROUP; } Modified: hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableDateObjectInspector.java URL: http://svn.apache.org/viewvc/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableDateObjectInspector.java?rev=1526102&r1=1526101&r2=1526102&view=diff ============================================================================== --- hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableDateObjectInspector.java (original) +++ hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableDateObjectInspector.java Wed Sep 25 04:25:30 2013 @@ -46,11 +46,17 @@ public class WritableDateObjectInspector } public Object set(Object o, Date d) { + if (d == null) { + return null; + } ((DateWritable) o).set(d); return o; } public Object set(Object o, DateWritable d) { + if (d == null) { + return null; + } ((DateWritable) o).set(d); return o; } Modified: hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableTimestampObjectInspector.java URL: http://svn.apache.org/viewvc/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableTimestampObjectInspector.java?rev=1526102&r1=1526101&r2=1526102&view=diff ============================================================================== --- hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableTimestampObjectInspector.java (original) +++ hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableTimestampObjectInspector.java Wed Sep 25 04:25:30 2013 @@ -48,11 +48,17 @@ public class WritableTimestampObjectInsp } public Object set(Object o, Timestamp t) { + if (t == null) { + return null; + } ((TimestampWritable) o).set(t); return o; } public Object set(Object o, TimestampWritable t) { + if (t == null) { + return null; + } ((TimestampWritable) o).set(t); return o; } Modified: hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/objectinspector/primitive/TestPrimitiveObjectInspectorUtils.java URL: http://svn.apache.org/viewvc/hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/objectinspector/primitive/TestPrimitiveObjectInspectorUtils.java?rev=1526102&r1=1526101&r2=1526102&view=diff ============================================================================== --- hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/objectinspector/primitive/TestPrimitiveObjectInspectorUtils.java (original) +++ hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/objectinspector/primitive/TestPrimitiveObjectInspectorUtils.java Wed Sep 25 04:25:30 2013 @@ -39,7 +39,7 @@ public class TestPrimitiveObjectInspecto assertEquals(PrimitiveGrouping.UNKNOWN_GROUP, PrimitiveObjectInspectorUtils.getPrimitiveGrouping(PrimitiveCategory.UNKNOWN)); - assertEquals(PrimitiveGrouping.UNKNOWN_GROUP, + assertEquals(PrimitiveGrouping.VOID_GROUP, PrimitiveObjectInspectorUtils.getPrimitiveGrouping(PrimitiveCategory.VOID)); } }