hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jd...@apache.org
Subject [6/6] hive git commit: HIVE-9917: After HIVE-3454 is done, make int to timestamp conversion configurable (Aihua Xu via Jason Dere)
Date Thu, 23 Apr 2015 18:26:40 GMT
HIVE-9917: After HIVE-3454 is done, make int to timestamp conversion configurable (Aihua Xu
via Jason Dere)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/84494ad6
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/84494ad6
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/84494ad6

Branch: refs/heads/master
Commit: 84494ad63150557147e09efbba2056e0f78b438d
Parents: fd65c4e
Author: Jason Dere <jdere@hortonworks.com>
Authored: Thu Apr 23 11:26:04 2015 -0700
Committer: Jason Dere <jdere@hortonworks.com>
Committed: Thu Apr 23 11:26:04 2015 -0700

----------------------------------------------------------------------
 .../apache/hadoop/hive/ant/GenVectorCode.java   |    2 +
 .../org/apache/hadoop/hive/conf/HiveConf.java   |    3 +
 .../src/test/queries/positive/hbase_timestamp.q |   22 +-
 .../test/results/positive/hbase_timestamp.q.out |   60 +-
 .../ql/exec/vector/VectorizationContext.java    |   16 +-
 .../ql/exec/vector/expressions/MathExpr.java    |    9 +-
 .../ql/udf/generic/GenericUDFTimestamp.java     |   24 +
 .../clientpositive/timestamp_ints_casts.q       |   74 +
 .../queries/clientpositive/vectorized_casts.q   |    2 +-
 .../vectorized_timestamp_ints_casts.q           |   76 +
 .../spark/vector_between_in.q.out               | 6088 +-----------------
 .../spark/vectorization_decimal_date.q.out      |   20 +-
 .../clientpositive/tez/vector_between_in.q.out  | 6088 +-----------------
 .../tez/vectorization_decimal_date.q.out        |   20 +-
 .../clientpositive/tez/vectorized_casts.q.out   |   52 +-
 .../clientpositive/timestamp_ints_casts.q.out   |  264 +
 .../clientpositive/vector_between_in.q.out      | 6088 +-----------------
 .../vectorization_decimal_date.q.out            |   20 +-
 .../clientpositive/vectorized_casts.q.out       |   52 +-
 .../vectorized_timestamp_ints_casts.q.out       |  266 +
 .../hive/serde2/io/TimestampWritable.java       |   11 +-
 .../PrimitiveObjectInspectorConverter.java      |    7 +-
 .../PrimitiveObjectInspectorUtils.java          |   42 +-
 .../TestPrimitiveObjectInspectorUtils.java      |   95 +-
 24 files changed, 1277 insertions(+), 18124 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/84494ad6/ant/src/org/apache/hadoop/hive/ant/GenVectorCode.java
----------------------------------------------------------------------
diff --git a/ant/src/org/apache/hadoop/hive/ant/GenVectorCode.java b/ant/src/org/apache/hadoop/hive/ant/GenVectorCode.java
index f7575a3..6485a2a 100644
--- a/ant/src/org/apache/hadoop/hive/ant/GenVectorCode.java
+++ b/ant/src/org/apache/hadoop/hive/ant/GenVectorCode.java
@@ -771,6 +771,8 @@ public class GenVectorCode extends Task {
             "", "", "timestamp"},
       {"ColumnUnaryFunc", "CastLongToTimestampVia", "long", "long", "MathExpr.longToTimestamp",
"",
           "", "", ""},
+      {"ColumnUnaryFunc", "CastMillisecondsLongToTimestampVia", "long", "long", "MathExpr.millisecondsLongToTimestamp",
"",
+            "", "", ""},
       {"ColumnUnaryFunc", "CastDoubleToTimestampVia", "long", "double",
          "MathExpr.doubleToTimestamp", "", "", "", ""},
 

http://git-wip-us.apache.org/repos/asf/hive/blob/84494ad6/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
----------------------------------------------------------------------
diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
index a3bd9aa..c9ee423 100644
--- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
+++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
@@ -919,6 +919,9 @@ public class HiveConf extends Configuration {
     HIVE_PARQUET_TIMESTAMP_SKIP_CONVERSION("hive.parquet.timestamp.skip.conversion", true,
       "Current Hive implementation of parquet stores timestamps to UTC, this flag allows
skipping of the conversion" +
       "on reading parquet files from other tools"),
+    HIVE_INT_TIMESTAMP_CONVERSION_IN_SECONDS("hive.int.timestamp.conversion.in.seconds",
false,
+        "Boolean/tinyint/smallint/int/bigint value is interpreted as milliseconds during
the timestamp conversion.\n" +
+        "Set this flag to true to interpret the value as seconds to be consistent with float/double."
),
     HIVE_ORC_FILE_MEMORY_POOL("hive.exec.orc.memory.pool", 0.5f,
         "Maximum fraction of heap that can be used by ORC file writers"),
     HIVE_ORC_WRITE_FORMAT("hive.exec.orc.write.format", null,

http://git-wip-us.apache.org/repos/asf/hive/blob/84494ad6/hbase-handler/src/test/queries/positive/hbase_timestamp.q
----------------------------------------------------------------------
diff --git a/hbase-handler/src/test/queries/positive/hbase_timestamp.q b/hbase-handler/src/test/queries/positive/hbase_timestamp.q
index 1a2cbdf..6f1e205 100644
--- a/hbase-handler/src/test/queries/positive/hbase_timestamp.q
+++ b/hbase-handler/src/test/queries/positive/hbase_timestamp.q
@@ -10,7 +10,7 @@ DROP TABLE hbase_table;
 CREATE TABLE hbase_table (key string, value string, time bigint)
   STORED BY 'org.apache.hadoop.hive.hbase.HBaseStorageHandler'
   WITH SERDEPROPERTIES ("hbase.columns.mapping" = ":key,cf:string,:timestamp");
-FROM src INSERT OVERWRITE TABLE hbase_table SELECT key, value, 1329959754 WHERE (key % 17)
= 0;
+FROM src INSERT OVERWRITE TABLE hbase_table SELECT key, value, 1329959754000 WHERE (key %
17) = 0;
 SELECT key, value, cast(time as timestamp) FROM hbase_table;
 
 DROP TABLE hbase_table;
@@ -19,23 +19,23 @@ CREATE TABLE hbase_table (key string, value string, time bigint)
   WITH SERDEPROPERTIES ("hbase.columns.mapping" = ":key,cf:string,:timestamp");
 insert overwrite table hbase_table select key,value,ts FROM
 (
-  select key, value, 100000000 as ts from src WHERE (key % 33) = 0
+  select key, value, 100000000000 as ts from src WHERE (key % 33) = 0
   UNION ALL
-  select key, value, 200000000 as ts from src WHERE (key % 37) = 0
+  select key, value, 200000000000 as ts from src WHERE (key % 37) = 0
 ) T;
 
 explain
-SELECT key, value, cast(time as timestamp) FROM hbase_table WHERE key > 100 AND key <
400 AND time < 200000000;
-SELECT key, value, cast(time as timestamp) FROM hbase_table WHERE key > 100 AND key <
400 AND time < 200000000;
+SELECT key, value, cast(time as timestamp) FROM hbase_table WHERE key > 100 AND key <
400 AND time < 200000000000;
+SELECT key, value, cast(time as timestamp) FROM hbase_table WHERE key > 100 AND key <
400 AND time < 200000000000;
 
 explain
-SELECT key, value, cast(time as timestamp) FROM hbase_table WHERE key > 100 AND key <
400 AND time > 100000000;
-SELECT key, value, cast(time as timestamp) FROM hbase_table WHERE key > 100 AND key <
400 AND time > 100000000;
+SELECT key, value, cast(time as timestamp) FROM hbase_table WHERE key > 100 AND key <
400 AND time > 100000000000;
+SELECT key, value, cast(time as timestamp) FROM hbase_table WHERE key > 100 AND key <
400 AND time > 100000000000;
 
 explain
-SELECT key, value, cast(time as timestamp) FROM hbase_table WHERE key > 100 AND key <
400 AND time <= 100000000;
-SELECT key, value, cast(time as timestamp) FROM hbase_table WHERE key > 100 AND key <
400 AND time <= 100000000;
+SELECT key, value, cast(time as timestamp) FROM hbase_table WHERE key > 100 AND key <
400 AND time <= 100000000000;
+SELECT key, value, cast(time as timestamp) FROM hbase_table WHERE key > 100 AND key <
400 AND time <= 100000000000;
 
 explain
-SELECT key, value, cast(time as timestamp) FROM hbase_table WHERE key > 100 AND key <
400 AND time >= 200000000;
-SELECT key, value, cast(time as timestamp) FROM hbase_table WHERE key > 100 AND key <
400 AND time >= 200000000;
+SELECT key, value, cast(time as timestamp) FROM hbase_table WHERE key > 100 AND key <
400 AND time >= 200000000000;
+SELECT key, value, cast(time as timestamp) FROM hbase_table WHERE key > 100 AND key <
400 AND time >= 200000000000;

http://git-wip-us.apache.org/repos/asf/hive/blob/84494ad6/hbase-handler/src/test/results/positive/hbase_timestamp.q.out
----------------------------------------------------------------------
diff --git a/hbase-handler/src/test/results/positive/hbase_timestamp.q.out b/hbase-handler/src/test/results/positive/hbase_timestamp.q.out
index 45c15ec..3aae7d0 100644
--- a/hbase-handler/src/test/results/positive/hbase_timestamp.q.out
+++ b/hbase-handler/src/test/results/positive/hbase_timestamp.q.out
@@ -81,11 +81,11 @@ POSTHOOK: query: CREATE TABLE hbase_table (key string, value string, time
bigint
 POSTHOOK: type: CREATETABLE
 POSTHOOK: Output: database:default
 POSTHOOK: Output: default@hbase_table
-PREHOOK: query: FROM src INSERT OVERWRITE TABLE hbase_table SELECT key, value, 1329959754
WHERE (key % 17) = 0
+PREHOOK: query: FROM src INSERT OVERWRITE TABLE hbase_table SELECT key, value, 1329959754000
WHERE (key % 17) = 0
 PREHOOK: type: QUERY
 PREHOOK: Input: default@src
 PREHOOK: Output: default@hbase_table
-POSTHOOK: query: FROM src INSERT OVERWRITE TABLE hbase_table SELECT key, value, 1329959754
WHERE (key % 17) = 0
+POSTHOOK: query: FROM src INSERT OVERWRITE TABLE hbase_table SELECT key, value, 1329959754000
WHERE (key % 17) = 0
 POSTHOOK: type: QUERY
 POSTHOOK: Input: default@src
 POSTHOOK: Output: default@hbase_table
@@ -139,27 +139,27 @@ POSTHOOK: Output: database:default
 POSTHOOK: Output: default@hbase_table
 PREHOOK: query: insert overwrite table hbase_table select key,value,ts FROM
 (
-  select key, value, 100000000 as ts from src WHERE (key % 33) = 0
+  select key, value, 100000000000 as ts from src WHERE (key % 33) = 0
   UNION ALL
-  select key, value, 200000000 as ts from src WHERE (key % 37) = 0
+  select key, value, 200000000000 as ts from src WHERE (key % 37) = 0
 ) T
 PREHOOK: type: QUERY
 PREHOOK: Input: default@src
 PREHOOK: Output: default@hbase_table
 POSTHOOK: query: insert overwrite table hbase_table select key,value,ts FROM
 (
-  select key, value, 100000000 as ts from src WHERE (key % 33) = 0
+  select key, value, 100000000000 as ts from src WHERE (key % 33) = 0
   UNION ALL
-  select key, value, 200000000 as ts from src WHERE (key % 37) = 0
+  select key, value, 200000000000 as ts from src WHERE (key % 37) = 0
 ) T
 POSTHOOK: type: QUERY
 POSTHOOK: Input: default@src
 POSTHOOK: Output: default@hbase_table
 PREHOOK: query: explain
-SELECT key, value, cast(time as timestamp) FROM hbase_table WHERE key > 100 AND key <
400 AND time < 200000000
+SELECT key, value, cast(time as timestamp) FROM hbase_table WHERE key > 100 AND key <
400 AND time < 200000000000
 PREHOOK: type: QUERY
 POSTHOOK: query: explain
-SELECT key, value, cast(time as timestamp) FROM hbase_table WHERE key > 100 AND key <
400 AND time < 200000000
+SELECT key, value, cast(time as timestamp) FROM hbase_table WHERE key > 100 AND key <
400 AND time < 200000000000
 POSTHOOK: type: QUERY
 STAGE DEPENDENCIES:
   Stage-1 is a root stage
@@ -171,10 +171,10 @@ STAGE PLANS:
       Map Operator Tree:
           TableScan
             alias: hbase_table
-            filterExpr: (((key > 100.0) and (key < 400.0)) and (time < 200000000))
(type: boolean)
+            filterExpr: (((key > 100.0) and (key < 400.0)) and (time < 200000000000))
(type: boolean)
             Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
             Filter Operator
-              predicate: ((UDFToDouble(key) > 100.0) and ((UDFToDouble(key) < 400.0)
and (time < 200000000))) (type: boolean)
+              predicate: ((UDFToDouble(key) > 100.0) and ((UDFToDouble(key) < 400.0)
and (time < 200000000000))) (type: boolean)
               Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
               Select Operator
                 expressions: key (type: string), value (type: string), CAST( time AS TIMESTAMP)
(type: timestamp)
@@ -194,21 +194,21 @@ STAGE PLANS:
       Processor Tree:
         ListSink
 
-PREHOOK: query: SELECT key, value, cast(time as timestamp) FROM hbase_table WHERE key >
100 AND key < 400 AND time < 200000000
+PREHOOK: query: SELECT key, value, cast(time as timestamp) FROM hbase_table WHERE key >
100 AND key < 400 AND time < 200000000000
 PREHOOK: type: QUERY
 PREHOOK: Input: default@hbase_table
 #### A masked pattern was here ####
-POSTHOOK: query: SELECT key, value, cast(time as timestamp) FROM hbase_table WHERE key >
100 AND key < 400 AND time < 200000000
+POSTHOOK: query: SELECT key, value, cast(time as timestamp) FROM hbase_table WHERE key >
100 AND key < 400 AND time < 200000000000
 POSTHOOK: type: QUERY
 POSTHOOK: Input: default@hbase_table
 #### A masked pattern was here ####
 165	val_165	1973-03-03 01:46:40
 396	val_396	1973-03-03 01:46:40
 PREHOOK: query: explain
-SELECT key, value, cast(time as timestamp) FROM hbase_table WHERE key > 100 AND key <
400 AND time > 100000000
+SELECT key, value, cast(time as timestamp) FROM hbase_table WHERE key > 100 AND key <
400 AND time > 100000000000
 PREHOOK: type: QUERY
 POSTHOOK: query: explain
-SELECT key, value, cast(time as timestamp) FROM hbase_table WHERE key > 100 AND key <
400 AND time > 100000000
+SELECT key, value, cast(time as timestamp) FROM hbase_table WHERE key > 100 AND key <
400 AND time > 100000000000
 POSTHOOK: type: QUERY
 STAGE DEPENDENCIES:
   Stage-1 is a root stage
@@ -220,10 +220,10 @@ STAGE PLANS:
       Map Operator Tree:
           TableScan
             alias: hbase_table
-            filterExpr: (((key > 100.0) and (key < 400.0)) and (time > 100000000))
(type: boolean)
+            filterExpr: (((key > 100.0) and (key < 400.0)) and (time > 100000000000))
(type: boolean)
             Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
             Filter Operator
-              predicate: ((UDFToDouble(key) > 100.0) and ((UDFToDouble(key) < 400.0)
and (time > 100000000))) (type: boolean)
+              predicate: ((UDFToDouble(key) > 100.0) and ((UDFToDouble(key) < 400.0)
and (time > 100000000000))) (type: boolean)
               Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
               Select Operator
                 expressions: key (type: string), value (type: string), CAST( time AS TIMESTAMP)
(type: timestamp)
@@ -243,11 +243,11 @@ STAGE PLANS:
       Processor Tree:
         ListSink
 
-PREHOOK: query: SELECT key, value, cast(time as timestamp) FROM hbase_table WHERE key >
100 AND key < 400 AND time > 100000000
+PREHOOK: query: SELECT key, value, cast(time as timestamp) FROM hbase_table WHERE key >
100 AND key < 400 AND time > 100000000000
 PREHOOK: type: QUERY
 PREHOOK: Input: default@hbase_table
 #### A masked pattern was here ####
-POSTHOOK: query: SELECT key, value, cast(time as timestamp) FROM hbase_table WHERE key >
100 AND key < 400 AND time > 100000000
+POSTHOOK: query: SELECT key, value, cast(time as timestamp) FROM hbase_table WHERE key >
100 AND key < 400 AND time > 100000000000
 POSTHOOK: type: QUERY
 POSTHOOK: Input: default@hbase_table
 #### A masked pattern was here ####
@@ -256,10 +256,10 @@ POSTHOOK: Input: default@hbase_table
 296	val_296	1976-05-03 12:33:20
 333	val_333	1976-05-03 12:33:20
 PREHOOK: query: explain
-SELECT key, value, cast(time as timestamp) FROM hbase_table WHERE key > 100 AND key <
400 AND time <= 100000000
+SELECT key, value, cast(time as timestamp) FROM hbase_table WHERE key > 100 AND key <
400 AND time <= 100000000000
 PREHOOK: type: QUERY
 POSTHOOK: query: explain
-SELECT key, value, cast(time as timestamp) FROM hbase_table WHERE key > 100 AND key <
400 AND time <= 100000000
+SELECT key, value, cast(time as timestamp) FROM hbase_table WHERE key > 100 AND key <
400 AND time <= 100000000000
 POSTHOOK: type: QUERY
 STAGE DEPENDENCIES:
   Stage-1 is a root stage
@@ -271,10 +271,10 @@ STAGE PLANS:
       Map Operator Tree:
           TableScan
             alias: hbase_table
-            filterExpr: (((key > 100.0) and (key < 400.0)) and (time <= 100000000))
(type: boolean)
+            filterExpr: (((key > 100.0) and (key < 400.0)) and (time <= 100000000000))
(type: boolean)
             Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
             Filter Operator
-              predicate: ((UDFToDouble(key) > 100.0) and ((UDFToDouble(key) < 400.0)
and (time <= 100000000))) (type: boolean)
+              predicate: ((UDFToDouble(key) > 100.0) and ((UDFToDouble(key) < 400.0)
and (time <= 100000000000))) (type: boolean)
               Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
               Select Operator
                 expressions: key (type: string), value (type: string), CAST( time AS TIMESTAMP)
(type: timestamp)
@@ -294,21 +294,21 @@ STAGE PLANS:
       Processor Tree:
         ListSink
 
-PREHOOK: query: SELECT key, value, cast(time as timestamp) FROM hbase_table WHERE key >
100 AND key < 400 AND time <= 100000000
+PREHOOK: query: SELECT key, value, cast(time as timestamp) FROM hbase_table WHERE key >
100 AND key < 400 AND time <= 100000000000
 PREHOOK: type: QUERY
 PREHOOK: Input: default@hbase_table
 #### A masked pattern was here ####
-POSTHOOK: query: SELECT key, value, cast(time as timestamp) FROM hbase_table WHERE key >
100 AND key < 400 AND time <= 100000000
+POSTHOOK: query: SELECT key, value, cast(time as timestamp) FROM hbase_table WHERE key >
100 AND key < 400 AND time <= 100000000000
 POSTHOOK: type: QUERY
 POSTHOOK: Input: default@hbase_table
 #### A masked pattern was here ####
 165	val_165	1973-03-03 01:46:40
 396	val_396	1973-03-03 01:46:40
 PREHOOK: query: explain
-SELECT key, value, cast(time as timestamp) FROM hbase_table WHERE key > 100 AND key <
400 AND time >= 200000000
+SELECT key, value, cast(time as timestamp) FROM hbase_table WHERE key > 100 AND key <
400 AND time >= 200000000000
 PREHOOK: type: QUERY
 POSTHOOK: query: explain
-SELECT key, value, cast(time as timestamp) FROM hbase_table WHERE key > 100 AND key <
400 AND time >= 200000000
+SELECT key, value, cast(time as timestamp) FROM hbase_table WHERE key > 100 AND key <
400 AND time >= 200000000000
 POSTHOOK: type: QUERY
 STAGE DEPENDENCIES:
   Stage-1 is a root stage
@@ -320,10 +320,10 @@ STAGE PLANS:
       Map Operator Tree:
           TableScan
             alias: hbase_table
-            filterExpr: (((key > 100.0) and (key < 400.0)) and (time >= 200000000))
(type: boolean)
+            filterExpr: (((key > 100.0) and (key < 400.0)) and (time >= 200000000000))
(type: boolean)
             Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
             Filter Operator
-              predicate: ((UDFToDouble(key) > 100.0) and ((UDFToDouble(key) < 400.0)
and (time >= 200000000))) (type: boolean)
+              predicate: ((UDFToDouble(key) > 100.0) and ((UDFToDouble(key) < 400.0)
and (time >= 200000000000))) (type: boolean)
               Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
               Select Operator
                 expressions: key (type: string), value (type: string), CAST( time AS TIMESTAMP)
(type: timestamp)
@@ -343,11 +343,11 @@ STAGE PLANS:
       Processor Tree:
         ListSink
 
-PREHOOK: query: SELECT key, value, cast(time as timestamp) FROM hbase_table WHERE key >
100 AND key < 400 AND time >= 200000000
+PREHOOK: query: SELECT key, value, cast(time as timestamp) FROM hbase_table WHERE key >
100 AND key < 400 AND time >= 200000000000
 PREHOOK: type: QUERY
 PREHOOK: Input: default@hbase_table
 #### A masked pattern was here ####
-POSTHOOK: query: SELECT key, value, cast(time as timestamp) FROM hbase_table WHERE key >
100 AND key < 400 AND time >= 200000000
+POSTHOOK: query: SELECT key, value, cast(time as timestamp) FROM hbase_table WHERE key >
100 AND key < 400 AND time >= 200000000000
 POSTHOOK: type: QUERY
 POSTHOOK: Input: default@hbase_table
 #### A masked pattern was here ####

http://git-wip-us.apache.org/repos/asf/hive/blob/84494ad6/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java
index abce606..df39218 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java
@@ -1152,8 +1152,9 @@ public class VectorizationContext {
       return getCastToChar(childExpr, returnType);
     } else if (udf instanceof GenericUDFToVarchar) {
       return getCastToVarChar(childExpr, returnType);
+    } else if (udf instanceof GenericUDFTimestamp) {
+      return getCastToTimestamp((GenericUDFTimestamp)udf, childExpr, mode, returnType);
     }
-
     // Now do a general lookup
     Class<?> udfClass = udf.getClass();
     if (udf instanceof GenericUDFBridge) {
@@ -1169,6 +1170,19 @@ public class VectorizationContext {
     return ve;
   }
 
+  private VectorExpression getCastToTimestamp(GenericUDFTimestamp udf,
+      List<ExprNodeDesc> childExpr, Mode mode, TypeInfo returnType) throws HiveException
{
+    VectorExpression ve = getVectorExpressionForUdf(udf.getClass(), childExpr, mode, returnType);
+
+    // Replace with the milliseconds conversion
+    if (!udf.isIntToTimestampInSeconds() && ve instanceof CastLongToTimestampViaLongToLong)
{
+      ve = createVectorExpression(CastMillisecondsLongToTimestampViaLongToLong.class,
+          childExpr, Mode.PROJECTION, returnType);
+    }
+
+    return ve;
+  }
+
   private VectorExpression getCoalesceExpression(List<ExprNodeDesc> childExpr, TypeInfo
returnType)
       throws HiveException {
     int[] inputColumns = new int[childExpr.size()];

http://git-wip-us.apache.org/repos/asf/hive/blob/84494ad6/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/MathExpr.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/MathExpr.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/MathExpr.java
index 3709aa3..aef923e 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/MathExpr.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/MathExpr.java
@@ -18,8 +18,6 @@
 
 package org.apache.hadoop.hive.ql.exec.vector.expressions;
 
-import java.io.IOException;
-import java.io.OutputStream;
 import org.apache.hadoop.hive.ql.exec.vector.DoubleColumnVector;
 
 /**
@@ -82,6 +80,13 @@ public class MathExpr {
     return v * 1000000000;
   }
 
+  /* Convert an integer value in milliseconds since the epoch to a timestamp value
+   * for use in a long column vector, which is represented in nanoseconds since the epoch.
+   */
+  public static long millisecondsLongToTimestamp(long v) {
+    return v * 1000000;
+  }
+
   // Convert seconds since the epoch (with fraction) to nanoseconds, as a long integer.
   public static long doubleToTimestamp(double v) {
     return (long)( v * 1000000000.0);

http://git-wip-us.apache.org/repos/asf/hive/blob/84494ad6/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFTimestamp.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFTimestamp.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFTimestamp.java
index 2f854f4..24b49a0 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFTimestamp.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFTimestamp.java
@@ -17,6 +17,9 @@
  */
 package org.apache.hadoop.hive.ql.udf.generic;
 
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
 import org.apache.hadoop.hive.ql.exec.Description;
 import org.apache.hadoop.hive.ql.exec.UDFArgumentException;
 import org.apache.hadoop.hive.ql.exec.UDFArgumentLengthException;
@@ -25,6 +28,8 @@ import org.apache.hadoop.hive.ql.exec.vector.expressions.CastDecimalToTimestamp;
 import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.CastDoubleToTimestampViaDoubleToLong;
 import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.CastLongToTimestampViaLongToLong;
 import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.ql.session.SessionState;
+import org.apache.hadoop.hive.ql.session.SessionState.LogHelper;
 import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
 import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector;
 import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorConverter.TimestampConverter;
@@ -48,6 +53,14 @@ public class GenericUDFTimestamp extends GenericUDF {
 
   private transient PrimitiveObjectInspector argumentOI;
   private transient TimestampConverter tc;
+  /*
+   * Integer value was interpreted to timestamp inconsistently in milliseconds comparing
+   * to float/double in seconds. Since the issue exists for a long time and some users may
+   * use in such inconsistent way, use the following flag to keep backward compatible.
+   * If the flag is set to false, integer value is interpreted as timestamp in milliseconds;
+   * otherwise, it's interpreted as timestamp in seconds.
+   */
+  private boolean intToTimestampInSeconds = false;
 
   @Override
   public ObjectInspector initialize(ObjectInspector[] arguments) throws UDFArgumentException
{
@@ -56,6 +69,12 @@ public class GenericUDFTimestamp extends GenericUDF {
           "The function TIMESTAMP requires at least one argument, got "
           + arguments.length);
     }
+
+    SessionState ss = SessionState.get();
+    if (ss != null) {
+      intToTimestampInSeconds = ss.getConf().getBoolVar(ConfVars.HIVE_INT_TIMESTAMP_CONVERSION_IN_SECONDS);
+    }
+
     try {
       argumentOI = (PrimitiveObjectInspector) arguments[0];
     } catch (ClassCastException e) {
@@ -65,6 +84,8 @@ public class GenericUDFTimestamp extends GenericUDF {
 
     tc = new TimestampConverter(argumentOI,
         PrimitiveObjectInspectorFactory.writableTimestampObjectInspector);
+    tc.setIntToTimestampInSeconds(intToTimestampInSeconds);
+
     return PrimitiveObjectInspectorFactory.writableTimestampObjectInspector;
   }
 
@@ -88,4 +109,7 @@ public class GenericUDFTimestamp extends GenericUDF {
     return sb.toString();
   }
 
+  public boolean isIntToTimestampInSeconds() {
+    return intToTimestampInSeconds;
+  }
 }

http://git-wip-us.apache.org/repos/asf/hive/blob/84494ad6/ql/src/test/queries/clientpositive/timestamp_ints_casts.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/timestamp_ints_casts.q b/ql/src/test/queries/clientpositive/timestamp_ints_casts.q
new file mode 100644
index 0000000..b043d59
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/timestamp_ints_casts.q
@@ -0,0 +1,74 @@
+set hive.int.timestamp.conversion.in.seconds=false;
+
+explain
+select
+-- to timestamp
+  cast (ctinyint as timestamp)
+  ,cast (csmallint as timestamp)
+  ,cast (cint as timestamp)
+  ,cast (cbigint as timestamp)
+  ,cast (cfloat as timestamp)
+  ,cast (cdouble as timestamp)
+  ,cast (cboolean1 as timestamp)
+  ,cast (cbigint * 0 as timestamp)
+  ,cast (ctimestamp1 as timestamp)
+  ,cast (cstring1 as timestamp)
+  ,cast (substr(cstring1, 1, 1) as timestamp)
+from alltypesorc
+-- limit output to a reasonably small number of rows
+where cbigint % 250 = 0;
+
+select
+-- to timestamp
+  cast (ctinyint as timestamp)
+  ,cast (csmallint as timestamp)
+  ,cast (cint as timestamp)
+  ,cast (cbigint as timestamp)
+  ,cast (cfloat as timestamp)
+  ,cast (cdouble as timestamp)
+  ,cast (cboolean1 as timestamp)
+  ,cast (cbigint * 0 as timestamp)
+  ,cast (ctimestamp1 as timestamp)
+  ,cast (cstring1 as timestamp)
+  ,cast (substr(cstring1, 1, 1) as timestamp)
+from alltypesorc
+-- limit output to a reasonably small number of rows
+where cbigint % 250 = 0;
+
+set hive.int.timestamp.conversion.in.seconds=true;
+
+explain
+select
+-- to timestamp
+  cast (ctinyint as timestamp)
+  ,cast (csmallint as timestamp)
+  ,cast (cint as timestamp)
+  ,cast (cbigint as timestamp)
+  ,cast (cfloat as timestamp)
+  ,cast (cdouble as timestamp)
+  ,cast (cboolean1 as timestamp)
+  ,cast (cbigint * 0 as timestamp)
+  ,cast (ctimestamp1 as timestamp)
+  ,cast (cstring1 as timestamp)
+  ,cast (substr(cstring1, 1, 1) as timestamp)
+from alltypesorc
+-- limit output to a reasonably small number of rows
+where cbigint % 250 = 0;
+
+select
+-- to timestamp
+  cast (ctinyint as timestamp)
+  ,cast (csmallint as timestamp)
+  ,cast (cint as timestamp)
+  ,cast (cbigint as timestamp)
+  ,cast (cfloat as timestamp)
+  ,cast (cdouble as timestamp)
+  ,cast (cboolean1 as timestamp)
+  ,cast (cbigint * 0 as timestamp)
+  ,cast (ctimestamp1 as timestamp)
+  ,cast (cstring1 as timestamp)
+  ,cast (substr(cstring1, 1, 1) as timestamp)
+from alltypesorc
+-- limit output to a reasonably small number of rows
+where cbigint % 250 = 0;
+

http://git-wip-us.apache.org/repos/asf/hive/blob/84494ad6/ql/src/test/queries/clientpositive/vectorized_casts.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/vectorized_casts.q b/ql/src/test/queries/clientpositive/vectorized_casts.q
index 3f818b1..9147a6a 100644
--- a/ql/src/test/queries/clientpositive/vectorized_casts.q
+++ b/ql/src/test/queries/clientpositive/vectorized_casts.q
@@ -146,4 +146,4 @@ from alltypesorc
 -- limit output to a reasonably small number of rows
 where cbigint % 250 = 0;
 
- 
\ No newline at end of file
+ 

http://git-wip-us.apache.org/repos/asf/hive/blob/84494ad6/ql/src/test/queries/clientpositive/vectorized_timestamp_ints_casts.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/vectorized_timestamp_ints_casts.q b/ql/src/test/queries/clientpositive/vectorized_timestamp_ints_casts.q
new file mode 100644
index 0000000..f297070
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/vectorized_timestamp_ints_casts.q
@@ -0,0 +1,76 @@
+SET hive.vectorized.execution.enabled = true;
+SET hive.int.timestamp.conversion.in.seconds=false;
+
+explain
+select
+-- to timestamp
+  cast (ctinyint as timestamp)
+  ,cast (csmallint as timestamp)
+  ,cast (cint as timestamp)
+  ,cast (cbigint as timestamp)
+  ,cast (cfloat as timestamp)
+  ,cast (cdouble as timestamp)
+  ,cast (cboolean1 as timestamp)
+  ,cast (cbigint * 0 as timestamp)
+  ,cast (ctimestamp1 as timestamp)
+  ,cast (cstring1 as timestamp)
+  ,cast (substr(cstring1, 1, 1) as timestamp)
+from alltypesorc
+-- limit output to a reasonably small number of rows
+where cbigint % 250 = 0;
+
+
+select
+-- to timestamp
+  cast (ctinyint as timestamp)
+  ,cast (csmallint as timestamp)
+  ,cast (cint as timestamp)
+  ,cast (cbigint as timestamp)
+  ,cast (cfloat as timestamp)
+  ,cast (cdouble as timestamp)
+  ,cast (cboolean1 as timestamp)
+  ,cast (cbigint * 0 as timestamp)
+  ,cast (ctimestamp1 as timestamp)
+  ,cast (cstring1 as timestamp)
+  ,cast (substr(cstring1, 1, 1) as timestamp)
+from alltypesorc
+-- limit output to a reasonably small number of rows
+where cbigint % 250 = 0;
+
+SET hive.int.timestamp.conversion.in.seconds=true;
+
+explain
+select
+-- to timestamp
+  cast (ctinyint as timestamp)
+  ,cast (csmallint as timestamp)
+  ,cast (cint as timestamp)
+  ,cast (cbigint as timestamp)
+  ,cast (cfloat as timestamp)
+  ,cast (cdouble as timestamp)
+  ,cast (cboolean1 as timestamp)
+  ,cast (cbigint * 0 as timestamp)
+  ,cast (ctimestamp1 as timestamp)
+  ,cast (cstring1 as timestamp)
+  ,cast (substr(cstring1, 1, 1) as timestamp)
+from alltypesorc
+-- limit output to a reasonably small number of rows
+where cbigint % 250 = 0;
+
+
+select
+-- to timestamp
+  cast (ctinyint as timestamp)
+  ,cast (csmallint as timestamp)
+  ,cast (cint as timestamp)
+  ,cast (cbigint as timestamp)
+  ,cast (cfloat as timestamp)
+  ,cast (cdouble as timestamp)
+  ,cast (cboolean1 as timestamp)
+  ,cast (cbigint * 0 as timestamp)
+  ,cast (ctimestamp1 as timestamp)
+  ,cast (cstring1 as timestamp)
+  ,cast (substr(cstring1, 1, 1) as timestamp)
+from alltypesorc
+-- limit output to a reasonably small number of rows
+where cbigint % 250 = 0;


Mime
View raw message