hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jd...@apache.org
Subject hive git commit: HIVE-12706: Incorrect output from from_utc_timestamp()/to_utc_timestamp when local timezone has DST (Jason Dere, reviewed by Ashutosh Chauhan)
Date Tue, 05 Jan 2016 00:12:46 GMT
Repository: hive
Updated Branches:
  refs/heads/master f7579b282 -> f0faf78dc


HIVE-12706: Incorrect output from from_utc_timestamp()/to_utc_timestamp when local timezone
has DST (Jason Dere, reviewed by Ashutosh Chauhan)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/f0faf78d
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/f0faf78d
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/f0faf78d

Branch: refs/heads/master
Commit: f0faf78dc79f9b9a6dc8c7fbc9a57ab0f57c6262
Parents: f7579b2
Author: Jason Dere <jdere@hortonworks.com>
Authored: Mon Jan 4 16:12:10 2016 -0800
Committer: Jason Dere <jdere@hortonworks.com>
Committed: Mon Jan 4 16:12:10 2016 -0800

----------------------------------------------------------------------
 .../udf/generic/GenericUDFFromUtcTimestamp.java | 59 ++++++++++---
 .../generic/TestGenericUDFFromUtcTimestamp.java | 92 ++++++++++++++++++++
 2 files changed, 141 insertions(+), 10 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/f0faf78d/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFFromUtcTimestamp.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFFromUtcTimestamp.java
b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFFromUtcTimestamp.java
index 331ee6b..33fe507 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFFromUtcTimestamp.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFFromUtcTimestamp.java
@@ -18,6 +18,8 @@
 package org.apache.hadoop.hive.ql.udf.generic;
 
 import java.sql.Timestamp;
+import java.text.ParseException;
+import java.text.SimpleDateFormat;
 import java.util.TimeZone;
 
 import org.slf4j.Logger;
@@ -43,6 +45,8 @@ public class GenericUDFFromUtcTimestamp extends GenericUDF {
   private transient PrimitiveObjectInspector[] argumentOIs;
   private transient TimestampConverter timestampConverter;
   private transient TextConverter textConverter;
+  private transient SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
+  private transient TimeZone tzUTC = TimeZone.getTimeZone("UTC");
 
   @Override
   public ObjectInspector initialize(ObjectInspector[] arguments)
@@ -66,6 +70,26 @@ public class GenericUDFFromUtcTimestamp extends GenericUDF {
     return PrimitiveObjectInspectorFactory.javaTimestampObjectInspector;
   }
 
+  /**
+   * Parse the timestamp string using the input TimeZone.
+   * This does not parse fractional seconds.
+   * @param tsString
+   * @param tz
+   * @return
+   */
+  protected Timestamp timestampFromString(String tsString, TimeZone tz) {
+    dateFormat.setTimeZone(tz);
+    try {
+      java.util.Date date = dateFormat.parse(tsString);
+      if (date == null) {
+        return null;
+      }
+      return new Timestamp(date.getTime());
+    } catch (ParseException err) {
+      return null;
+    }
+  }
+
   @Override
   public Object evaluate(DeferredObject[] arguments) throws HiveException {
     Object o0 = arguments[0].get();
@@ -82,23 +106,38 @@ public class GenericUDFFromUtcTimestamp extends GenericUDF {
       return null;
     }
 
-    Timestamp timestamp = ((TimestampWritable) converted_o0).getTimestamp();
+    Timestamp inputTs = ((TimestampWritable) converted_o0).getTimestamp();
 
     String tzStr = textConverter.convert(o1).toString();
     TimeZone timezone = TimeZone.getTimeZone(tzStr);
-    int offset = timezone.getOffset(timestamp.getTime());
+
+    TimeZone fromTz;
+    TimeZone toTz;
     if (invert()) {
-      offset = -offset;
+      fromTz = timezone;
+      toTz = tzUTC;
+    } else {
+      fromTz = tzUTC;
+      toTz = timezone;
+    }
+
+    // inputTs is the year/month/day/hour/minute/second in the local timezone.
+    // For this UDF we want it in the timezone represented by fromTz
+    Timestamp fromTs = timestampFromString(inputTs.toString(), fromTz);
+    if (fromTs == null) {
+      return null;
+    }
+
+    // Now output this timestamp's millis value to the equivalent toTz.
+    dateFormat.setTimeZone(toTz);
+    Timestamp result = Timestamp.valueOf(dateFormat.format(fromTs));
+
+    if (inputTs.getNanos() != 0) {
+      result.setNanos(inputTs.getNanos());
     }
-    return applyOffset(offset, timestamp);
-  }
 
-  protected Timestamp applyOffset(long offset, Timestamp t) {
-    long newTime = t.getTime() + offset;
-    Timestamp t2 = new Timestamp(newTime);
-    t2.setNanos(t.getNanos());
+    return result;
 
-    return t2;
   }
 
   @Override

http://git-wip-us.apache.org/repos/asf/hive/blob/f0faf78d/ql/src/test/org/apache/hadoop/hive/ql/udf/generic/TestGenericUDFFromUtcTimestamp.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/udf/generic/TestGenericUDFFromUtcTimestamp.java
b/ql/src/test/org/apache/hadoop/hive/ql/udf/generic/TestGenericUDFFromUtcTimestamp.java
new file mode 100644
index 0000000..ae4785a
--- /dev/null
+++ b/ql/src/test/org/apache/hadoop/hive/ql/udf/generic/TestGenericUDFFromUtcTimestamp.java
@@ -0,0 +1,92 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.udf.generic;
+
+import java.sql.Date;
+import java.sql.Timestamp;
+
+import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDF.DeferredJavaObject;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDF.DeferredObject;
+import org.apache.hadoop.hive.serde2.io.DateWritable;
+import org.apache.hadoop.hive.serde2.io.TimestampWritable;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
+import org.apache.hadoop.io.LongWritable;
+import org.apache.hadoop.io.Text;
+
+import junit.framework.TestCase;
+
+public class TestGenericUDFFromUtcTimestamp extends TestCase {
+  public static void runAndVerify(GenericUDF udf,
+      Object arg1, Object arg2, Object expected) throws HiveException {
+    DeferredObject[] args = { new DeferredJavaObject(arg1), new DeferredJavaObject(arg2)
};
+    Object result = udf.evaluate(args);
+
+    if (expected == null) {
+      assertNull(result);
+    } else {
+      assertEquals(expected.toString(), result.toString());
+    }
+  }
+
+  public void testFromUtcTimestamp() throws Exception {
+    ObjectInspector valueOI = PrimitiveObjectInspectorFactory.writableStringObjectInspector;
+    GenericUDFFromUtcTimestamp udf = new GenericUDFFromUtcTimestamp();
+    ObjectInspector[] args2 = {valueOI, valueOI};
+    udf.initialize(args2);
+
+    runAndVerify(udf,
+        new Text("2015-03-28 17:00:00"), new Text("Europe/London"),
+        Timestamp.valueOf("2015-03-28 17:00:00"));
+    runAndVerify(udf,
+        new Text("2015-03-28 18:00:00"), new Text("Europe/London"),
+        Timestamp.valueOf("2015-03-28 18:00:00"));
+    runAndVerify(udf,
+        new Text("2015-03-28 19:00:00"), new Text("Europe/London"),
+        Timestamp.valueOf("2015-03-28 19:00:00"));
+
+    // Make sure nanos are preserved
+    runAndVerify(udf,
+        new Text("2015-03-28 18:00:00.123456789"), new Text("Europe/London"),
+        Timestamp.valueOf("2015-03-28 18:00:00.123456789"));
+  }
+
+  public void testToUtcTimestamp() throws Exception {
+    ObjectInspector valueOI = PrimitiveObjectInspectorFactory.writableStringObjectInspector;
+    GenericUDFToUtcTimestamp udf = new GenericUDFToUtcTimestamp();
+    ObjectInspector[] args2 = {valueOI, valueOI};
+    udf.initialize(args2);
+
+    runAndVerify(udf,
+        new Text("2015-03-28 17:00:00"), new Text("Europe/London"),
+        Timestamp.valueOf("2015-03-28 17:00:00"));
+    runAndVerify(udf,
+        new Text("2015-03-28 18:00:00"), new Text("Europe/London"),
+        Timestamp.valueOf("2015-03-28 18:00:00"));
+    runAndVerify(udf,
+        new Text("2015-03-28 19:00:00"), new Text("Europe/London"),
+        Timestamp.valueOf("2015-03-28 19:00:00"));
+
+    // Make sure nanos are preserved
+    runAndVerify(udf,
+        new Text("2015-03-28 18:00:00.123456789"), new Text("Europe/London"),
+        Timestamp.valueOf("2015-03-28 18:00:00.123456789"));
+  }
+}


Mime
View raw message