orc-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From omal...@apache.org
Subject [orc] branch branch-1.6 updated: ORC-526: Make orc-tools convert respect second fractions.
Date Tue, 08 Oct 2019 22:34:53 GMT
This is an automated email from the ASF dual-hosted git repository.

omalley pushed a commit to branch branch-1.6
in repository https://gitbox.apache.org/repos/asf/orc.git


The following commit(s) were added to refs/heads/branch-1.6 by this push:
     new 6afb694  ORC-526: Make orc-tools convert respect second fractions.
6afb694 is described below

commit 6afb6940648f6695a1a539af66150cc44348f3de
Author: Yukihiro Okada <callistoiv+git@gmail.com>
AuthorDate: Sat Aug 31 13:20:57 2019 +0900

    ORC-526: Make orc-tools convert respect second fractions.
    
    Fixes #425
    
    Signed-off-by: Owen O'Malley <omalley@apache.org>
---
 .../org/apache/orc/tools/convert/CsvReader.java    | 12 ++++++---
 .../org/apache/orc/tools/convert/JsonReader.java   | 10 ++++++--
 .../apache/orc/tools/convert/TestCsvReader.java    | 11 ++++----
 .../apache/orc/tools/convert/TestJsonReader.java   | 30 ++++++++++++++++++++++
 4 files changed, 52 insertions(+), 11 deletions(-)

diff --git a/java/tools/src/java/org/apache/orc/tools/convert/CsvReader.java b/java/tools/src/java/org/apache/orc/tools/convert/CsvReader.java
index 62c458d..3967a49 100644
--- a/java/tools/src/java/org/apache/orc/tools/convert/CsvReader.java
+++ b/java/tools/src/java/org/apache/orc/tools/convert/CsvReader.java
@@ -254,11 +254,15 @@ public class CsvReader implements RecordReader {
             dateTimeFormatter.parseBest(values[offset],
                 ZonedDateTime.FROM, LocalDateTime.FROM);
         if (temporalAccessor instanceof ZonedDateTime) {
-          vector.set(row, new Timestamp(
-              ((ZonedDateTime) temporalAccessor).toEpochSecond() * 1000L));
+          ZonedDateTime zonedDateTime = ((ZonedDateTime) temporalAccessor);
+          Timestamp timestamp = new Timestamp(zonedDateTime.toEpochSecond() * 1000L);
+          timestamp.setNanos(zonedDateTime.getNano());
+          vector.set(row, timestamp);
         } else if (temporalAccessor instanceof LocalDateTime) {
-          vector.set(row, new Timestamp(((LocalDateTime) temporalAccessor)
-              .atZone(ZoneId.systemDefault()).toEpochSecond() * 1000L));
+          ZonedDateTime tz = ((LocalDateTime) temporalAccessor).atZone(ZoneId.systemDefault());
+          Timestamp timestamp = new Timestamp(tz.toEpochSecond() * 1000L);
+          timestamp.setNanos(tz.getNano());
+          vector.set(row, timestamp);
         } else {
           column.noNulls = false;
           column.isNull[row] = true;
diff --git a/java/tools/src/java/org/apache/orc/tools/convert/JsonReader.java b/java/tools/src/java/org/apache/orc/tools/convert/JsonReader.java
index 7db3c67..c3720af 100644
--- a/java/tools/src/java/org/apache/orc/tools/convert/JsonReader.java
+++ b/java/tools/src/java/org/apache/orc/tools/convert/JsonReader.java
@@ -147,9 +147,15 @@ public class JsonReader implements RecordReader {
         TemporalAccessor temporalAccessor = dateTimeFormatter.parseBest(value.getAsString(),
           ZonedDateTime.FROM, LocalDateTime.FROM);
         if (temporalAccessor instanceof ZonedDateTime) {
-          vector.set(row, new Timestamp(((ZonedDateTime) temporalAccessor).toEpochSecond()
* 1000L));
+          ZonedDateTime zonedDateTime = ((ZonedDateTime) temporalAccessor);
+          Timestamp timestamp = new Timestamp(zonedDateTime.toEpochSecond() * 1000L);
+          timestamp.setNanos(zonedDateTime.getNano());
+          vector.set(row, timestamp);
         } else if (temporalAccessor instanceof LocalDateTime) {
-          vector.set(row, new Timestamp(((LocalDateTime) temporalAccessor).atZone(ZoneId.systemDefault()).toEpochSecond()
* 1000L));
+          ZonedDateTime tz = ((LocalDateTime) temporalAccessor).atZone(ZoneId.systemDefault());
+          Timestamp timestamp = new Timestamp(tz.toEpochSecond() * 1000L);
+          timestamp.setNanos(tz.getNano());
+          vector.set(row, timestamp);
         } else {
           vect.noNulls = false;
           vect.isNull[row] = true;
diff --git a/java/tools/src/test/org/apache/orc/tools/convert/TestCsvReader.java b/java/tools/src/test/org/apache/orc/tools/convert/TestCsvReader.java
index 6752de6..06ed5d2 100644
--- a/java/tools/src/test/org/apache/orc/tools/convert/TestCsvReader.java
+++ b/java/tools/src/test/org/apache/orc/tools/convert/TestCsvReader.java
@@ -183,10 +183,11 @@ public class TestCsvReader {
 
   @Test
   public void testCustomTimestampFormat() throws Exception {
-    String tsFormat = "d[d] MMM yyyy HH:mm:ss";
+    String tsFormat = "d[d] MMM yyyy HH:mm:ss.SSSSSS";
+
     StringReader input = new StringReader(
-            "'21 Mar 2018 12:23:34'\n" +
-                    "'3 Feb 2018 18:04:51'\n"
+            "'21 Mar 2018 12:23:34.123456'\n" +
+                    "'3 Feb 2018 18:04:51.456789'\n"
     );
     TypeDescription schema = TypeDescription.fromString(
             "struct<a:timestamp>");
@@ -196,7 +197,7 @@ public class TestCsvReader {
     assertEquals(true, reader.nextBatch(batch));
     assertEquals(2, batch.size);
     TimestampColumnVector cv = (TimestampColumnVector) batch.cols[0];
-    assertEquals("2018-03-21 12:23:34.0", cv.asScratchTimestamp(0).toString());
-    assertEquals("2018-02-03 18:04:51.0", cv.asScratchTimestamp(1).toString());
+    assertEquals("2018-03-21 12:23:34.123456", cv.asScratchTimestamp(0).toString());
+    assertEquals("2018-02-03 18:04:51.456789", cv.asScratchTimestamp(1).toString());
   }
 }
diff --git a/java/tools/src/test/org/apache/orc/tools/convert/TestJsonReader.java b/java/tools/src/test/org/apache/orc/tools/convert/TestJsonReader.java
new file mode 100644
index 0000000..10529d8
--- /dev/null
+++ b/java/tools/src/test/org/apache/orc/tools/convert/TestJsonReader.java
@@ -0,0 +1,30 @@
+package org.apache.orc.tools.convert;
+
+import org.apache.hadoop.hive.ql.exec.vector.TimestampColumnVector;
+import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch;
+import org.apache.orc.TypeDescription;
+import org.junit.Test;
+
+import java.io.StringReader;
+
+import static org.junit.Assert.assertEquals;
+
+public class TestJsonReader {
+    @Test
+    public void testCustomTimestampFormat() throws Exception {
+        String tsFormat = "yyyy-MM-dd HH:mm:ss.SSSSSS";
+
+        String s = "{\"a\":\"2018-03-21 12:23:34.123456\"}\n" +
+                "{\"a\":\"2018-02-03 18:04:51.456789\"}\n";
+        StringReader input = new StringReader(s);
+        TypeDescription schema = TypeDescription.fromString(
+                "struct<a:timestamp>");
+        JsonReader reader = new JsonReader(input, null, 1, schema, tsFormat);
+        VectorizedRowBatch batch = schema.createRowBatch(2);
+        assertEquals(true, reader.nextBatch(batch));
+        assertEquals(2, batch.size);
+        TimestampColumnVector cv = (TimestampColumnVector) batch.cols[0];
+        assertEquals("2018-03-21 12:23:34.123456", cv.asScratchTimestamp(0).toString());
+        assertEquals("2018-02-03 18:04:51.456789", cv.asScratchTimestamp(1).toString());
+    }
+}


Mime
View raw message