parquet-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ziva...@apache.org
Subject [parquet-mr] branch master updated: PARQUET-1436: TimestampMicrosStringifier shows wrong microseconds for timestamps before 1970 (#529)
Date Fri, 05 Oct 2018 09:16:54 GMT
This is an automated email from the ASF dual-hosted git repository.

zivanfi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/parquet-mr.git


The following commit(s) were added to refs/heads/master by this push:
     new 63003ac  PARQUET-1436: TimestampMicrosStringifier shows wrong microseconds for timestamps
before 1970 (#529)
63003ac is described below

commit 63003ac204369c077f8e7148799b22c2aa10d2d6
Author: nandorKollar <nandorKollar@users.noreply.github.com>
AuthorDate: Fri Oct 5 11:16:42 2018 +0200

    PARQUET-1436: TimestampMicrosStringifier shows wrong microseconds for timestamps before
1970 (#529)
---
 .../parquet/schema/PrimitiveStringifier.java       | 57 +++++++++-------------
 .../parquet/schema/TestPrimitiveStringifier.java   |  8 +--
 2 files changed, 27 insertions(+), 38 deletions(-)

diff --git a/parquet-column/src/main/java/org/apache/parquet/schema/PrimitiveStringifier.java
b/parquet-column/src/main/java/org/apache/parquet/schema/PrimitiveStringifier.java
index 3c3417e..03786ed 100644
--- a/parquet-column/src/main/java/org/apache/parquet/schema/PrimitiveStringifier.java
+++ b/parquet-column/src/main/java/org/apache/parquet/schema/PrimitiveStringifier.java
@@ -29,8 +29,9 @@ import java.math.BigDecimal;
 import java.math.BigInteger;
 import java.nio.ByteBuffer;
 import java.nio.ByteOrder;
-import java.text.SimpleDateFormat;
-import java.util.TimeZone;
+import java.time.Instant;
+import java.time.ZoneOffset;
+import java.time.format.DateTimeFormatter;
 import java.util.concurrent.TimeUnit;
 
 import javax.naming.OperationNotSupportedException;
@@ -243,80 +244,68 @@ public abstract class PrimitiveStringifier {
   };
 
   private static class DateStringifier extends PrimitiveStringifier {
-    private final SimpleDateFormat formatter;
-    private static final TimeZone UTC = TimeZone.getTimeZone("utc");
+    private final DateTimeFormatter formatter;
 
     private DateStringifier(String name, String format) {
       super(name);
-      formatter = new SimpleDateFormat(format);
-      formatter.setTimeZone(UTC);
+      formatter = DateTimeFormatter.ofPattern(format).withZone(ZoneOffset.UTC);
     }
 
     @Override
     public String stringify(int value) {
-      return toFormattedString(toMillis(value));
+      return toFormattedString(getInstant(value));
     }
 
     @Override
     public String stringify(long value) {
-      return toFormattedString(toMillis(value));
+      return toFormattedString(getInstant(value));
     }
 
-    private String toFormattedString(long millis) {
-      return formatter.format(millis);
+    private String toFormattedString(Instant instant) {
+      return formatter.format(instant);
     }
 
-    long toMillis(int value) {
+    Instant getInstant(int value) {
       // throw the related unsupported exception
       super.stringify(value);
-      return 0;
+      return null;
     }
 
-    long toMillis(long value) {
+    Instant getInstant(long value) {
       // throw the related unsupported exception
       super.stringify(value);
-      return 0;
+      return null;
     }
   }
 
   static final PrimitiveStringifier DATE_STRINGIFIER = new DateStringifier("DATE_STRINGIFIER",
"yyyy-MM-dd") {
     @Override
-    long toMillis(int value) {
-      return TimeUnit.DAYS.toMillis(value);
+    Instant getInstant(int value) {
+      return Instant.ofEpochMilli(TimeUnit.DAYS.toMillis(value));
     };
   };
 
   static final PrimitiveStringifier TIMESTAMP_MILLIS_STRINGIFIER = new DateStringifier(
       "TIMESTAMP_MILLIS_STRINGIFIER", "yyyy-MM-dd'T'HH:mm:ss.SSS") {
     @Override
-    long toMillis(long value) {
-      return value;
+    Instant getInstant(long value) {
+      return Instant.ofEpochMilli(value);
     }
   };
 
   static final PrimitiveStringifier TIMESTAMP_MICROS_STRINGIFIER = new DateStringifier(
-      "TIMESTAMP_MICROS_STRINGIFIER", "yyyy-MM-dd'T'HH:mm:ss.SSS") {
+      "TIMESTAMP_MICROS_STRINGIFIER", "yyyy-MM-dd'T'HH:mm:ss.SSSSSS") {
     @Override
-    public String stringify(long value) {
-      return super.stringify(value) + String.format("%03d", Math.abs(value % 1000));
-    }
-
-    @Override
-    long toMillis(long value) {
-      return value / 1000;
+    Instant getInstant(long value) {
+      return Instant.ofEpochSecond(MICROSECONDS.toSeconds(value), MICROSECONDS.toNanos(value
% SECONDS.toMicros(1)));
     }
   };
 
   static final PrimitiveStringifier TIMESTAMP_NANOS_STRINGIFIER = new DateStringifier(
-    "TIMESTAMP_NANOS_STRINGIFIER", "yyyy-MM-dd'T'HH:mm:ss.SSS") {
-    @Override
-    public String stringify(long value) {
-      return super.stringify(value) + String.format("%06d", Math.abs(value % 1_000_000));
-    }
-
+    "TIMESTAMP_NANOS_STRINGIFIER", "yyyy-MM-dd'T'HH:mm:ss.SSSSSSSSS") {
     @Override
-    long toMillis(long value) {
-      return value / 1_000_000;
+    Instant getInstant(long value) {
+      return Instant.ofEpochSecond(NANOSECONDS.toSeconds(value), NANOSECONDS.toNanos(value
% SECONDS.toNanos(1)));
     }
   };
 
diff --git a/parquet-column/src/test/java/org/apache/parquet/schema/TestPrimitiveStringifier.java
b/parquet-column/src/test/java/org/apache/parquet/schema/TestPrimitiveStringifier.java
index b4e7062..afc8684 100644
--- a/parquet-column/src/test/java/org/apache/parquet/schema/TestPrimitiveStringifier.java
+++ b/parquet-column/src/test/java/org/apache/parquet/schema/TestPrimitiveStringifier.java
@@ -154,8 +154,8 @@ public class TestPrimitiveStringifier {
     assertEquals("2017-12-14", stringifier.stringify((int) MILLISECONDS.toDays(cal.getTimeInMillis())));
 
     cal.clear();
-    cal.set(1492, Calendar.AUGUST, 3);
-    assertEquals("1492-08-03", stringifier.stringify((int) MILLISECONDS.toDays(cal.getTimeInMillis())));
+    cal.set(1583, Calendar.AUGUST, 3);
+    assertEquals("1583-08-03", stringifier.stringify((int) MILLISECONDS.toDays(cal.getTimeInMillis())));
 
     checkThrowingUnsupportedException(stringifier, Integer.TYPE);
   }
@@ -197,7 +197,7 @@ public class TestPrimitiveStringifier {
     cal.set(1848, Calendar.MARCH, 15, 9, 23, 59);
     cal.set(Calendar.MILLISECOND, 765);
     micros = cal.getTimeInMillis() * 1000 - 1;
-    assertEquals("1848-03-15T09:23:59.765001", stringifier.stringify(micros));
+    assertEquals("1848-03-15T09:23:59.764999", stringifier.stringify(micros));
 
     checkThrowingUnsupportedException(stringifier, Long.TYPE);
   }
@@ -219,7 +219,7 @@ public class TestPrimitiveStringifier {
     cal.set(1848, Calendar.MARCH, 15, 9, 23, 59);
     cal.set(Calendar.MILLISECOND, 765);
     nanos = cal.getTimeInMillis() * 1_000_000 - 1;
-    assertEquals("1848-03-15T09:23:59.765000001", stringifier.stringify(nanos));
+    assertEquals("1848-03-15T09:23:59.764999999", stringifier.stringify(nanos));
 
     checkThrowingUnsupportedException(stringifier, Long.TYPE);
   }


Mime
View raw message