This is an automated email from the ASF dual-hosted git repository.
zivanfi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/parquet-mr.git
The following commit(s) were added to refs/heads/master by this push:
new 63003ac PARQUET-1436: TimestampMicrosStringifier shows wrong microseconds for timestamps
before 1970 (#529)
63003ac is described below
commit 63003ac204369c077f8e7148799b22c2aa10d2d6
Author: nandorKollar <nandorKollar@users.noreply.github.com>
AuthorDate: Fri Oct 5 11:16:42 2018 +0200
PARQUET-1436: TimestampMicrosStringifier shows wrong microseconds for timestamps before
1970 (#529)
---
.../parquet/schema/PrimitiveStringifier.java | 57 +++++++++-------------
.../parquet/schema/TestPrimitiveStringifier.java | 8 +--
2 files changed, 27 insertions(+), 38 deletions(-)
diff --git a/parquet-column/src/main/java/org/apache/parquet/schema/PrimitiveStringifier.java
b/parquet-column/src/main/java/org/apache/parquet/schema/PrimitiveStringifier.java
index 3c3417e..03786ed 100644
--- a/parquet-column/src/main/java/org/apache/parquet/schema/PrimitiveStringifier.java
+++ b/parquet-column/src/main/java/org/apache/parquet/schema/PrimitiveStringifier.java
@@ -29,8 +29,9 @@ import java.math.BigDecimal;
import java.math.BigInteger;
import java.nio.ByteBuffer;
import java.nio.ByteOrder;
-import java.text.SimpleDateFormat;
-import java.util.TimeZone;
+import java.time.Instant;
+import java.time.ZoneOffset;
+import java.time.format.DateTimeFormatter;
import java.util.concurrent.TimeUnit;
import javax.naming.OperationNotSupportedException;
@@ -243,80 +244,68 @@ public abstract class PrimitiveStringifier {
};
private static class DateStringifier extends PrimitiveStringifier {
- private final SimpleDateFormat formatter;
- private static final TimeZone UTC = TimeZone.getTimeZone("utc");
+ private final DateTimeFormatter formatter;
private DateStringifier(String name, String format) {
super(name);
- formatter = new SimpleDateFormat(format);
- formatter.setTimeZone(UTC);
+ formatter = DateTimeFormatter.ofPattern(format).withZone(ZoneOffset.UTC);
}
@Override
public String stringify(int value) {
- return toFormattedString(toMillis(value));
+ return toFormattedString(getInstant(value));
}
@Override
public String stringify(long value) {
- return toFormattedString(toMillis(value));
+ return toFormattedString(getInstant(value));
}
- private String toFormattedString(long millis) {
- return formatter.format(millis);
+ private String toFormattedString(Instant instant) {
+ return formatter.format(instant);
}
- long toMillis(int value) {
+ Instant getInstant(int value) {
// throw the related unsupported exception
super.stringify(value);
- return 0;
+ return null;
}
- long toMillis(long value) {
+ Instant getInstant(long value) {
// throw the related unsupported exception
super.stringify(value);
- return 0;
+ return null;
}
}
static final PrimitiveStringifier DATE_STRINGIFIER = new DateStringifier("DATE_STRINGIFIER",
"yyyy-MM-dd") {
@Override
- long toMillis(int value) {
- return TimeUnit.DAYS.toMillis(value);
+ Instant getInstant(int value) {
+ return Instant.ofEpochMilli(TimeUnit.DAYS.toMillis(value));
};
};
static final PrimitiveStringifier TIMESTAMP_MILLIS_STRINGIFIER = new DateStringifier(
"TIMESTAMP_MILLIS_STRINGIFIER", "yyyy-MM-dd'T'HH:mm:ss.SSS") {
@Override
- long toMillis(long value) {
- return value;
+ Instant getInstant(long value) {
+ return Instant.ofEpochMilli(value);
}
};
static final PrimitiveStringifier TIMESTAMP_MICROS_STRINGIFIER = new DateStringifier(
- "TIMESTAMP_MICROS_STRINGIFIER", "yyyy-MM-dd'T'HH:mm:ss.SSS") {
+ "TIMESTAMP_MICROS_STRINGIFIER", "yyyy-MM-dd'T'HH:mm:ss.SSSSSS") {
@Override
- public String stringify(long value) {
- return super.stringify(value) + String.format("%03d", Math.abs(value % 1000));
- }
-
- @Override
- long toMillis(long value) {
- return value / 1000;
+ Instant getInstant(long value) {
+ return Instant.ofEpochSecond(MICROSECONDS.toSeconds(value), MICROSECONDS.toNanos(value
% SECONDS.toMicros(1)));
}
};
static final PrimitiveStringifier TIMESTAMP_NANOS_STRINGIFIER = new DateStringifier(
- "TIMESTAMP_NANOS_STRINGIFIER", "yyyy-MM-dd'T'HH:mm:ss.SSS") {
- @Override
- public String stringify(long value) {
- return super.stringify(value) + String.format("%06d", Math.abs(value % 1_000_000));
- }
-
+ "TIMESTAMP_NANOS_STRINGIFIER", "yyyy-MM-dd'T'HH:mm:ss.SSSSSSSSS") {
@Override
- long toMillis(long value) {
- return value / 1_000_000;
+ Instant getInstant(long value) {
+ return Instant.ofEpochSecond(NANOSECONDS.toSeconds(value), NANOSECONDS.toNanos(value
% SECONDS.toNanos(1)));
}
};
diff --git a/parquet-column/src/test/java/org/apache/parquet/schema/TestPrimitiveStringifier.java
b/parquet-column/src/test/java/org/apache/parquet/schema/TestPrimitiveStringifier.java
index b4e7062..afc8684 100644
--- a/parquet-column/src/test/java/org/apache/parquet/schema/TestPrimitiveStringifier.java
+++ b/parquet-column/src/test/java/org/apache/parquet/schema/TestPrimitiveStringifier.java
@@ -154,8 +154,8 @@ public class TestPrimitiveStringifier {
assertEquals("2017-12-14", stringifier.stringify((int) MILLISECONDS.toDays(cal.getTimeInMillis())));
cal.clear();
- cal.set(1492, Calendar.AUGUST, 3);
- assertEquals("1492-08-03", stringifier.stringify((int) MILLISECONDS.toDays(cal.getTimeInMillis())));
+ cal.set(1583, Calendar.AUGUST, 3);
+ assertEquals("1583-08-03", stringifier.stringify((int) MILLISECONDS.toDays(cal.getTimeInMillis())));
checkThrowingUnsupportedException(stringifier, Integer.TYPE);
}
@@ -197,7 +197,7 @@ public class TestPrimitiveStringifier {
cal.set(1848, Calendar.MARCH, 15, 9, 23, 59);
cal.set(Calendar.MILLISECOND, 765);
micros = cal.getTimeInMillis() * 1000 - 1;
- assertEquals("1848-03-15T09:23:59.765001", stringifier.stringify(micros));
+ assertEquals("1848-03-15T09:23:59.764999", stringifier.stringify(micros));
checkThrowingUnsupportedException(stringifier, Long.TYPE);
}
@@ -219,7 +219,7 @@ public class TestPrimitiveStringifier {
cal.set(1848, Calendar.MARCH, 15, 9, 23, 59);
cal.set(Calendar.MILLISECOND, 765);
nanos = cal.getTimeInMillis() * 1_000_000 - 1;
- assertEquals("1848-03-15T09:23:59.765000001", stringifier.stringify(nanos));
+ assertEquals("1848-03-15T09:23:59.764999999", stringifier.stringify(nanos));
checkThrowingUnsupportedException(stringifier, Long.TYPE);
}
|