arrow-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jul...@apache.org
Subject arrow git commit: ARROW-674: [Java] Support additional Timestamp timezone metadata
Date Thu, 23 Mar 2017 03:16:00 GMT
Repository: arrow
Updated Branches:
  refs/heads/master f67974b19 -> e8f6a492d


ARROW-674: [Java] Support additional Timestamp timezone metadata

Author: Julien Le Dem <julien@dremio.com>

Closes #408 from julienledem/timestamp_md and squashes the following commits:

e394526 [Julien Le Dem] ARROW-674: [Java] Support additional Timestamp timezone metadata


Project: http://git-wip-us.apache.org/repos/asf/arrow/repo
Commit: http://git-wip-us.apache.org/repos/asf/arrow/commit/e8f6a492
Tree: http://git-wip-us.apache.org/repos/asf/arrow/tree/e8f6a492
Diff: http://git-wip-us.apache.org/repos/asf/arrow/diff/e8f6a492

Branch: refs/heads/master
Commit: e8f6a492d30d32cd67fe3a537b3aec4cbae566c9
Parents: f67974b
Author: Julien Le Dem <julien@dremio.com>
Authored: Wed Mar 22 20:15:55 2017 -0700
Committer: Julien Le Dem <julien@dremio.com>
Committed: Wed Mar 22 20:15:55 2017 -0700

----------------------------------------------------------------------
 .../vector/src/main/codegen/data/ArrowTypes.tdd |  2 +-
 .../org/apache/arrow/vector/types/Types.java    | 16 ++--
 .../apache/arrow/vector/pojo/TestConvert.java   |  2 +-
 .../arrow/vector/types/pojo/TestSchema.java     | 90 ++++++++++++--------
 4 files changed, 66 insertions(+), 44 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/arrow/blob/e8f6a492/java/vector/src/main/codegen/data/ArrowTypes.tdd
----------------------------------------------------------------------
diff --git a/java/vector/src/main/codegen/data/ArrowTypes.tdd b/java/vector/src/main/codegen/data/ArrowTypes.tdd
index 8f99752..94fe31e 100644
--- a/java/vector/src/main/codegen/data/ArrowTypes.tdd
+++ b/java/vector/src/main/codegen/data/ArrowTypes.tdd
@@ -62,7 +62,7 @@
     },
     {
       name: "Timestamp",
-      fields: [{name: "unit", type: short, valueType: TimeUnit}]
+      fields: [{name: "unit", type: short, valueType: TimeUnit}, {name: "timezone", type:
String}]
     },
     {
       name: "Interval",

http://git-wip-us.apache.org/repos/asf/arrow/blob/e8f6a492/java/vector/src/main/java/org/apache/arrow/vector/types/Types.java
----------------------------------------------------------------------
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/types/Types.java b/java/vector/src/main/java/org/apache/arrow/vector/types/Types.java
index 7cbf3c5..81743b5 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/types/Types.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/types/Types.java
@@ -109,10 +109,10 @@ public class Types {
   private static final Field UINT8_FIELD = new Field("", true, new Int(64, false), null);
   private static final Field DATE_FIELD = new Field("", true, Date.INSTANCE, null);
   private static final Field TIME_FIELD = new Field("", true, new Time(TimeUnit.MILLISECOND,
32), null);
-  private static final Field TIMESTAMPSEC_FIELD = new Field("", true, new Timestamp(TimeUnit.SECOND),
null);
-  private static final Field TIMESTAMPMILLI_FIELD = new Field("", true, new Timestamp(TimeUnit.MILLISECOND),
null);
-  private static final Field TIMESTAMPMICRO_FIELD = new Field("", true, new Timestamp(TimeUnit.MICROSECOND),
null);
-  private static final Field TIMESTAMPNANO_FIELD = new Field("", true, new Timestamp(TimeUnit.NANOSECOND),
null);
+  private static final Field TIMESTAMPSEC_FIELD = new Field("", true, new Timestamp(TimeUnit.SECOND,
"UTC"), null);
+  private static final Field TIMESTAMPMILLI_FIELD = new Field("", true, new Timestamp(TimeUnit.MILLISECOND,
"UTC"), null);
+  private static final Field TIMESTAMPMICRO_FIELD = new Field("", true, new Timestamp(TimeUnit.MICROSECOND,
"UTC"), null);
+  private static final Field TIMESTAMPNANO_FIELD = new Field("", true, new Timestamp(TimeUnit.NANOSECOND,
"UTC"), null);
   private static final Field INTERVALDAY_FIELD = new Field("", true, new Interval(IntervalUnit.DAY_TIME),
null);
   private static final Field INTERVALYEAR_FIELD = new Field("", true, new Interval(IntervalUnit.YEAR_MONTH),
null);
   private static final Field FLOAT4_FIELD = new Field("", true, new FloatingPoint(FloatingPointPrecision.SINGLE),
null);
@@ -252,7 +252,7 @@ public class Types {
       }
     },
     // time in second from the Unix epoch, 00:00:00.000000 on 1 January 1970, UTC.
-    TIMESTAMPSEC(new Timestamp(org.apache.arrow.vector.types.TimeUnit.SECOND)) {
+    TIMESTAMPSEC(new Timestamp(org.apache.arrow.vector.types.TimeUnit.SECOND, "UTC")) {
       @Override
       public Field getField() {
         return TIMESTAMPSEC_FIELD;
@@ -269,7 +269,7 @@ public class Types {
       }
     },
     // time in millis from the Unix epoch, 00:00:00.000 on 1 January 1970, UTC.
-    TIMESTAMPMILLI(new Timestamp(org.apache.arrow.vector.types.TimeUnit.MILLISECOND)) {
+    TIMESTAMPMILLI(new Timestamp(org.apache.arrow.vector.types.TimeUnit.MILLISECOND, "UTC"))
{
       @Override
       public Field getField() {
         return TIMESTAMPMILLI_FIELD;
@@ -286,7 +286,7 @@ public class Types {
       }
     },
     // time in microsecond from the Unix epoch, 00:00:00.000000 on 1 January 1970, UTC.
-    TIMESTAMPMICRO(new Timestamp(org.apache.arrow.vector.types.TimeUnit.MICROSECOND)) {
+    TIMESTAMPMICRO(new Timestamp(org.apache.arrow.vector.types.TimeUnit.MICROSECOND, "UTC"))
{
       @Override
       public Field getField() {
         return TIMESTAMPMICRO_FIELD;
@@ -303,7 +303,7 @@ public class Types {
       }
     },
     // time in nanosecond from the Unix epoch, 00:00:00.000000000 on 1 January 1970, UTC.
-    TIMESTAMPNANO(new Timestamp(org.apache.arrow.vector.types.TimeUnit.NANOSECOND)) {
+    TIMESTAMPNANO(new Timestamp(org.apache.arrow.vector.types.TimeUnit.NANOSECOND, "UTC"))
{
       @Override
       public Field getField() {
         return TIMESTAMPNANO_FIELD;

http://git-wip-us.apache.org/repos/asf/arrow/blob/e8f6a492/java/vector/src/test/java/org/apache/arrow/vector/pojo/TestConvert.java
----------------------------------------------------------------------
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/pojo/TestConvert.java b/java/vector/src/test/java/org/apache/arrow/vector/pojo/TestConvert.java
index 65823e2..824c62a 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/pojo/TestConvert.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/pojo/TestConvert.java
@@ -81,7 +81,7 @@ public class TestConvert {
         new Field("child4.1", true, Utf8.INSTANCE, null)
         )));
     childrenBuilder.add(new Field("child5", true, new Union(UnionMode.Sparse, new int[] {
MinorType.TIMESTAMPMILLI.ordinal(), MinorType.FLOAT8.ordinal() } ), ImmutableList.<Field>of(
-        new Field("child5.1", true, new Timestamp(TimeUnit.MILLISECOND), null),
+        new Field("child5.1", true, new Timestamp(TimeUnit.MILLISECOND, "UTC"), null),
         new Field("child5.2", true, new FloatingPoint(DOUBLE), ImmutableList.<Field>of())
         )));
     Schema initialSchema = new Schema(childrenBuilder.build());

http://git-wip-us.apache.org/repos/asf/arrow/blob/e8f6a492/java/vector/src/test/java/org/apache/arrow/vector/types/pojo/TestSchema.java
----------------------------------------------------------------------
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/types/pojo/TestSchema.java
b/java/vector/src/test/java/org/apache/arrow/vector/types/pojo/TestSchema.java
index a7d1cce..9f1b2e0 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/types/pojo/TestSchema.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/types/pojo/TestSchema.java
@@ -28,6 +28,20 @@ import org.apache.arrow.vector.types.FloatingPointPrecision;
 import org.apache.arrow.vector.types.IntervalUnit;
 import org.apache.arrow.vector.types.TimeUnit;
 import org.apache.arrow.vector.types.UnionMode;
+import org.apache.arrow.vector.types.pojo.ArrowType.Binary;
+import org.apache.arrow.vector.types.pojo.ArrowType.Bool;
+import org.apache.arrow.vector.types.pojo.ArrowType.Date;
+import org.apache.arrow.vector.types.pojo.ArrowType.Decimal;
+import org.apache.arrow.vector.types.pojo.ArrowType.FloatingPoint;
+import org.apache.arrow.vector.types.pojo.ArrowType.Int;
+import org.apache.arrow.vector.types.pojo.ArrowType.Interval;
+import org.apache.arrow.vector.types.pojo.ArrowType.List;
+import org.apache.arrow.vector.types.pojo.ArrowType.Null;
+import org.apache.arrow.vector.types.pojo.ArrowType.Struct;
+import org.apache.arrow.vector.types.pojo.ArrowType.Time;
+import org.apache.arrow.vector.types.pojo.ArrowType.Timestamp;
+import org.apache.arrow.vector.types.pojo.ArrowType.Union;
+import org.apache.arrow.vector.types.pojo.ArrowType.Utf8;
 import org.junit.Test;
 
 public class TestSchema {
@@ -43,38 +57,40 @@ public class TestSchema {
   @Test
   public void testComplex() throws IOException {
     Schema schema = new Schema(asList(
-        field("a", false, new ArrowType.Int(8, true)),
-        field("b", new ArrowType.Struct(),
-            field("c", new ArrowType.Int(16, true)),
-            field("d", new ArrowType.Utf8())),
-        field("e", new ArrowType.List(), field(null, new ArrowType.Date())),
-        field("f", new ArrowType.FloatingPoint(FloatingPointPrecision.SINGLE)),
-        field("g", new ArrowType.Timestamp(TimeUnit.MILLISECOND)),
-        field("h", new ArrowType.Interval(IntervalUnit.DAY_TIME))
+        field("a", false, new Int(8, true)),
+        field("b", new Struct(),
+            field("c", new Int(16, true)),
+            field("d", new Utf8())),
+        field("e", new List(), field(null, new Date())),
+        field("f", new FloatingPoint(FloatingPointPrecision.SINGLE)),
+        field("g", new Timestamp(TimeUnit.MILLISECOND, "UTC")),
+        field("h", new Timestamp(TimeUnit.MICROSECOND, null)),
+        field("i", new Interval(IntervalUnit.DAY_TIME))
         ));
     roundTrip(schema);
     assertEquals(
-        "Schema<a: Int(8, true) not null, b: Struct<c: Int(16, true), d: Utf8>,
e: List<Date>, f: FloatingPoint(SINGLE), g: Timestamp(MILLISECOND), h: Interval(DAY_TIME)>",
+        "Schema<a: Int(8, true) not null, b: Struct<c: Int(16, true), d: Utf8>,
e: List<Date>, f: FloatingPoint(SINGLE), g: Timestamp(MILLISECOND, UTC), h: Timestamp(MICROSECOND,
null), i: Interval(DAY_TIME)>",
         schema.toString());
   }
 
   @Test
   public void testAll() throws IOException {
     Schema schema = new Schema(asList(
-        field("a", false, new ArrowType.Null()),
-        field("b", new ArrowType.Struct(), field("ba", new ArrowType.Null())),
-        field("c", new ArrowType.List(), field("ca", new ArrowType.Null())),
-        field("d", new ArrowType.Union(UnionMode.Sparse, new int[] {1, 2, 3}), field("da",
new ArrowType.Null())),
-        field("e", new ArrowType.Int(8, true)),
-        field("f", new ArrowType.FloatingPoint(FloatingPointPrecision.SINGLE)),
-        field("g", new ArrowType.Utf8()),
-        field("h", new ArrowType.Binary()),
-        field("i", new ArrowType.Bool()),
-        field("j", new ArrowType.Decimal(5, 5)),
-        field("k", new ArrowType.Date()),
-        field("l", new ArrowType.Time(TimeUnit.MILLISECOND, 32)),
-        field("m", new ArrowType.Timestamp(TimeUnit.MILLISECOND)),
-        field("n", new ArrowType.Interval(IntervalUnit.DAY_TIME))
+        field("a", false, new Null()),
+        field("b", new Struct(), field("ba", new Null())),
+        field("c", new List(), field("ca", new Null())),
+        field("d", new Union(UnionMode.Sparse, new int[] {1, 2, 3}), field("da", new Null())),
+        field("e", new Int(8, true)),
+        field("f", new FloatingPoint(FloatingPointPrecision.SINGLE)),
+        field("g", new Utf8()),
+        field("h", new Binary()),
+        field("i", new Bool()),
+        field("j", new Decimal(5, 5)),
+        field("k", new Date()),
+        field("l", new Time(TimeUnit.MILLISECOND, 32)),
+        field("m", new Timestamp(TimeUnit.MILLISECOND, "UTC")),
+        field("n", new Timestamp(TimeUnit.MICROSECOND, null)),
+        field("o", new Interval(IntervalUnit.DAY_TIME))
         ));
     roundTrip(schema);
   }
@@ -82,7 +98,7 @@ public class TestSchema {
   @Test
   public void testUnion() throws IOException {
     Schema schema = new Schema(asList(
-        field("d", new ArrowType.Union(UnionMode.Sparse, new int[] {1, 2, 3}), field("da",
new ArrowType.Null()))
+        field("d", new Union(UnionMode.Sparse, new int[] {1, 2, 3}), field("da", new Null()))
         ));
     roundTrip(schema);
     contains(schema, "Sparse");
@@ -91,20 +107,26 @@ public class TestSchema {
   @Test
   public void testTS() throws IOException {
     Schema schema = new Schema(asList(
-        field("a", new ArrowType.Timestamp(TimeUnit.SECOND)),
-        field("b", new ArrowType.Timestamp(TimeUnit.MILLISECOND)),
-        field("c", new ArrowType.Timestamp(TimeUnit.MICROSECOND)),
-        field("d", new ArrowType.Timestamp(TimeUnit.NANOSECOND))
+        field("a", new Timestamp(TimeUnit.SECOND, "UTC")),
+        field("b", new Timestamp(TimeUnit.MILLISECOND, "UTC")),
+        field("c", new Timestamp(TimeUnit.MICROSECOND, "UTC")),
+        field("d", new Timestamp(TimeUnit.NANOSECOND, "UTC")),
+        field("e", new Timestamp(TimeUnit.SECOND, null)),
+        field("f", new Timestamp(TimeUnit.MILLISECOND, null)),
+        field("g", new Timestamp(TimeUnit.MICROSECOND, null)),
+        field("h", new Timestamp(TimeUnit.NANOSECOND, null))
         ));
     roundTrip(schema);
-    contains(schema, "SECOND", "MILLISECOND", "MICROSECOND", "NANOSECOND");
+    assertEquals(
+        "Schema<a: Timestamp(SECOND, UTC), b: Timestamp(MILLISECOND, UTC), c: Timestamp(MICROSECOND,
UTC), d: Timestamp(NANOSECOND, UTC), e: Timestamp(SECOND, null), f: Timestamp(MILLISECOND,
null), g: Timestamp(MICROSECOND, null), h: Timestamp(NANOSECOND, null)>",
+        schema.toString());
   }
 
   @Test
   public void testInterval() throws IOException {
     Schema schema = new Schema(asList(
-        field("a", new ArrowType.Interval(IntervalUnit.YEAR_MONTH)),
-        field("b", new ArrowType.Interval(IntervalUnit.DAY_TIME))
+        field("a", new Interval(IntervalUnit.YEAR_MONTH)),
+        field("b", new Interval(IntervalUnit.DAY_TIME))
         ));
     roundTrip(schema);
     contains(schema, "YEAR_MONTH", "DAY_TIME");
@@ -113,9 +135,9 @@ public class TestSchema {
   @Test
   public void testFP() throws IOException {
     Schema schema = new Schema(asList(
-        field("a", new ArrowType.FloatingPoint(FloatingPointPrecision.HALF)),
-        field("b", new ArrowType.FloatingPoint(FloatingPointPrecision.SINGLE)),
-        field("c", new ArrowType.FloatingPoint(FloatingPointPrecision.DOUBLE))
+        field("a", new FloatingPoint(FloatingPointPrecision.HALF)),
+        field("b", new FloatingPoint(FloatingPointPrecision.SINGLE)),
+        field("c", new FloatingPoint(FloatingPointPrecision.DOUBLE))
         ));
     roundTrip(schema);
     contains(schema, "HALF", "SINGLE", "DOUBLE");


Mime
View raw message