arrow-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From w...@apache.org
Subject arrow git commit: ARROW-582: [Java] Added JSON reader/writer unit test for date, time, and timestamp
Date Thu, 06 Apr 2017 13:33:23 GMT
Repository: arrow
Updated Branches:
  refs/heads/master ddf880b31 -> 621d52740


ARROW-582: [Java] Added JSON reader/writer unit test for date, time, and timestamp

New unit test to verify Java JSON reader/writer round-trip with date, time and timestamp types

Author: Bryan Cutler <cutlerb@gmail.com>

Closes #495 from BryanCutler/java-json-DateTime-Test-ARROW-582 and squashes the following
commits:

e80683b [Bryan Cutler] added JSON read and write support unit test


Project: http://git-wip-us.apache.org/repos/asf/arrow/repo
Commit: http://git-wip-us.apache.org/repos/asf/arrow/commit/621d5274
Tree: http://git-wip-us.apache.org/repos/asf/arrow/tree/621d5274
Diff: http://git-wip-us.apache.org/repos/asf/arrow/diff/621d5274

Branch: refs/heads/master
Commit: 621d52740b52af4042c4aaa3ac424c5916aa94da
Parents: ddf880b
Author: Bryan Cutler <cutlerb@gmail.com>
Authored: Thu Apr 6 09:33:17 2017 -0400
Committer: Wes McKinney <wes.mckinney@twosigma.com>
Committed: Thu Apr 6 09:33:17 2017 -0400

----------------------------------------------------------------------
 .../arrow/vector/file/json/JsonFileReader.java  |  2 +-
 .../apache/arrow/vector/file/BaseFileTest.java  | 46 ++++++++++++++++++++
 .../arrow/vector/file/json/TestJSONFile.java    | 38 +++++++++++++++-
 3 files changed, 84 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/arrow/blob/621d5274/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileReader.java
----------------------------------------------------------------------
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileReader.java
b/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileReader.java
index 2f91205..fde9954 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileReader.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileReader.java
@@ -275,7 +275,7 @@ public class JsonFileReader implements AutoCloseable {
       break;
     case TIMESTAMPNANO:
       ((TimeStampNanoVector)valueVector).getMutator().set(i, parser.readValueAs(Long.class));
-    break;
+      break;
     default:
       throw new UnsupportedOperationException("minor type: " + valueVector.getMinorType());
     }

http://git-wip-us.apache.org/repos/asf/arrow/blob/621d5274/java/vector/src/test/java/org/apache/arrow/vector/file/BaseFileTest.java
----------------------------------------------------------------------
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/file/BaseFileTest.java b/java/vector/src/test/java/org/apache/arrow/vector/file/BaseFileTest.java
index 774bead..5c68a19 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/file/BaseFileTest.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/file/BaseFileTest.java
@@ -32,8 +32,12 @@ import org.apache.arrow.vector.complex.writer.BaseWriter.ComplexWriter;
 import org.apache.arrow.vector.complex.writer.BaseWriter.ListWriter;
 import org.apache.arrow.vector.complex.writer.BaseWriter.MapWriter;
 import org.apache.arrow.vector.complex.writer.BigIntWriter;
+import org.apache.arrow.vector.complex.writer.DateMilliWriter;
 import org.apache.arrow.vector.complex.writer.IntWriter;
+import org.apache.arrow.vector.complex.writer.TimeMilliWriter;
+import org.apache.arrow.vector.complex.writer.TimeStampMilliWriter;
 import org.apache.arrow.vector.holders.NullableTimeStampMilliHolder;
+import org.joda.time.DateTime;
 import org.joda.time.DateTimeZone;
 import org.junit.After;
 import org.junit.Assert;
@@ -138,6 +142,48 @@ public class BaseFileTest {
     }
   }
 
+  private DateTime makeDateTimeFromCount(int i) {
+    return new DateTime(2000 + i, 1 + i, 1 + i, i, i, i, i, DateTimeZone.UTC);
+  }
+
+  protected void writeDateTimeData(int count, NullableMapVector parent) {
+    Assert.assertTrue(count < 100);
+    ComplexWriter writer = new ComplexWriterImpl("root", parent);
+    MapWriter rootWriter = writer.rootAsMap();
+    DateMilliWriter dateWriter = rootWriter.dateMilli("date");
+    TimeMilliWriter timeWriter = rootWriter.timeMilli("time");
+    TimeStampMilliWriter timeStampMilliWriter = rootWriter.timeStampMilli("timestamp-milli");
+    for (int i = 0; i < count; i++) {
+      DateTime dt = makeDateTimeFromCount(i);
+      // Number of days in milliseconds since epoch, stored as 64-bit integer, only date
part is used
+      dateWriter.setPosition(i);
+      long dateLong = dt.minusMillis(dt.getMillisOfDay()).getMillis();
+      dateWriter.writeDateMilli(dateLong);
+      // Time is a value in milliseconds since midnight, stored as 32-bit integer
+      timeWriter.setPosition(i);
+      timeWriter.writeTimeMilli(dt.getMillisOfDay());
+      // Timestamp is milliseconds since the epoch, stored as 64-bit integer
+      timeStampMilliWriter.setPosition(i);
+      timeStampMilliWriter.writeTimeStampMilli(dt.getMillis());
+    }
+    writer.setValueCount(count);
+  }
+
+  protected void validateDateTimeContent(int count, VectorSchemaRoot root) {
+    Assert.assertEquals(count, root.getRowCount());
+    printVectors(root.getFieldVectors());
+    for (int i = 0; i < count; i++) {
+      Object dateVal = root.getVector("date").getAccessor().getObject(i);
+      DateTime dt = makeDateTimeFromCount(i);
+      DateTime dateExpected = dt.minusMillis(dt.getMillisOfDay());
+      Assert.assertEquals(dateExpected.getMillis(), dateVal);
+      Object timeVal = root.getVector("time").getAccessor().getObject(i);
+      Assert.assertEquals(dt.getMillisOfDay(), timeVal);
+      Object timestampMilliVal = root.getVector("timestamp-milli").getAccessor().getObject(i);
+      Assert.assertTrue(dt.withZoneRetainFields(DateTimeZone.getDefault()).equals(timestampMilliVal));
+    }
+  }
+
   protected void writeData(int count, MapVector parent) {
     ComplexWriter writer = new ComplexWriterImpl("root", parent);
     MapWriter rootWriter = writer.rootAsMap();

http://git-wip-us.apache.org/repos/asf/arrow/blob/621d5274/java/vector/src/test/java/org/apache/arrow/vector/file/json/TestJSONFile.java
----------------------------------------------------------------------
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/file/json/TestJSONFile.java
b/java/vector/src/test/java/org/apache/arrow/vector/file/json/TestJSONFile.java
index c88958c..6369c07 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/file/json/TestJSONFile.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/file/json/TestJSONFile.java
@@ -103,7 +103,7 @@ public class TestJSONFile extends BaseFileTest {
 
       writeJSON(file, root);
     }
- // read
+    // read
     try (
         BufferAllocator readerAllocator = allocator.newChildAllocator("reader", 0, Integer.MAX_VALUE);
         BufferAllocator vectorAllocator = allocator.newChildAllocator("final vectors", 0,
Integer.MAX_VALUE);
@@ -120,6 +120,42 @@ public class TestJSONFile extends BaseFileTest {
   }
 
   @Test
+  public void testWriteReadDateTimeJSON() throws IOException {
+    File file = new File("target/mytest_datetime.json");
+    int count = COUNT;
+
+    // write
+    try (
+        BufferAllocator vectorAllocator = allocator.newChildAllocator("original vectors",
0, Integer.MAX_VALUE);
+        NullableMapVector parent = new NullableMapVector("parent", vectorAllocator, null,
null)) {
+
+      writeDateTimeData(count, parent);
+
+      printVectors(parent.getChildrenFromFields());
+
+      VectorSchemaRoot root = new VectorSchemaRoot(parent.getChild("root"));
+      validateDateTimeContent(count, root);
+
+      writeJSON(file, new VectorSchemaRoot(parent.getChild("root")));
+    }
+
+    // read
+    try (
+        BufferAllocator readerAllocator = allocator.newChildAllocator("reader", 0, Integer.MAX_VALUE);
+    ) {
+      JsonFileReader reader = new JsonFileReader(file, readerAllocator);
+      Schema schema = reader.start();
+      LOGGER.debug("reading schema: " + schema);
+
+      // initialize vectors
+      try (VectorSchemaRoot root = reader.read();) {
+        validateDateTimeContent(count, root);
+      }
+      reader.close();
+    }
+  }
+
+  @Test
   public void testSetStructLength() throws IOException {
     File file = new File("../../integration/data/struct_example.json");
     try (


Mime
View raw message