arrow-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From w...@apache.org
Subject arrow git commit: ARROW-1497: [Java] Fix JsonReader to initialize count correctly
Date Wed, 20 Sep 2017 17:40:16 GMT
Repository: arrow
Updated Branches:
  refs/heads/master 9997a1a3b -> 975f32b1e


ARROW-1497: [Java] Fix JsonReader to initialize count correctly

Author: Li Jin <ice.xelloss@gmail.com>

Closes #1067 from icexelloss/json-reader-ARROW-1497 and squashes the following commits:

6d4e1dfb [Li Jin] Fix JsonReader to read union vectors correctly


Project: http://git-wip-us.apache.org/repos/asf/arrow/repo
Commit: http://git-wip-us.apache.org/repos/asf/arrow/commit/975f32b1
Tree: http://git-wip-us.apache.org/repos/asf/arrow/tree/975f32b1
Diff: http://git-wip-us.apache.org/repos/asf/arrow/diff/975f32b1

Branch: refs/heads/master
Commit: 975f32b1eff2aac48eb439938ae16c61ba44b560
Parents: 9997a1a
Author: Li Jin <ice.xelloss@gmail.com>
Authored: Wed Sep 20 13:40:10 2017 -0400
Committer: Wes McKinney <wes.mckinney@twosigma.com>
Committed: Wed Sep 20 13:40:10 2017 -0400

----------------------------------------------------------------------
 .../arrow/vector/file/json/JsonFileReader.java  | 11 ++++---
 .../arrow/vector/file/json/TestJSONFile.java    | 32 +++++++++-----------
 2 files changed, 22 insertions(+), 21 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/arrow/blob/975f32b1/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileReader.java
----------------------------------------------------------------------
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileReader.java
b/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileReader.java
index 71685d1..8bb0f26 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileReader.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileReader.java
@@ -67,7 +67,6 @@ import org.apache.arrow.vector.ValueVector.Mutator;
 import org.apache.arrow.vector.VarBinaryVector;
 import org.apache.arrow.vector.VarCharVector;
 import org.apache.arrow.vector.VectorSchemaRoot;
-import org.apache.arrow.vector.complex.NullableMapVector;
 import org.apache.arrow.vector.dictionary.Dictionary;
 import org.apache.arrow.vector.dictionary.DictionaryProvider;
 import org.apache.arrow.vector.schema.ArrowVectorType;
@@ -217,6 +216,11 @@ public class JsonFileReader implements AutoCloseable, DictionaryProvider
{
     }
   }
 
+  /*
+   * TODO: This method doesn't load some vectors correctly. For instance, it doesn't initialize
+   * `lastSet` in ListVector, VarCharVector, NullableVarBinaryVector A better way of implementing
+   * this function is to use `loadFieldBuffers` methods in FieldVector.
+   */
   private void readVector(Field field, FieldVector vector) throws JsonParseException, IOException
{
     List<ArrowVectorType> vectorTypes = field.getTypeLayout().getVectorTypes();
     List<BufferBacked> fieldInnerVectors = vector.getFieldInnerVectors();
@@ -231,6 +235,8 @@ public class JsonFileReader implements AutoCloseable, DictionaryProvider
{
         throw new IllegalArgumentException("Expected field " + field.getName() + " but got
" + name);
       }
       int count = readNextField("count", Integer.class);
+      vector.allocateNew();
+      vector.getMutator().setValueCount(count);
       for (int v = 0; v < vectorTypes.size(); v++) {
         ArrowVectorType vectorType = vectorTypes.get(v);
         BufferBacked innerVector = fieldInnerVectors.get(v);
@@ -266,9 +272,6 @@ public class JsonFileReader implements AutoCloseable, DictionaryProvider
{
         }
         readToken(END_ARRAY);
       }
-      if (vector instanceof NullableMapVector) {
-        ((NullableMapVector) vector).valueCount = count;
-      }
     }
     readToken(END_OBJECT);
   }

http://git-wip-us.apache.org/repos/asf/arrow/blob/975f32b1/java/vector/src/test/java/org/apache/arrow/vector/file/json/TestJSONFile.java
----------------------------------------------------------------------
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/file/json/TestJSONFile.java
b/java/vector/src/test/java/org/apache/arrow/vector/file/json/TestJSONFile.java
index b7c0632..960567f 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/file/json/TestJSONFile.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/file/json/TestJSONFile.java
@@ -30,6 +30,7 @@ import org.apache.arrow.vector.dictionary.DictionaryProvider;
 import org.apache.arrow.vector.dictionary.DictionaryProvider.MapDictionaryProvider;
 import org.apache.arrow.vector.file.BaseFileTest;
 import org.apache.arrow.vector.types.pojo.Schema;
+import org.apache.arrow.vector.util.Validator;
 import org.junit.Assert;
 import org.junit.Test;
 import org.slf4j.Logger;
@@ -96,28 +97,25 @@ public class TestJSONFile extends BaseFileTest {
     try (
         BufferAllocator vectorAllocator = allocator.newChildAllocator("original vectors",
0, Integer.MAX_VALUE);
         NullableMapVector parent = NullableMapVector.empty("parent", vectorAllocator)) {
-
       writeUnionData(count, parent);
-
       printVectors(parent.getChildrenFromFields());
 
-      VectorSchemaRoot root = new VectorSchemaRoot(parent.getChild("root"));
-      validateUnionData(count, root);
+      try (VectorSchemaRoot root = new VectorSchemaRoot(parent.getChild("root"))) {
+        validateUnionData(count, root);
+        writeJSON(file, root, null);
 
-      writeJSON(file, root, null);
-    }
-    // read
-    try (
-        BufferAllocator readerAllocator = allocator.newChildAllocator("reader", 0, Integer.MAX_VALUE);
-        BufferAllocator vectorAllocator = allocator.newChildAllocator("final vectors", 0,
Integer.MAX_VALUE);
-    ) {
-      JsonFileReader reader = new JsonFileReader(file, readerAllocator);
-      Schema schema = reader.start();
-      LOGGER.debug("reading schema: " + schema);
+        // read
+        try (BufferAllocator readerAllocator = allocator.newChildAllocator("reader", 0, Integer.MAX_VALUE))
{
+          JsonFileReader reader = new JsonFileReader(file, readerAllocator);
 
-      // initialize vectors
-      try (VectorSchemaRoot root = reader.read();) {
-        validateUnionData(count, root);
+          Schema schema = reader.start();
+          LOGGER.debug("reading schema: " + schema);
+
+          try (VectorSchemaRoot rootFromJson = reader.read();) {
+            validateUnionData(count, rootFromJson);
+            Validator.compareVectorSchemaRoot(root, rootFromJson);
+          }
+        }
       }
     }
   }


Mime
View raw message