arrow-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From w...@apache.org
Subject arrow git commit: ARROW-400: set struct length on load
Date Fri, 09 Dec 2016 17:04:31 GMT
Repository: arrow
Updated Branches:
  refs/heads/master d06c49144 -> 14ed1be2d


ARROW-400: set struct length on load

Adds unit test, closes #233

Author: Julien Le Dem <julien@dremio.com>
Author: Wes McKinney <wes.mckinney@twosigma.com>

Closes #234 from wesm/ARROW-400 and squashes the following commits:

f516ba1 [Wes McKinney] Add unit test for ARROW-400
741ff71 [Julien Le Dem] ARROW-400: set struct length on json load


Project: http://git-wip-us.apache.org/repos/asf/arrow/repo
Commit: http://git-wip-us.apache.org/repos/asf/arrow/commit/14ed1be2
Tree: http://git-wip-us.apache.org/repos/asf/arrow/tree/14ed1be2
Diff: http://git-wip-us.apache.org/repos/asf/arrow/diff/14ed1be2

Branch: refs/heads/master
Commit: 14ed1be2d89fedc31f4015456cda28216f926dcc
Parents: d06c491
Author: Julien Le Dem <julien@dremio.com>
Authored: Fri Dec 9 12:04:25 2016 -0500
Committer: Wes McKinney <wes.mckinney@twosigma.com>
Committed: Fri Dec 9 12:04:25 2016 -0500

----------------------------------------------------------------------
 integration/data/struct_example.json            | 237 +++++++++++++++++++
 .../apache/arrow/vector/complex/MapVector.java  |   2 +-
 .../arrow/vector/file/json/JsonFileReader.java  |   4 +
 .../arrow/vector/file/json/TestJSONFile.java    |  20 ++
 4 files changed, 262 insertions(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/arrow/blob/14ed1be2/integration/data/struct_example.json
----------------------------------------------------------------------
diff --git a/integration/data/struct_example.json b/integration/data/struct_example.json
new file mode 100644
index 0000000..3ea062d
--- /dev/null
+++ b/integration/data/struct_example.json
@@ -0,0 +1,237 @@
+{
+  "schema": {
+    "fields": [
+      {
+        "name": "struct_nullable",
+        "type": {
+          "name": "struct"
+        },
+        "nullable": true,
+        "children": [
+          {
+            "name": "f1",
+            "type": {
+              "name": "int",
+              "isSigned": true,
+              "bitWidth": 32
+            },
+            "nullable": true,
+            "children": [],
+            "typeLayout": {
+              "vectors": [
+                {
+                  "type": "VALIDITY",
+                  "typeBitWidth": 1
+                },
+                {
+                  "type": "DATA",
+                  "typeBitWidth": 32
+                }
+              ]
+            }
+          },
+          {
+            "name": "f2",
+            "type": {
+              "name": "utf8"
+            },
+            "nullable": true,
+            "children": [],
+            "typeLayout": {
+              "vectors": [
+                {
+                  "type": "VALIDITY",
+                  "typeBitWidth": 1
+                },
+                {
+                  "type": "OFFSET",
+                  "typeBitWidth": 32
+                },
+                {
+                  "type": "DATA",
+                  "typeBitWidth": 8
+                }
+              ]
+            }
+          }
+        ],
+        "typeLayout": {
+          "vectors": [
+            {
+              "type": "VALIDITY",
+              "typeBitWidth": 1
+            }
+          ]
+        }
+      }
+    ]
+  },
+  "batches": [
+    {
+      "count": 7,
+      "columns": [
+        {
+          "name": "struct_nullable",
+          "count": 7,
+          "VALIDITY": [
+            0,
+            1,
+            1,
+            1,
+            0,
+            1,
+            0
+          ],
+          "children": [
+            {
+              "name": "f1",
+              "count": 7,
+              "VALIDITY": [
+                1,
+                0,
+                1,
+                1,
+                1,
+                0,
+                0
+              ],
+              "DATA": [
+                1402032511,
+                290876774,
+                137773603,
+                410361374,
+                1959836418,
+                1995074679,
+                -163525262
+              ]
+            },
+            {
+              "name": "f2",
+              "count": 7,
+              "VALIDITY": [
+                0,
+                1,
+                1,
+                1,
+                0,
+                1,
+                0
+              ],
+              "OFFSET": [
+                0,
+                0,
+                7,
+                14,
+                21,
+                21,
+                28,
+                28
+              ],
+              "DATA": [
+                "",
+                "MhRNxD4",
+                "3F9HBxK",
+                "aVd88fp",
+                "",
+                "3loZrRf",
+                ""
+              ]
+            }
+          ]
+        }
+      ]
+    },
+    {
+      "count": 10,
+      "columns": [
+        {
+          "name": "struct_nullable",
+          "count": 10,
+          "VALIDITY": [
+            0,
+            1,
+            1,
+            0,
+            1,
+            0,
+            0,
+            1,
+            1,
+            1
+          ],
+          "children": [
+            {
+              "name": "f1",
+              "count": 10,
+              "VALIDITY": [
+                0,
+                0,
+                0,
+                0,
+                0,
+                0,
+                1,
+                0,
+                0,
+                0
+              ],
+              "DATA": [
+                -2041500147,
+                1715692943,
+                -35444996,
+                1425496657,
+                112765084,
+                1760754983,
+                413888857,
+                2039738337,
+                -1924327700,
+                670528518
+              ]
+            },
+            {
+              "name": "f2",
+              "count": 10,
+              "VALIDITY": [
+                1,
+                0,
+                0,
+                1,
+                1,
+                1,
+                1,
+                1,
+                1,
+                0
+              ],
+              "OFFSET": [
+                0,
+                7,
+                7,
+                7,
+                14,
+                21,
+                28,
+                35,
+                42,
+                49,
+                49
+              ],
+              "DATA": [
+                "AS5oARE",
+                "",
+                "",
+                "JGdagcX",
+                "78SLiRw",
+                "vbGf7OY",
+                "5uh5fTs",
+                "0ilsf82",
+                "LjS9MbU",
+                ""
+              ]
+            }
+          ]
+        }
+      ]
+    }
+  ]
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/arrow/blob/14ed1be2/java/vector/src/main/java/org/apache/arrow/vector/complex/MapVector.java
----------------------------------------------------------------------
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/MapVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/MapVector.java
index c2f216b..31a1bb7 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/MapVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/MapVector.java
@@ -50,7 +50,7 @@ public class MapVector extends AbstractMapVector {
   private final SingleMapReaderImpl reader = new SingleMapReaderImpl(this);
   private final Accessor accessor = new Accessor();
   private final Mutator mutator = new Mutator();
-  int valueCount;
+  public int valueCount;
 
   public MapVector(String name, BufferAllocator allocator, CallBack callBack) {
     super(name, allocator, callBack);

http://git-wip-us.apache.org/repos/asf/arrow/blob/14ed1be2/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileReader.java
----------------------------------------------------------------------
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileReader.java
b/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileReader.java
index 26dd3f6..152867c 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileReader.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileReader.java
@@ -47,6 +47,7 @@ import org.apache.arrow.vector.ValueVector;
 import org.apache.arrow.vector.ValueVector.Mutator;
 import org.apache.arrow.vector.VarCharVector;
 import org.apache.arrow.vector.VectorSchemaRoot;
+import org.apache.arrow.vector.complex.NullableMapVector;
 import org.apache.arrow.vector.schema.ArrowVectorType;
 import org.apache.arrow.vector.types.pojo.Field;
 import org.apache.arrow.vector.types.pojo.Schema;
@@ -153,6 +154,9 @@ public class JsonFileReader implements AutoCloseable {
         }
         readToken(END_ARRAY);
       }
+      if (vector instanceof NullableMapVector) {
+        ((NullableMapVector)vector).valueCount = count;
+      }
     }
     readToken(END_OBJECT);
   }

http://git-wip-us.apache.org/repos/asf/arrow/blob/14ed1be2/java/vector/src/test/java/org/apache/arrow/vector/file/json/TestJSONFile.java
----------------------------------------------------------------------
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/file/json/TestJSONFile.java
b/java/vector/src/test/java/org/apache/arrow/vector/file/json/TestJSONFile.java
index 7d25003..3720a13 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/file/json/TestJSONFile.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/file/json/TestJSONFile.java
@@ -21,11 +21,13 @@ import java.io.File;
 import java.io.IOException;
 
 import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.vector.FieldVector;
 import org.apache.arrow.vector.VectorSchemaRoot;
 import org.apache.arrow.vector.complex.MapVector;
 import org.apache.arrow.vector.complex.NullableMapVector;
 import org.apache.arrow.vector.file.BaseFileTest;
 import org.apache.arrow.vector.types.pojo.Schema;
+import org.junit.Assert;
 import org.junit.Test;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -117,4 +119,22 @@ public class TestJSONFile extends BaseFileTest {
     }
   }
 
+  @Test
+  public void testSetStructLength() throws IOException {
+    File file = new File("../../integration/data/struct_example.json");
+    try (
+            BufferAllocator readerAllocator = allocator.newChildAllocator("reader", 0, Integer.MAX_VALUE);
+    ) {
+      JsonFileReader reader = new JsonFileReader(file, readerAllocator);
+      Schema schema = reader.start();
+      LOGGER.debug("reading schema: " + schema);
+
+      // initialize vectors
+      try (VectorSchemaRoot root = reader.read();) {
+        FieldVector vector = root.getVector("struct_nullable");
+        Assert.assertEquals(7, vector.getAccessor().getValueCount());
+      }
+    }
+  }
+
 }


Mime
View raw message