arrow-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From w...@apache.org
Subject arrow git commit: ARROW-815 [Java] Exposing reAlloc for ValueVector
Date Fri, 14 Apr 2017 20:37:38 GMT
Repository: arrow
Updated Branches:
  refs/heads/master 01114d831 -> b6033378c


ARROW-815 [Java] Exposing reAlloc for ValueVector

Author: Emilio Lahr-Vivaz <elahrvivaz@ccri.com>

Closes #534 from elahrvivaz/ARROW-815 and squashes the following commits:

cf14944 [Emilio Lahr-Vivaz] unit test
45fa773 [Emilio Lahr-Vivaz] ARROW-815 [Java] Exposing reAlloc for ValueVector


Project: http://git-wip-us.apache.org/repos/asf/arrow/repo
Commit: http://git-wip-us.apache.org/repos/asf/arrow/commit/b6033378
Tree: http://git-wip-us.apache.org/repos/asf/arrow/tree/b6033378
Diff: http://git-wip-us.apache.org/repos/asf/arrow/diff/b6033378

Branch: refs/heads/master
Commit: b6033378c2533ed7b396f111cc5aed10450907fb
Parents: 01114d8
Author: Emilio Lahr-Vivaz <elahrvivaz@ccri.com>
Authored: Fri Apr 14 16:37:25 2017 -0400
Committer: Wes McKinney <wes.mckinney@twosigma.com>
Committed: Fri Apr 14 16:37:25 2017 -0400

----------------------------------------------------------------------
 .../codegen/templates/NullableValueVectors.java |   6 +
 .../src/main/codegen/templates/UnionVector.java |   6 +
 .../org/apache/arrow/vector/ValueVector.java    |   6 +
 .../org/apache/arrow/vector/ZeroVector.java     |   3 +
 .../arrow/vector/complex/AbstractMapVector.java |   7 +
 .../vector/complex/BaseRepeatedValueVector.java |  14 +-
 .../apache/arrow/vector/complex/ListVector.java |   6 +
 .../arrow/vector/complex/NullableMapVector.java |   7 +
 .../apache/arrow/vector/TestVectorReAlloc.java  | 144 +++++++++++++++++++
 9 files changed, 194 insertions(+), 5 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/arrow/blob/b6033378/java/vector/src/main/codegen/templates/NullableValueVectors.java
----------------------------------------------------------------------
diff --git a/java/vector/src/main/codegen/templates/NullableValueVectors.java b/java/vector/src/main/codegen/templates/NullableValueVectors.java
index e5257ce..acee6cb 100644
--- a/java/vector/src/main/codegen/templates/NullableValueVectors.java
+++ b/java/vector/src/main/codegen/templates/NullableValueVectors.java
@@ -242,6 +242,12 @@ public final class ${className} extends BaseDataValueVector implements
<#if type
     return success;
   }
 
+  @Override
+  public void reAlloc() {
+    bits.reAlloc();
+    values.reAlloc();
+  }
+
   <#if type.major == "VarLen">
   @Override
   public void allocateNew(int totalBytes, int valueCount) {

http://git-wip-us.apache.org/repos/asf/arrow/blob/b6033378/java/vector/src/main/codegen/templates/UnionVector.java
----------------------------------------------------------------------
diff --git a/java/vector/src/main/codegen/templates/UnionVector.java b/java/vector/src/main/codegen/templates/UnionVector.java
index 797b293..d70cbae 100644
--- a/java/vector/src/main/codegen/templates/UnionVector.java
+++ b/java/vector/src/main/codegen/templates/UnionVector.java
@@ -215,6 +215,12 @@ public class UnionVector implements FieldVector {
   }
 
   @Override
+  public void reAlloc() {
+    internalMap.reAlloc();
+    typeVector.reAlloc();
+  }
+
+  @Override
   public void setInitialCapacity(int numRecords) {
   }
 

http://git-wip-us.apache.org/repos/asf/arrow/blob/b6033378/java/vector/src/main/java/org/apache/arrow/vector/ValueVector.java
----------------------------------------------------------------------
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/ValueVector.java b/java/vector/src/main/java/org/apache/arrow/vector/ValueVector.java
index 8e35398..685b0be 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/ValueVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/ValueVector.java
@@ -68,6 +68,12 @@ public interface ValueVector extends Closeable, Iterable<ValueVector>
{
    */
   boolean allocateNewSafe();
 
+  /**
+   * Allocate new buffer with double capacity, and copy data into the new buffer.
+   * Replace vector's buffer with new buffer, and release old one
+   */
+  void reAlloc();
+
   BufferAllocator getAllocator();
 
   /**

http://git-wip-us.apache.org/repos/asf/arrow/blob/b6033378/java/vector/src/main/java/org/apache/arrow/vector/ZeroVector.java
----------------------------------------------------------------------
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/ZeroVector.java b/java/vector/src/main/java/org/apache/arrow/vector/ZeroVector.java
index 73f858e..e48788c 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/ZeroVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/ZeroVector.java
@@ -143,6 +143,9 @@ public class ZeroVector implements FieldVector {
   }
 
   @Override
+  public void reAlloc() {}
+
+  @Override
   public BufferAllocator getAllocator() {
     throw new UnsupportedOperationException("Tried to get allocator from ZeroVector");
   }

http://git-wip-us.apache.org/repos/asf/arrow/blob/b6033378/java/vector/src/main/java/org/apache/arrow/vector/complex/AbstractMapVector.java
----------------------------------------------------------------------
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/AbstractMapVector.java
b/java/vector/src/main/java/org/apache/arrow/vector/complex/AbstractMapVector.java
index dc833ed..15e8a5b 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/AbstractMapVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/AbstractMapVector.java
@@ -81,6 +81,13 @@ public abstract class AbstractMapVector extends AbstractContainerVector
{
     return true;
   }
 
+  @Override
+  public void reAlloc() {
+    for (final ValueVector v: vectors.values()) {
+      v.reAlloc();
+    }
+  }
+
   /**
    * Adds a new field with the given parameters or replaces the existing one and consequently
returns the resultant
    * {@link org.apache.arrow.vector.ValueVector}.

http://git-wip-us.apache.org/repos/asf/arrow/blob/b6033378/java/vector/src/main/java/org/apache/arrow/vector/complex/BaseRepeatedValueVector.java
----------------------------------------------------------------------
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/BaseRepeatedValueVector.java
b/java/vector/src/main/java/org/apache/arrow/vector/complex/BaseRepeatedValueVector.java
index 6b240c0..da221e3 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/BaseRepeatedValueVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/BaseRepeatedValueVector.java
@@ -20,6 +20,10 @@ package org.apache.arrow.vector.complex;
 import java.util.Collections;
 import java.util.Iterator;
 
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ObjectArrays;
+
+import io.netty.buffer.ArrowBuf;
 import org.apache.arrow.memory.BufferAllocator;
 import org.apache.arrow.vector.AddOrGetResult;
 import org.apache.arrow.vector.BaseValueVector;
@@ -31,11 +35,6 @@ import org.apache.arrow.vector.types.pojo.FieldType;
 import org.apache.arrow.vector.util.CallBack;
 import org.apache.arrow.vector.util.SchemaChangeRuntimeException;
 
-import com.google.common.base.Preconditions;
-import com.google.common.collect.ObjectArrays;
-
-import io.netty.buffer.ArrowBuf;
-
 public abstract class BaseRepeatedValueVector extends BaseValueVector implements RepeatedValueVector
{
 
   public final static FieldVector DEFAULT_DATA_VECTOR = ZeroVector.INSTANCE;
@@ -79,6 +78,11 @@ public abstract class BaseRepeatedValueVector extends BaseValueVector implements
     return success;
   }
 
+  @Override
+  public void reAlloc() {
+    offsets.reAlloc();
+    vector.reAlloc();
+  }
 
   @Override
   public UInt4Vector getOffsetVector() {

http://git-wip-us.apache.org/repos/asf/arrow/blob/b6033378/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java
----------------------------------------------------------------------
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java
index 0461a8d..63235df 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java
@@ -124,6 +124,12 @@ public class ListVector extends BaseRepeatedValueVector implements FieldVector
{
     bits.allocateNewSafe();
   }
 
+  @Override
+  public void reAlloc() {
+    super.reAlloc();
+    bits.reAlloc();
+  }
+
   public void copyFromSafe(int inIndex, int outIndex, ListVector from) {
     copyFrom(inIndex, outIndex, from);
   }

http://git-wip-us.apache.org/repos/asf/arrow/blob/b6033378/java/vector/src/main/java/org/apache/arrow/vector/complex/NullableMapVector.java
----------------------------------------------------------------------
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/NullableMapVector.java
b/java/vector/src/main/java/org/apache/arrow/vector/complex/NullableMapVector.java
index 71fee67..647ab28 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/NullableMapVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/NullableMapVector.java
@@ -197,6 +197,13 @@ public class NullableMapVector extends MapVector implements FieldVector
{
     bits.zeroVector();
     return success;
   }
+
+  @Override
+  public void reAlloc() {
+    bits.reAlloc();
+    super.reAlloc();
+  }
+
   public final class Accessor extends MapVector.Accessor  {
     final BitVector.Accessor bAccessor = bits.getAccessor();
 

http://git-wip-us.apache.org/repos/asf/arrow/blob/b6033378/java/vector/src/test/java/org/apache/arrow/vector/TestVectorReAlloc.java
----------------------------------------------------------------------
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestVectorReAlloc.java b/java/vector/src/test/java/org/apache/arrow/vector/TestVectorReAlloc.java
new file mode 100644
index 0000000..a7c35b6
--- /dev/null
+++ b/java/vector/src/test/java/org/apache/arrow/vector/TestVectorReAlloc.java
@@ -0,0 +1,144 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.arrow.vector;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
+
+import java.nio.charset.StandardCharsets;
+
+import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.memory.RootAllocator;
+import org.apache.arrow.vector.complex.ListVector;
+import org.apache.arrow.vector.complex.NullableMapVector;
+import org.apache.arrow.vector.types.Types.MinorType;
+import org.apache.arrow.vector.types.pojo.FieldType;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+
+public class TestVectorReAlloc {
+
+  private BufferAllocator allocator;
+
+  @Before
+  public void init() {
+    allocator = new RootAllocator(Long.MAX_VALUE);
+  }
+
+  @After
+  public void terminate() throws Exception {
+    allocator.close();
+  }
+
+  @Test
+  public void testFixedType() {
+    try (final UInt4Vector vector = new UInt4Vector("", allocator)) {
+      final UInt4Vector.Mutator m = vector.getMutator();
+      vector.setInitialCapacity(512);
+      vector.allocateNew();
+
+      assertEquals(512, vector.getValueCapacity());
+
+      try {
+        m.set(512, 0);
+        Assert.fail("Expected out of bounds exception");
+      } catch (Exception e) {
+        // ok
+      }
+
+      vector.reAlloc();
+      assertEquals(1024, vector.getValueCapacity());
+
+      m.set(512, 100);
+      assertEquals(100, vector.getAccessor().get(512));
+    }
+  }
+
+  @Test
+  public void testNullableType() {
+    try (final NullableVarCharVector vector = new NullableVarCharVector("", allocator)) {
+      final NullableVarCharVector.Mutator m = vector.getMutator();
+      vector.setInitialCapacity(512);
+      vector.allocateNew();
+
+      assertEquals(512, vector.getValueCapacity());
+
+      try {
+        m.set(512, "foo".getBytes(StandardCharsets.UTF_8));
+        Assert.fail("Expected out of bounds exception");
+      } catch (Exception e) {
+        // ok
+      }
+
+      vector.reAlloc();
+      assertEquals(1023, vector.getValueCapacity()); // note: size - 1 for some reason...
+
+      m.set(512, "foo".getBytes(StandardCharsets.UTF_8));
+      assertEquals("foo", new String(vector.getAccessor().get(512), StandardCharsets.UTF_8));
+    }
+  }
+
+  @Test
+  public void testListType() {
+    try (final ListVector vector = new ListVector("", allocator, null)) {
+      vector.addOrGetVector(FieldType.nullable(MinorType.INT.getType()));
+
+      vector.setInitialCapacity(512);
+      vector.allocateNew();
+
+      assertEquals(1023, vector.getValueCapacity()); // TODO this doubles for some reason...
+
+      try {
+        vector.getOffsetVector().getAccessor().get(2014);
+        Assert.fail("Expected out of bounds exception");
+      } catch (Exception e) {
+        // ok
+      }
+
+      vector.reAlloc();
+      assertEquals(2047, vector.getValueCapacity()); // note: size - 1
+      assertEquals(0, vector.getOffsetVector().getAccessor().get(2014));
+    }
+  }
+
+  @Test
+  public void testMapType() {
+    try (final NullableMapVector vector = new NullableMapVector("", allocator, null)) {
+      vector.addOrGet("", FieldType.nullable(MinorType.INT.getType()), NullableIntVector.class);
+
+      vector.setInitialCapacity(512);
+      vector.allocateNew();
+
+      assertEquals(512, vector.getValueCapacity());
+
+      try {
+        vector.getAccessor().getObject(513);
+        Assert.fail("Expected out of bounds exception");
+      } catch (Exception e) {
+        // ok
+      }
+
+      vector.reAlloc();
+      assertEquals(1024, vector.getValueCapacity());
+      assertNull(vector.getAccessor().getObject(513));
+    }
+  }
+}


Mime
View raw message