arrow-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From s..@apache.org
Subject [1/2] arrow git commit: Make BaseValueVector#MAX_ALLOCATION_SIZE configurable
Date Thu, 19 May 2016 20:40:14 GMT
Repository: arrow
Updated Branches:
  refs/heads/master e0fb3698e -> e316b3f76


Make BaseValueVector#MAX_ALLOCATION_SIZE configurable

This closes #65

Some of the tests are based on the assumption that the JVM can allocate at least
2GB of memory, which is not a common occurence (JVM usually defaults at 512MB).
Current Travis CI VM only have 3GB of memory total, which would have make challenging
to run some of the tests on them

Add a system property to change BaseValueVector.MAX_ALLOCATION_SIZE to allow to use
a much smaller value during tests.


Project: http://git-wip-us.apache.org/repos/asf/arrow/repo
Commit: http://git-wip-us.apache.org/repos/asf/arrow/commit/c0985a47
Tree: http://git-wip-us.apache.org/repos/asf/arrow/tree/c0985a47
Diff: http://git-wip-us.apache.org/repos/asf/arrow/diff/c0985a47

Branch: refs/heads/master
Commit: c0985a47665f8ce8847a6a0215e6e3c0f1db28f4
Parents: e0fb369
Author: Laurent Goujon <laurent@dremio.com>
Authored: Mon Apr 18 11:07:22 2016 -0700
Committer: Steven Phillips <steven@dremio.com>
Committed: Thu May 19 13:17:03 2016 -0700

----------------------------------------------------------------------
 .../apache/arrow/vector/BaseValueVector.java    | 14 ++++----
 .../apache/arrow/vector/TestValueVector.java    | 36 ++++++++++++++++----
 2 files changed, 38 insertions(+), 12 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/arrow/blob/c0985a47/java/vector/src/main/java/org/apache/arrow/vector/BaseValueVector.java
----------------------------------------------------------------------
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/BaseValueVector.java b/java/vector/src/main/java/org/apache/arrow/vector/BaseValueVector.java
index 8bca3c0..932e6f1 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/BaseValueVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/BaseValueVector.java
@@ -17,23 +17,24 @@
  */
 package org.apache.arrow.vector;
 
-import io.netty.buffer.ArrowBuf;
-
 import java.util.Iterator;
 
-import com.google.common.base.Preconditions;
-import com.google.common.collect.Iterators;
-
 import org.apache.arrow.memory.BufferAllocator;
 import org.apache.arrow.vector.types.MaterializedField;
 import org.apache.arrow.vector.util.TransferPair;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Iterators;
+
+import io.netty.buffer.ArrowBuf;
+
 public abstract class BaseValueVector implements ValueVector {
   private static final Logger logger = LoggerFactory.getLogger(BaseValueVector.class);
 
-  public static final int MAX_ALLOCATION_SIZE = Integer.MAX_VALUE;
+  public static final String MAX_ALLOCATION_SIZE_PROPERTY = "arrow.vector.max_allocation_bytes";
+  public static final int MAX_ALLOCATION_SIZE = Integer.getInteger(MAX_ALLOCATION_SIZE_PROPERTY,
Integer.MAX_VALUE);
   public static final int INITIAL_VALUE_ALLOCATION = 4096;
 
   protected final BufferAllocator allocator;
@@ -99,6 +100,7 @@ public abstract class BaseValueVector implements ValueVector {
     public void generateTestData(int values) {}
 
     //TODO: consider making mutator stateless(if possible) on another issue.
+    @Override
     public void reset() {}
   }
 

http://git-wip-us.apache.org/repos/asf/arrow/blob/c0985a47/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java
----------------------------------------------------------------------
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java
index ac3eebe..b5c4509 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java
@@ -23,16 +23,12 @@ import static org.junit.Assert.assertTrue;
 
 import java.nio.charset.Charset;
 
+import org.apache.arrow.memory.BufferAllocator;
 import org.apache.arrow.memory.RootAllocator;
 import org.apache.arrow.vector.complex.ListVector;
 import org.apache.arrow.vector.complex.MapVector;
 import org.apache.arrow.vector.complex.RepeatedListVector;
 import org.apache.arrow.vector.complex.RepeatedMapVector;
-import org.apache.arrow.vector.types.MaterializedField;
-import org.apache.arrow.vector.types.Types;
-import org.apache.arrow.vector.types.Types.MinorType;
-import org.apache.arrow.vector.util.BasicTypeHelper;
-import org.apache.arrow.vector.util.OversizedAllocationException;
 import org.apache.arrow.vector.holders.BitHolder;
 import org.apache.arrow.vector.holders.IntHolder;
 import org.apache.arrow.vector.holders.NullableFloat4Holder;
@@ -44,10 +40,16 @@ import org.apache.arrow.vector.holders.RepeatedIntHolder;
 import org.apache.arrow.vector.holders.RepeatedVarBinaryHolder;
 import org.apache.arrow.vector.holders.UInt4Holder;
 import org.apache.arrow.vector.holders.VarCharHolder;
-import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.vector.types.MaterializedField;
+import org.apache.arrow.vector.types.Types;
+import org.apache.arrow.vector.types.Types.MinorType;
+import org.apache.arrow.vector.util.BasicTypeHelper;
+import org.apache.arrow.vector.util.OversizedAllocationException;
 import org.junit.After;
 import org.junit.Before;
+import org.junit.Rule;
 import org.junit.Test;
+import org.junit.rules.ExternalResource;
 
 
 public class TestValueVector {
@@ -57,6 +59,28 @@ public class TestValueVector {
 
   private BufferAllocator allocator;
 
+  // Rule to adjust MAX_ALLOCATION_SIZE and restore it back after the tests
+  @Rule
+  public final ExternalResource rule = new ExternalResource() {
+    private final String systemValue = System.getProperty(BaseValueVector.MAX_ALLOCATION_SIZE_PROPERTY);
+    private final String testValue = Long.toString(32*1024*1024);
+
+    @Override
+    protected void before() throws Throwable {
+      System.setProperty(BaseValueVector.MAX_ALLOCATION_SIZE_PROPERTY, testValue);
+    }
+
+    @Override
+    protected void after() {
+      if (systemValue != null) {
+        System.setProperty(BaseValueVector.MAX_ALLOCATION_SIZE_PROPERTY, systemValue);
+      }
+      else {
+        System.clearProperty(BaseValueVector.MAX_ALLOCATION_SIZE_PROPERTY);
+      }
+    }
+  };
+
   @Before
   public void init() {
     allocator = new RootAllocator(Long.MAX_VALUE);


Mime
View raw message