arrow-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From w...@apache.org
Subject arrow git commit: ARROW-801: Provide direct access to underlying buffer memory addresses
Date Wed, 16 Aug 2017 00:16:34 GMT
Repository: arrow
Updated Branches:
  refs/heads/master 94b7cfafa -> 31457ae82


ARROW-801: Provide direct access to underlying buffer memory addresses

Added following methods at FieldVector interface

public long getValidityBufferAddress();
public long getDataBufferAddress();
public long getOffsetBufferAddress();

Couple of points:

For the UnionVector, we don't have an explicit BitVector and the typeVector also acts as a
validity vector during get(), isNull(). So getValidityBufferAddress() returns the address
of buffer associated with typeVector.

Both ListVector and FixedSizeListVector are backed by a FieldVector which acts as the dataVector.
Right now the getDataBufferAddress() is unsupported for these two vectors but I think that
we could just simply delegate this call to the dataVector instead of throwing exception

Author: siddharth <siddharth@dremio.com>

Closes #958 from siddharthteotia/ARROW-801 and squashes the following commits:

cb6dd25 [siddharth] ARROW-801: Provide direct access to underlying buffer memory addresses


Project: http://git-wip-us.apache.org/repos/asf/arrow/repo
Commit: http://git-wip-us.apache.org/repos/asf/arrow/commit/31457ae8
Tree: http://git-wip-us.apache.org/repos/asf/arrow/tree/31457ae8
Diff: http://git-wip-us.apache.org/repos/asf/arrow/diff/31457ae8

Branch: refs/heads/master
Commit: 31457ae826b7710db4982ae6680172c1068e4d88
Parents: 94b7cfa
Author: siddharth <siddharth@dremio.com>
Authored: Tue Aug 15 20:16:27 2017 -0400
Committer: Wes McKinney <wes.mckinney@twosigma.com>
Committed: Tue Aug 15 20:16:27 2017 -0400

----------------------------------------------------------------------
 .../codegen/templates/NullableValueVectors.java | 19 +++++
 .../src/main/codegen/templates/UnionVector.java | 15 ++++
 .../org/apache/arrow/vector/FieldVector.java    | 21 +++++
 .../org/apache/arrow/vector/ZeroVector.java     | 15 ++++
 .../vector/complex/FixedSizeListVector.java     | 15 ++++
 .../apache/arrow/vector/complex/ListVector.java | 15 ++++
 .../arrow/vector/complex/NullableMapVector.java | 15 ++++
 .../org/apache/arrow/vector/TestListVector.java | 63 ++++++++++++++-
 .../apache/arrow/vector/TestUnionVector.java    | 70 +++++++++++++++++
 .../apache/arrow/vector/TestValueVector.java    | 81 ++++++++++++++++++++
 10 files changed, 327 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/arrow/blob/31457ae8/java/vector/src/main/codegen/templates/NullableValueVectors.java
----------------------------------------------------------------------
diff --git a/java/vector/src/main/codegen/templates/NullableValueVectors.java b/java/vector/src/main/codegen/templates/NullableValueVectors.java
index 624ba9d..bd322ea 100644
--- a/java/vector/src/main/codegen/templates/NullableValueVectors.java
+++ b/java/vector/src/main/codegen/templates/NullableValueVectors.java
@@ -435,6 +435,25 @@ protected final static byte[] emptyByteArray = new byte[]{};
     <#if type.major == "VarLen">mutator.lastSet = thisIndex;</#if>
   }
 
+  @Override
+  public long getValidityBufferAddress() {
+    return (bits.getBuffer().memoryAddress());
+  }
+
+  @Override
+  public long getDataBufferAddress() {
+    return (values.getBuffer().memoryAddress());
+  }
+
+  @Override
+  public long getOffsetBufferAddress() {
+    <#if type.major != "VarLen">
+        throw new UnsupportedOperationException();
+    <#else>
+        return (values.getOffsetAddr());
+    </#if>
+  }
+
   public final class Accessor extends BaseDataValueVector.BaseAccessor <#if type.major
= "VarLen">implements VariableWidthVector.VariableWidthAccessor</#if> {
     final BitVector.Accessor bAccessor = bits.getAccessor();
     final ${valuesName}.Accessor vAccessor = values.getAccessor();

http://git-wip-us.apache.org/repos/asf/arrow/blob/31457ae8/java/vector/src/main/codegen/templates/UnionVector.java
----------------------------------------------------------------------
diff --git a/java/vector/src/main/codegen/templates/UnionVector.java b/java/vector/src/main/codegen/templates/UnionVector.java
index eabe42a..8c01c4a 100644
--- a/java/vector/src/main/codegen/templates/UnionVector.java
+++ b/java/vector/src/main/codegen/templates/UnionVector.java
@@ -132,6 +132,21 @@ public class UnionVector implements FieldVector {
     return internalMap.addOrGet(fieldName(minorType), fieldType(minorType), c);
   }
 
+  @Override
+  public long getValidityBufferAddress() {
+    return typeVector.getBuffer().memoryAddress();
+  }
+
+  @Override
+  public long getDataBufferAddress() {
+    throw new UnsupportedOperationException();
+  }
+
+  @Override
+  public long getOffsetBufferAddress() {
+    throw new UnsupportedOperationException();
+  }
+
   public NullableMapVector getMap() {
     if (mapVector == null) {
       int vectorCount = internalMap.size();

http://git-wip-us.apache.org/repos/asf/arrow/blob/31457ae8/java/vector/src/main/java/org/apache/arrow/vector/FieldVector.java
----------------------------------------------------------------------
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/FieldVector.java b/java/vector/src/main/java/org/apache/arrow/vector/FieldVector.java
index af7a791..c2ed17e 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/FieldVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/FieldVector.java
@@ -66,4 +66,25 @@ public interface FieldVector extends ValueVector {
    * @return the inner vectors for this field as defined by the TypeLayout
    */
   List<BufferBacked> getFieldInnerVectors();
+
+  /**
+   * Gets the starting address of the underlying buffer associated with validity vector
+   *
+   * @return buffer address
+   */
+  public long getValidityBufferAddress();
+
+  /**
+   * Gets the starting address of the underlying buffer associated with data vector
+   *
+   * @return buffer address
+   */
+  public long getDataBufferAddress();
+
+  /**
+   * Gets the starting address of the underlying buffer associated with offset vector
+   *
+   * @return buffer address
+   */
+  public long getOffsetBufferAddress();
 }

http://git-wip-us.apache.org/repos/asf/arrow/blob/31457ae8/java/vector/src/main/java/org/apache/arrow/vector/ZeroVector.java
----------------------------------------------------------------------
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/ZeroVector.java b/java/vector/src/main/java/org/apache/arrow/vector/ZeroVector.java
index cce7389..fe224ba 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/ZeroVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/ZeroVector.java
@@ -229,4 +229,19 @@ public class ZeroVector implements FieldVector {
   public List<BufferBacked> getFieldInnerVectors() {
     return Collections.emptyList();
   }
+
+  @Override
+  public long getValidityBufferAddress() {
+    throw new UnsupportedOperationException();
+  }
+
+  @Override
+  public long getDataBufferAddress() {
+    throw new UnsupportedOperationException();
+  }
+
+  @Override
+  public long getOffsetBufferAddress() {
+    throw new UnsupportedOperationException();
+  }
 }

http://git-wip-us.apache.org/repos/asf/arrow/blob/31457ae8/java/vector/src/main/java/org/apache/arrow/vector/complex/FixedSizeListVector.java
----------------------------------------------------------------------
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/FixedSizeListVector.java
b/java/vector/src/main/java/org/apache/arrow/vector/complex/FixedSizeListVector.java
index 3f0f1b0..aab6e57 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/FixedSizeListVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/FixedSizeListVector.java
@@ -293,6 +293,21 @@ public class FixedSizeListVector extends BaseValueVector implements FieldVector,
     return vector;
   }
 
+  @Override
+  public long getValidityBufferAddress() {
+    return (bits.getBuffer().memoryAddress());
+  }
+
+  @Override
+  public long getDataBufferAddress() {
+    throw new UnsupportedOperationException();
+  }
+
+  @Override
+  public long getOffsetBufferAddress() {
+    throw new UnsupportedOperationException();
+  }
+
   public class Accessor extends BaseValueVector.BaseAccessor {
 
     @Override

http://git-wip-us.apache.org/repos/asf/arrow/blob/31457ae8/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java
----------------------------------------------------------------------
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java
index 7b6b97a..4b2808d 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java
@@ -178,6 +178,21 @@ public class ListVector extends BaseRepeatedValueVector implements FieldVector,
     return new TransferImpl((ListVector) target);
   }
 
+  @Override
+  public long getValidityBufferAddress() {
+    return (bits.getBuffer().memoryAddress());
+  }
+
+  @Override
+  public long getDataBufferAddress() {
+    throw new UnsupportedOperationException();
+  }
+
+  @Override
+  public long getOffsetBufferAddress() {
+    return (offsets.getBuffer().memoryAddress());
+  }
+
   private class TransferImpl implements TransferPair {
 
     ListVector to;

http://git-wip-us.apache.org/repos/asf/arrow/blob/31457ae8/java/vector/src/main/java/org/apache/arrow/vector/complex/NullableMapVector.java
----------------------------------------------------------------------
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/NullableMapVector.java
b/java/vector/src/main/java/org/apache/arrow/vector/complex/NullableMapVector.java
index fda9c14..09560c2 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/NullableMapVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/NullableMapVector.java
@@ -231,6 +231,21 @@ public class NullableMapVector extends MapVector implements FieldVector
{
     super.reAlloc();
   }
 
+  @Override
+  public long getValidityBufferAddress() {
+    return bits.getBuffer().memoryAddress();
+  }
+
+  @Override
+  public long getDataBufferAddress() {
+    throw new UnsupportedOperationException();
+  }
+
+  @Override
+  public long getOffsetBufferAddress() {
+    throw new UnsupportedOperationException();
+  }
+
   public final class Accessor extends MapVector.Accessor {
     final BitVector.Accessor bAccessor = bits.getAccessor();
 

http://git-wip-us.apache.org/repos/asf/arrow/blob/31457ae8/java/vector/src/test/java/org/apache/arrow/vector/TestListVector.java
----------------------------------------------------------------------
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestListVector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestListVector.java
index fdb576e..276121c 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/TestListVector.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/TestListVector.java
@@ -42,6 +42,8 @@ import org.junit.Test;
 import java.util.ArrayList;
 import java.util.List;
 
+import io.netty.buffer.ArrowBuf;
+
 
 public class TestListVector {
 
@@ -436,14 +438,14 @@ public class TestListVector {
             dataLength2 = offsetAccessor1.get(i + 1) - offsetAccessor1.get(i);
 
             assertEquals("Different data lengths at index: " + i + " and start: " + start,
-                dataLength1, dataLength2);
+                    dataLength1, dataLength2);
 
             offset1 = offsetAccessor.get(start + i);
             offset2 = offsetAccessor1.get(i);
 
             for (int j = 0; j < dataLength1; j++) {
               assertEquals("Different data at indexes: " + offset1 + " and " + offset2,
-                  valueAccessor.getObject(offset1), valueAccessor1.getObject(offset2));
+                      valueAccessor.getObject(offset1), valueAccessor1.getObject(offset2));
 
               offset1++;
               offset2++;
@@ -569,4 +571,61 @@ public class TestListVector {
       assertEquals(5, offsetAccessor.get(2));
     }
   }
+
+  @Test
+  public void testGetBufferAddress() throws Exception {
+    try (ListVector listVector = ListVector.empty("vector", allocator)) {
+
+      UnionListWriter listWriter = listVector.getWriter();
+      boolean error = false;
+
+      listWriter.allocate();
+
+      listWriter.setPosition(0);
+      listWriter.startList();
+      listWriter.bigInt().writeBigInt(50);
+      listWriter.bigInt().writeBigInt(100);
+      listWriter.bigInt().writeBigInt(200);
+      listWriter.endList();
+
+      listWriter.setPosition(1);
+      listWriter.startList();
+      listWriter.bigInt().writeBigInt(250);
+      listWriter.bigInt().writeBigInt(300);
+      listWriter.endList();
+
+      final ListVector.Accessor accessor = listVector.getAccessor();
+
+      /* check listVector contents */
+      Object result = accessor.getObject(0);
+      ArrayList<Long> resultSet = (ArrayList<Long>) result;
+      assertEquals(3, resultSet.size());
+      assertEquals(new Long(50), resultSet.get(0));
+      assertEquals(new Long(100), resultSet.get(1));
+      assertEquals(new Long(200), resultSet.get(2));
+
+      result = accessor.getObject(1);
+      resultSet = (ArrayList<Long>) result;
+      assertEquals(2, resultSet.size());
+      assertEquals(new Long(250), resultSet.get(0));
+      assertEquals(new Long(300), resultSet.get(1));
+
+      List<ArrowBuf> buffers = listVector.getFieldBuffers();
+
+      long bitAddress = listVector.getValidityBufferAddress();
+      long offsetAddress = listVector.getOffsetBufferAddress();
+
+      try {
+        long dataAddress = listVector.getDataBufferAddress();
+      } catch (UnsupportedOperationException ue) {
+        error = true;
+      } finally {
+        assertTrue(error);
+      }
+
+      assertEquals(2, buffers.size());
+      assertEquals(bitAddress, buffers.get(0).memoryAddress());
+      assertEquals(offsetAddress, buffers.get(1).memoryAddress());
+    }
+  }
 }

http://git-wip-us.apache.org/repos/asf/arrow/blob/31457ae8/java/vector/src/test/java/org/apache/arrow/vector/TestUnionVector.java
----------------------------------------------------------------------
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestUnionVector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestUnionVector.java
index a75b196..86f0bf3 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/TestUnionVector.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/TestUnionVector.java
@@ -19,6 +19,9 @@
 package org.apache.arrow.vector;
 
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+import java.util.List;
 
 import org.apache.arrow.memory.BufferAllocator;
 import org.apache.arrow.vector.complex.UnionVector;
@@ -33,6 +36,8 @@ import org.junit.After;
 import org.junit.Before;
 import org.junit.Test;
 
+import io.netty.buffer.ArrowBuf;
+
 public class TestUnionVector {
   private final static String EMPTY_SCHEMA_PATH = "";
 
@@ -292,6 +297,71 @@ public class TestUnionVector {
     }
   }
 
+  @Test
+  public void testGetBufferAddress() throws Exception {
+    try (UnionVector vector = new UnionVector(EMPTY_SCHEMA_PATH, allocator, null)) {
+      final UnionVector.Mutator mutator = vector.getMutator();
+      final UnionVector.Accessor accessor = vector.getAccessor();
+      boolean error = false;
+
+      vector.allocateNew();
+
+      /* populate the UnionVector */
+      mutator.setType(0, MinorType.INT);
+      mutator.setSafe(0, newIntHolder(5));
+
+      mutator.setType(1, MinorType.FLOAT4);
+      mutator.setSafe(1, newFloat4Holder(5.5f));
+
+      mutator.setType(2, MinorType.INT);
+      mutator.setSafe(2, newIntHolder(10));
+
+      mutator.setType(3, MinorType.FLOAT4);
+      mutator.setSafe(3, newFloat4Holder(10.5f));
+
+      mutator.setValueCount(10);
+
+      /* check the vector output */
+      assertEquals(10, accessor.getValueCount());
+      assertEquals(false, accessor.isNull(0));
+      assertEquals(5, accessor.getObject(0));
+      assertEquals(false, accessor.isNull(1));
+      assertEquals(5.5f, accessor.getObject(1));
+      assertEquals(false, accessor.isNull(2));
+      assertEquals(10, accessor.getObject(2));
+      assertEquals(false, accessor.isNull(3));
+      assertEquals(10.5f, accessor.getObject(3));
+
+      List<ArrowBuf> buffers = vector.getFieldBuffers();
+
+      long bitAddress = vector.getValidityBufferAddress();
+
+      try {
+        long offsetAddress = vector.getOffsetBufferAddress();
+      }
+      catch (UnsupportedOperationException ue) {
+        error = true;
+      }
+      finally {
+        assertTrue(error);
+        error = false;
+      }
+
+      try {
+        long dataAddress = vector.getDataBufferAddress();
+      }
+      catch (UnsupportedOperationException ue) {
+        error = true;
+      }
+      finally {
+        assertTrue(error);
+      }
+
+      assertEquals(1, buffers.size());
+      assertEquals(bitAddress, buffers.get(0).memoryAddress());
+    }
+  }
+
   private static NullableIntHolder newIntHolder(int value) {
     final NullableIntHolder holder = new NullableIntHolder();
     holder.isSet = 1;

http://git-wip-us.apache.org/repos/asf/arrow/blob/31457ae8/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java
----------------------------------------------------------------------
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java
index 4e3021b..cb1fa89 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java
@@ -1164,6 +1164,87 @@ public class TestValueVector {
     }
   }
 
+  @Test /* NullableVarCharVector */
+  public void testGetBufferAddress1() {
+
+    try (final NullableVarCharVector vector = new NullableVarCharVector("myvector", allocator))
{
+
+      final NullableVarCharVector.Mutator mutator = vector.getMutator();
+      final NullableVarCharVector.Accessor accessor = vector.getAccessor();
+
+      vector.allocateNew(1024 * 10, 1024);
+
+      /* populate the vector */
+      mutator.set(0, STR1);
+      mutator.set(1, STR2);
+      mutator.set(2, STR3);
+      mutator.set(3, STR4);
+      mutator.set(4, STR5);
+      mutator.set(5, STR6);
+
+      mutator.setValueCount(15);
+
+      /* check the vector output */
+      assertArrayEquals(STR1, accessor.get(0));
+      assertArrayEquals(STR2, accessor.get(1));
+      assertArrayEquals(STR3, accessor.get(2));
+      assertArrayEquals(STR4, accessor.get(3));
+      assertArrayEquals(STR5, accessor.get(4));
+      assertArrayEquals(STR6, accessor.get(5));
+
+      List<ArrowBuf> buffers = vector.getFieldBuffers();
+      long bitAddress = vector.getValidityBufferAddress();
+      long offsetAddress = vector.getOffsetBufferAddress();
+      long dataAddress = vector.getDataBufferAddress();
+
+      assertEquals(3, buffers.size());
+      assertEquals(bitAddress, buffers.get(0).memoryAddress());
+      assertEquals(offsetAddress, buffers.get(1).memoryAddress());
+      assertEquals(dataAddress, buffers.get(2).memoryAddress());
+    }
+  }
+
+  @Test /* NullableIntVector */
+  public void testGetBufferAddress2() {
+
+    try (final NullableIntVector vector = new NullableIntVector("myvector", allocator)) {
+
+      final NullableIntVector.Mutator mutator = vector.getMutator();
+      final NullableIntVector.Accessor accessor = vector.getAccessor();
+      boolean error = false;
+
+      vector.allocateNew(16);
+
+      /* populate the vector */
+      for(int i = 0; i < 16; i += 2) {
+        mutator.set(i, i+10);
+      }
+
+      /* check the vector output */
+      for(int i = 0; i < 16; i += 2) {
+        assertEquals(i+10, accessor.get(i));
+      }
+
+      List<ArrowBuf> buffers = vector.getFieldBuffers();
+      long bitAddress = vector.getValidityBufferAddress();
+      long dataAddress = vector.getDataBufferAddress();
+
+      try {
+        long offsetAddress = vector.getOffsetBufferAddress();
+      }
+      catch (UnsupportedOperationException ue) {
+        error = true;
+      }
+      finally {
+        assertTrue(error);
+      }
+
+      assertEquals(2, buffers.size());
+      assertEquals(bitAddress, buffers.get(0).memoryAddress());
+      assertEquals(dataAddress, buffers.get(1).memoryAddress());
+    }
+  }
+
   @Test
   public void testMultipleClose() {
     BufferAllocator vectorAllocator = allocator.newChildAllocator("vector_allocator", 0,
Long.MAX_VALUE);


Mime
View raw message