arrow-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From u..@apache.org
Subject arrow git commit: ARROW-524: provide apis to access nested vectors and buffers
Date Sun, 05 Feb 2017 13:06:45 GMT
Repository: arrow
Updated Branches:
  refs/heads/master c45c3b3e1 -> 70c05be21


ARROW-524: provide apis to access nested vectors and buffers

Author: Julien Le Dem <julien@dremio.com>

Closes #314 from julienledem/setRangeToOne and squashes the following commits:

0d526bd [Julien Le Dem] ARROW-524: provide apis to access nested vectors and buffers


Project: http://git-wip-us.apache.org/repos/asf/arrow/repo
Commit: http://git-wip-us.apache.org/repos/asf/arrow/commit/70c05be2
Tree: http://git-wip-us.apache.org/repos/asf/arrow/tree/70c05be2
Diff: http://git-wip-us.apache.org/repos/asf/arrow/diff/70c05be2

Branch: refs/heads/master
Commit: 70c05be2130bdbb650a83bc46f7c4f8fc8a231df
Parents: c45c3b3
Author: Julien Le Dem <julien@dremio.com>
Authored: Sun Feb 5 14:06:37 2017 +0100
Committer: Uwe L. Korn <uwelk@xhochy.com>
Committed: Sun Feb 5 14:06:37 2017 +0100

----------------------------------------------------------------------
 .../codegen/templates/NullableValueVectors.java | 21 +++--
 .../java/org/apache/arrow/vector/BitVector.java | 88 +++++++++++++++++++-
 .../org/apache/arrow/vector/NullableVector.java |  2 +
 .../apache/arrow/vector/TestValueVector.java    | 36 ++++++++
 4 files changed, 137 insertions(+), 10 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/arrow/blob/70c05be2/java/vector/src/main/codegen/templates/NullableValueVectors.java
----------------------------------------------------------------------
diff --git a/java/vector/src/main/codegen/templates/NullableValueVectors.java b/java/vector/src/main/codegen/templates/NullableValueVectors.java
index ce63710..6b25fb3 100644
--- a/java/vector/src/main/codegen/templates/NullableValueVectors.java
+++ b/java/vector/src/main/codegen/templates/NullableValueVectors.java
@@ -132,6 +132,11 @@ public final class ${className} extends BaseDataValueVector implements
<#if type
   </#if>
 
   @Override
+  public BitVector getValidityVector() {
+    return bits;
+  }
+
+  @Override
   public List<BufferBacked> getFieldInnerVectors() {
     return innerVectors;
   }
@@ -426,7 +431,7 @@ public final class ${className} extends BaseDataValueVector implements
<#if type
     mutator.fillEmpties(thisIndex);
     </#if>
     values.copyFromSafe(fromIndex, thisIndex, from);
-    bits.getMutator().setSafe(thisIndex, 1);
+    bits.getMutator().setSafeToOne(thisIndex);
   }
 
   public void copyFromSafe(int fromIndex, int thisIndex, ${className} from){
@@ -525,7 +530,7 @@ public final class ${className} extends BaseDataValueVector implements
<#if type
 
     @Override
     public void setIndexDefined(int index){
-      bits.getMutator().set(index, 1);
+      bits.getMutator().setToOne(index);
     }
 
     /**
@@ -543,7 +548,7 @@ public final class ${className} extends BaseDataValueVector implements
<#if type
         valuesMutator.set(i, emptyByteArray);
       }
       </#if>
-      bitsMutator.set(index, 1);
+      bitsMutator.setToOne(index);
       valuesMutator.set(index, value);
       <#if type.major == "VarLen">lastSet = index;</#if>
     }
@@ -574,7 +579,7 @@ public final class ${className} extends BaseDataValueVector implements
<#if type
       <#else>
       fillEmpties(index);
 
-      bits.getMutator().setSafe(index, 1);
+      bits.getMutator().setSafeToOne(index);
       values.getMutator().setSafe(index, value, start, length);
       setCount++;
       <#if type.major == "VarLen">lastSet = index;</#if>
@@ -587,7 +592,7 @@ public final class ${className} extends BaseDataValueVector implements
<#if type
       <#else>
       fillEmpties(index);
 
-      bits.getMutator().setSafe(index, 1);
+      bits.getMutator().setSafeToOne(index);
       values.getMutator().setSafe(index, value, start, length);
       setCount++;
       <#if type.major == "VarLen">lastSet = index;</#if>
@@ -626,7 +631,7 @@ public final class ${className} extends BaseDataValueVector implements
<#if type
         valuesMutator.set(i, emptyByteArray);
       }
       </#if>
-      bits.getMutator().set(index, 1);
+      bits.getMutator().setToOne(index);
       valuesMutator.set(index, holder);
       <#if type.major == "VarLen">lastSet = index;</#if>
     }
@@ -676,7 +681,7 @@ public final class ${className} extends BaseDataValueVector implements
<#if type
       <#if type.major == "VarLen">
       fillEmpties(index);
       </#if>
-      bits.getMutator().setSafe(index, 1);
+      bits.getMutator().setSafeToOne(index);
       values.getMutator().setSafe(index, value);
       setCount++;
       <#if type.major == "VarLen">lastSet = index;</#if>
@@ -687,7 +692,7 @@ public final class ${className} extends BaseDataValueVector implements
<#if type
         <#if type.major == "VarLen">
         fillEmpties(index);
         </#if>
-        bits.getMutator().setSafe(index, 1);
+        bits.getMutator().setSafeToOne(index);
         values.getMutator().setSafe(index, value);
         setCount++;
       }

http://git-wip-us.apache.org/repos/asf/arrow/blob/70c05be2/java/vector/src/main/java/org/apache/arrow/vector/BitVector.java
----------------------------------------------------------------------
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/BitVector.java b/java/vector/src/main/java/org/apache/arrow/vector/BitVector.java
index 9beabcb..d1e9abe 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/BitVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/BitVector.java
@@ -423,8 +423,8 @@ public final class BitVector extends BaseDataValueVector implements FixedWidthVe
      *          value to set (either 1 or 0)
      */
     public final void set(int index, int value) {
-      int byteIndex = index >> 3;
-      int bitIndex = index & 7;
+      int byteIndex = byteIndex(index);
+      int bitIndex = bitIndex(index);
       byte currentByte = data.getByte(byteIndex);
       byte bitMask = (byte) (1L << bitIndex);
       if (value != 0) {
@@ -432,10 +432,87 @@ public final class BitVector extends BaseDataValueVector implements
FixedWidthVe
       } else {
         currentByte -= (bitMask & currentByte);
       }
+      data.setByte(byteIndex, currentByte);
+    }
 
+    /**
+     * Set the bit at the given index to 1.
+     *
+     * @param index position of the bit to set
+     */
+    public final void setToOne(int index) {
+      int byteIndex = byteIndex(index);
+      int bitIndex = bitIndex(index);
+      byte currentByte = data.getByte(byteIndex);
+      byte bitMask = (byte) (1L << bitIndex);
+      currentByte |= bitMask;
       data.setByte(byteIndex, currentByte);
     }
 
+    /**
+     * set count bits to 1 in data starting at firstBitIndex
+     * @param data the buffer to set
+     * @param firstBitIndex the index of the first bit to set
+     * @param count the number of bits to set
+     */
+    public void setRangeToOne(int firstBitIndex, int count) {
+      int starByteIndex = byteIndex(firstBitIndex);
+      final int lastBitIndex = firstBitIndex + count;
+      final int endByteIndex = byteIndex(lastBitIndex);
+      final int startByteBitIndex = bitIndex(firstBitIndex);
+      final int endBytebitIndex = bitIndex(lastBitIndex);
+      if (count < 8 && starByteIndex == endByteIndex) {
+        // handles the case where we don't have a first and a last byte
+        byte bitMask = 0;
+        for (int i = startByteBitIndex; i < endBytebitIndex; ++i) {
+          bitMask |= (byte) (1L << i);
+        }
+        byte currentByte = data.getByte(starByteIndex);
+        currentByte |= bitMask;
+        data.setByte(starByteIndex, currentByte);
+      } else {
+        // fill in first byte (if it's not full)
+        if (startByteBitIndex != 0) {
+          byte currentByte = data.getByte(starByteIndex);
+          final byte bitMask = (byte) (0xFFL << startByteBitIndex);
+          currentByte |= bitMask;
+          data.setByte(starByteIndex, currentByte);
+          ++ starByteIndex;
+        }
+
+        // fill in one full byte at a time
+        for (int i = starByteIndex; i < endByteIndex; i++) {
+          data.setByte(i, 0xFF);
+        }
+
+        // fill in the last byte (if it's not full)
+        if (endBytebitIndex != 0) {
+          final int byteIndex = byteIndex(lastBitIndex - endBytebitIndex);
+          byte currentByte = data.getByte(byteIndex);
+          final byte bitMask = (byte) (0xFFL >>> ((8 - endBytebitIndex) & 7));
+          currentByte |= bitMask;
+          data.setByte(byteIndex, currentByte);
+        }
+
+      }
+    }
+
+    /**
+     * @param absoluteBitIndex the index of the bit in the buffer
+     * @return the index of the byte containing that bit
+     */
+    private int byteIndex(int absoluteBitIndex) {
+      return absoluteBitIndex >> 3;
+    }
+
+    /**
+     * @param absoluteBitIndex the index of the bit in the buffer
+     * @return the index of the bit inside the byte
+     */
+    private int bitIndex(int absoluteBitIndex) {
+      return absoluteBitIndex & 7;
+    }
+
     public final void set(int index, BitHolder holder) {
       set(index, holder.value);
     }
@@ -451,6 +528,13 @@ public final class BitVector extends BaseDataValueVector implements FixedWidthVe
       set(index, value);
     }
 
+    public void setSafeToOne(int index) {
+      while(index >= getValueCapacity()) {
+        reAlloc();
+      }
+      setToOne(index);
+    }
+
     public void setSafe(int index, BitHolder holder) {
       while(index >= getValueCapacity()) {
         reAlloc();

http://git-wip-us.apache.org/repos/asf/arrow/blob/70c05be2/java/vector/src/main/java/org/apache/arrow/vector/NullableVector.java
----------------------------------------------------------------------
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableVector.java
index 0212b3c..b49e916 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/NullableVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableVector.java
@@ -19,5 +19,7 @@ package org.apache.arrow.vector;
 
 public interface NullableVector extends ValueVector {
 
+  BitVector getValidityVector();
+
   ValueVector getValuesVector();
 }

http://git-wip-us.apache.org/repos/asf/arrow/blob/70c05be2/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java
----------------------------------------------------------------------
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java
index b33919b..774b59e 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java
@@ -30,6 +30,7 @@ import org.apache.arrow.vector.schema.TypeLayout;
 import org.apache.arrow.vector.types.Types.MinorType;
 import org.apache.arrow.vector.types.pojo.Field;
 import org.junit.After;
+import org.junit.Assert;
 import org.junit.Before;
 import org.junit.Test;
 
@@ -365,6 +366,41 @@ public class TestValueVector {
   }
 
   @Test
+  public void testBitVectorRangeSetAllOnes() {
+    validateRange(1000, 0, 1000);
+    validateRange(1000, 0, 1);
+    validateRange(1000, 1, 2);
+    validateRange(1000, 5, 6);
+    validateRange(1000, 5, 10);
+    validateRange(1000, 5, 150);
+    validateRange(1000, 5, 27);
+    for (int i = 0; i < 8; i++) {
+      for (int j = 0; j < 8; j++) {
+        validateRange(1000, 10 + i, 27 + j);
+        validateRange(1000, i, j);
+      }
+    }
+  }
+
+  private void validateRange(int length, int start, int count) {
+    String desc = "[" + start + ", "  + (start + count) + ") ";
+    try (BitVector bitVector = new BitVector("bits", allocator)) {
+      bitVector.reset();
+      bitVector.allocateNew(length);
+      bitVector.getMutator().setRangeToOne(start, count);
+      for (int i = 0; i < start; i++) {
+        Assert.assertEquals(desc + i, 0, bitVector.getAccessor().get(i));
+      }
+      for (int i = start; i < start + count; i++) {
+        Assert.assertEquals(desc + i, 1, bitVector.getAccessor().get(i));
+      }
+      for (int i = start + count; i < length; i++) {
+        Assert.assertEquals(desc + i, 0, bitVector.getAccessor().get(i));
+      }
+    }
+  }
+
+  @Test
   public void testReAllocNullableFixedWidthVector() {
     // Create a new value vector for 1024 integers
     try (final NullableFloat4Vector vector = (NullableFloat4Vector) MinorType.FLOAT4.getNewVector(EMPTY_SCHEMA_PATH,
allocator, null)) {


Mime
View raw message