arrow-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From w...@apache.org
Subject [10/12] arrow git commit: ARROW-1111: [JAVA] Make aligning buffers optional, and allow -1 for unknown null count
Date Sun, 11 Jun 2017 15:13:58 GMT
ARROW-1111: [JAVA] Make aligning buffers optional, and allow -1 for unknown null count

Closes #747


Project: http://git-wip-us.apache.org/repos/asf/arrow/repo
Commit: http://git-wip-us.apache.org/repos/asf/arrow/commit/11deee61
Tree: http://git-wip-us.apache.org/repos/asf/arrow/tree/11deee61
Diff: http://git-wip-us.apache.org/repos/asf/arrow/diff/11deee61

Branch: refs/heads/master
Commit: 11deee61aa2fd6782df1f8eefa15938a7676a546
Parents: c6cf124
Author: Steven Phillips <steven@dremio.com>
Authored: Tue Jun 6 15:19:00 2017 -0700
Committer: Wes McKinney <wes.mckinney@twosigma.com>
Committed: Sun Jun 11 11:13:25 2017 -0400

----------------------------------------------------------------------
 .../java/org/apache/arrow/vector/VectorUnloader.java    | 12 ++++++++++--
 .../apache/arrow/vector/schema/ArrowRecordBatch.java    |  8 ++++++--
 2 files changed, 16 insertions(+), 4 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/arrow/blob/11deee61/java/vector/src/main/java/org/apache/arrow/vector/VectorUnloader.java
----------------------------------------------------------------------
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/VectorUnloader.java b/java/vector/src/main/java/org/apache/arrow/vector/VectorUnloader.java
index 8e9ff6d..9965245 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/VectorUnloader.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/VectorUnloader.java
@@ -29,9 +29,17 @@ import org.apache.arrow.vector.schema.ArrowVectorType;
 public class VectorUnloader {
 
   private final VectorSchemaRoot root;
+  private final boolean includeNullCount;
+  private final boolean alignBuffers;
 
   public VectorUnloader(VectorSchemaRoot root) {
+    this(root, true, true);
+  }
+
+  public VectorUnloader(VectorSchemaRoot root, boolean includeNullCount, boolean alignBuffers)
{
     this.root = root;
+    this.includeNullCount = includeNullCount;
+    this.alignBuffers = alignBuffers;
   }
 
   public ArrowRecordBatch getRecordBatch() {
@@ -40,12 +48,12 @@ public class VectorUnloader {
     for (FieldVector vector : root.getFieldVectors()) {
       appendNodes(vector, nodes, buffers);
     }
-    return new ArrowRecordBatch(root.getRowCount(), nodes, buffers);
+    return new ArrowRecordBatch(root.getRowCount(), nodes, buffers, alignBuffers);
   }
 
   private void appendNodes(FieldVector vector, List<ArrowFieldNode> nodes, List<ArrowBuf>
buffers) {
     Accessor accessor = vector.getAccessor();
-    nodes.add(new ArrowFieldNode(accessor.getValueCount(), accessor.getNullCount()));
+    nodes.add(new ArrowFieldNode(accessor.getValueCount(), includeNullCount ? accessor.getNullCount()
: -1));
     List<ArrowBuf> fieldBuffers = vector.getFieldBuffers();
     List<ArrowVectorType> expectedBuffers = vector.getField().getTypeLayout().getVectorTypes();
     if (fieldBuffers.size() != expectedBuffers.size()) {

http://git-wip-us.apache.org/repos/asf/arrow/blob/11deee61/java/vector/src/main/java/org/apache/arrow/vector/schema/ArrowRecordBatch.java
----------------------------------------------------------------------
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/schema/ArrowRecordBatch.java
b/java/vector/src/main/java/org/apache/arrow/vector/schema/ArrowRecordBatch.java
index 6ef514e..6a716fa 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/schema/ArrowRecordBatch.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/schema/ArrowRecordBatch.java
@@ -48,12 +48,16 @@ public class ArrowRecordBatch implements ArrowMessage {
 
   private boolean closed = false;
 
+  public ArrowRecordBatch(int length, List<ArrowFieldNode> nodes, List<ArrowBuf>
buffers) {
+    this(length, nodes, buffers, true);
+  }
+
   /**
    * @param length how many rows in this batch
    * @param nodes field level info
    * @param buffers will be retained until this recordBatch is closed
    */
-  public ArrowRecordBatch(int length, List<ArrowFieldNode> nodes, List<ArrowBuf>
buffers) {
+  public ArrowRecordBatch(int length, List<ArrowFieldNode> nodes, List<ArrowBuf>
buffers, boolean alignBuffers) {
     super();
     this.length = length;
     this.nodes = nodes;
@@ -66,7 +70,7 @@ public class ArrowRecordBatch implements ArrowMessage {
       arrowBuffers.add(new ArrowBuffer(0, offset, size));
       LOGGER.debug(String.format("Buffer in RecordBatch at %d, length: %d", offset, size));
       offset += size;
-      if (offset % 8 != 0) { // align on 8 byte boundaries
+      if (alignBuffers && offset % 8 != 0) { // align on 8 byte boundaries
         offset += 8 - (offset % 8);
       }
     }


Mime
View raw message