drill-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From h.@apache.org
Subject drill git commit: DRILL-3920: Additional tests added to TestValueVectors for serialization and loading. Some light cleanup of a few vector implementations.
Date Tue, 13 Oct 2015 23:40:06 GMT
Repository: drill
Updated Branches:
  refs/heads/master b4d47c56b -> 27364122c


DRILL-3920: Additional tests added to TestValueVectors for serialization and loading.
Some light cleanup of a few vector implementations.

closes #194


Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/27364122
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/27364122
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/27364122

Branch: refs/heads/master
Commit: 27364122ca0a82c6dd5552f292b2711bb2b28cf5
Parents: b4d47c5
Author: Chris Westin <cwestin@yahoo.com>
Authored: Thu Oct 8 18:15:57 2015 -0700
Committer: Hanifi Gunes <hgunes@maprtech.com>
Committed: Tue Oct 13 16:35:22 2015 -0700

----------------------------------------------------------------------
 .../drill/exec/vector/BaseValueVector.java      |  16 +-
 .../apache/drill/exec/vector/ValueVector.java   |  12 +-
 .../exec/vector/complex/AbstractMapVector.java  |  55 +--
 .../drill/exec/vector/complex/MapVector.java    |  24 +-
 .../exec/record/vector/TestValueVector.java     | 352 ++++++++++++++-----
 5 files changed, 329 insertions(+), 130 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/drill/blob/27364122/exec/java-exec/src/main/java/org/apache/drill/exec/vector/BaseValueVector.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/BaseValueVector.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/BaseValueVector.java
index cc287c4..7b3ab41 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/BaseValueVector.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/BaseValueVector.java
@@ -17,10 +17,13 @@
  */
 package org.apache.drill.exec.vector;
 
+import io.netty.buffer.DrillBuf;
+
 import java.util.Iterator;
 
 import com.google.common.base.Preconditions;
 import com.google.common.collect.Iterators;
+
 import org.apache.drill.common.expression.FieldReference;
 import org.apache.drill.exec.memory.BufferAllocator;
 import org.apache.drill.exec.proto.UserBitShared.SerializedField;
@@ -91,10 +94,10 @@ public abstract class BaseValueVector implements ValueVector {
     protected BaseMutator() { }
 
     @Override
-    public void generateTestData(int values) { }
+    public void generateTestData(int values) {}
 
     //TODO: consider making mutator stateless(if possible) on another issue.
-    public void reset() { }
+    public void reset() {}
   }
 
   @Override
@@ -102,5 +105,14 @@ public abstract class BaseValueVector implements ValueVector {
     return Iterators.emptyIterator();
   }
 
+  public static boolean checkBufRefs(final ValueVector vv) {
+    for(final DrillBuf buffer : vv.getBuffers(false)) {
+      if (buffer.refCnt() <= 0) {
+        throw new IllegalStateException("zero refcount");
+      }
+    }
+
+    return true;
+  }
 }
 

http://git-wip-us.apache.org/repos/asf/drill/blob/27364122/exec/java-exec/src/main/java/org/apache/drill/exec/vector/ValueVector.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/ValueVector.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/ValueVector.java
index 691d1b7..8627d1a 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/ValueVector.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/ValueVector.java
@@ -17,10 +17,10 @@
  */
 package org.apache.drill.exec.vector;
 
-import io.netty.buffer.DrillBuf;
-
 import java.io.Closeable;
 
+import io.netty.buffer.DrillBuf;
+
 import org.apache.drill.common.expression.FieldReference;
 import org.apache.drill.exec.memory.OutOfMemoryRuntimeException;
 import org.apache.drill.exec.proto.UserBitShared.SerializedField;
@@ -80,8 +80,9 @@ public interface ValueVector extends Closeable, Iterable<ValueVector>
{
   int getValueCapacity();
 
   /**
-   * Alternative to clear(). Allows use as closeable in try-with-resources.
+   * Alternative to clear(). Allows use as an AutoCloseable in try-with-resources.
    */
+  @Override
   void close();
 
   /**
@@ -155,9 +156,8 @@ public interface ValueVector extends Closeable, Iterable<ValueVector>
{
    * Return the underlying buffers associated with this vector. Note that this doesn't impact
the reference counts for
    * this buffer so it only should be used for in-context access. Also note that this buffer
changes regularly thus
    * external classes shouldn't hold a reference to it (unless they change it).
-   *
-   * @param clear
-   *          Whether to clear vector
+   * @param clear Whether to clear vector before returning; the buffers will still be refcounted;
+   *   but the returned array will be the only reference to them
    *
    * @return The underlying {@link io.netty.buffer.DrillBuf buffers} that is used by this
vector instance.
    */

http://git-wip-us.apache.org/repos/asf/drill/blob/27364122/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/AbstractMapVector.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/AbstractMapVector.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/AbstractMapVector.java
index efba46d..2c93c31 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/AbstractMapVector.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/AbstractMapVector.java
@@ -36,6 +36,7 @@ import java.util.List;
  * Base class for MapVectors. Currently used by RepeatedMapVector and MapVector
  */
 public abstract class AbstractMapVector extends AbstractContainerVector {
+  private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(AbstractContainerVector.class);
 
   // Maintains a map with key as field name and value is the vector itself
   private final MapWithOrdinal<String, ValueVector> vectors =  new MapWithOrdinal<>();
@@ -46,14 +47,24 @@ public abstract class AbstractMapVector extends AbstractContainerVector
{
     // create the hierarchy of the child vectors based on the materialized field
     for (MaterializedField child : clonedField.getChildren()) {
       if (!child.equals(BaseRepeatedValueVector.OFFSETS_FIELD)) {
-        String fieldName = child.getLastName();
-        ValueVector v = TypeHelper.getNewVector(child, allocator, callBack);
+        final String fieldName = child.getLastName();
+        final ValueVector v = TypeHelper.getNewVector(child, allocator, callBack);
         putVector(fieldName, v);
       }
     }
   }
 
   @Override
+  public void close() {
+    for(final ValueVector valueVector : vectors.values()) {
+      valueVector.close();
+    }
+    vectors.clear();
+
+    super.close();
+  }
+
+  @Override
   public boolean allocateNewSafe() {
     /* boolean to keep track if all the memory allocation were successful
      * Used in the case of composite vectors when we need to allocate multiple
@@ -62,8 +73,7 @@ public abstract class AbstractMapVector extends AbstractContainerVector
{
      */
     boolean success = false;
     try {
-
-      for (ValueVector v : vectors.values()) {
+      for (final ValueVector v : vectors.values()) {
         if (!v.allocateNewSafe()) {
           return false;
         }
@@ -105,13 +115,14 @@ public abstract class AbstractMapVector extends AbstractContainerVector
{
    *
    * @return resultant {@link org.apache.drill.exec.vector.ValueVector}
    */
+  @Override
   public <T extends ValueVector> T addOrGet(String name, TypeProtos.MajorType type,
Class<T> clazz) {
     final ValueVector existing = getChild(name);
     boolean create = false;
     if (existing == null) {
       create = true;
     } else if (clazz.isAssignableFrom(existing.getClass())) {
-      return (T)existing;
+      return (T) existing;
     } else if (nullFilled(existing)) {
       existing.clear();
       create = true;
@@ -129,7 +140,7 @@ public abstract class AbstractMapVector extends AbstractContainerVector
{
   }
 
   private boolean nullFilled(ValueVector vector) {
-    for (int r=0; r<vector.getAccessor().getValueCount(); r++) {
+    for (int r = 0; r < vector.getAccessor().getValueCount(); r++) {
       if (!vector.getAccessor().isNull(r)) {
         return false;
       }
@@ -148,8 +159,9 @@ public abstract class AbstractMapVector extends AbstractContainerVector
{
    * Returns a {@link org.apache.drill.exec.vector.ValueVector} instance of subtype of <T>
corresponding to the given
    * field name if exists or null.
    */
+  @Override
   public <T extends ValueVector> T getChild(String name, Class<T> clazz) {
-    ValueVector v = vectors.get(name.toLowerCase());
+    final ValueVector v = vectors.get(name.toLowerCase());
     if (v == null) {
       return null;
     }
@@ -172,7 +184,7 @@ public abstract class AbstractMapVector extends AbstractContainerVector
{
    * @param vector  vector to be inserted
    */
   protected void putVector(String name, ValueVector vector) {
-    ValueVector old = vectors.put(
+    final ValueVector old = vectors.put(
         Preconditions.checkNotNull(name, "field name cannot be null").toLowerCase(),
         Preconditions.checkNotNull(vector, "vector cannot be null")
     );
@@ -192,6 +204,7 @@ public abstract class AbstractMapVector extends AbstractContainerVector
{
   /**
    * Returns the number of underlying child vectors.
    */
+  @Override
   public int size() {
     return vectors.size();
   }
@@ -205,8 +218,8 @@ public abstract class AbstractMapVector extends AbstractContainerVector
{
    * Returns a list of scalar child vectors recursing the entire vector hierarchy.
    */
   public List<ValueVector> getPrimitiveVectors() {
-    List<ValueVector> primitiveVectors = Lists.newArrayList();
-    for (ValueVector v : vectors.values()) {
+    final List<ValueVector> primitiveVectors = Lists.newArrayList();
+    for (final ValueVector v : vectors.values()) {
       if (v instanceof AbstractMapVector) {
         AbstractMapVector mapVector = (AbstractMapVector) v;
         primitiveVectors.addAll(mapVector.getPrimitiveVectors());
@@ -220,6 +233,7 @@ public abstract class AbstractMapVector extends AbstractContainerVector
{
   /**
    * Returns a vector with its corresponding ordinal mapping if field exists or null.
    */
+  @Override
   public VectorWithOrdinal getChildVectorWithOrdinal(String name) {
     final int ordinal = vectors.getOrdinal(name.toLowerCase());
     if (ordinal < 0) {
@@ -231,13 +245,13 @@ public abstract class AbstractMapVector extends AbstractContainerVector
{
 
   @Override
   public DrillBuf[] getBuffers(boolean clear) {
-    List<DrillBuf> buffers = Lists.newArrayList();
+    final List<DrillBuf> buffers = Lists.newArrayList();
 
-    for (ValueVector vector : vectors.values()) {
-      for (DrillBuf buf : vector.getBuffers(false)) {
+    for (final ValueVector vector : vectors.values()) {
+      for (final DrillBuf buf : vector.getBuffers(false)) {
         buffers.add(buf);
         if (clear) {
-          buf.retain();
+          buf.retain(1);
         }
       }
       if (clear) {
@@ -252,20 +266,11 @@ public abstract class AbstractMapVector extends AbstractContainerVector
{
   public int getBufferSize() {
     int actualBufSize = 0 ;
 
-    for (ValueVector v : vectors.values()) {
-      for (DrillBuf buf : v.getBuffers(false)) {
+    for (final ValueVector v : vectors.values()) {
+      for (final DrillBuf buf : v.getBuffers(false)) {
         actualBufSize += buf.writerIndex();
       }
     }
     return actualBufSize;
   }
-
-  @Override
-  public void close() {
-   for(final ValueVector valueVector : vectors.values()) {
-     valueVector.close();
-   }
-
-   super.close();
-  }
 }

http://git-wip-us.apache.org/repos/asf/drill/blob/27364122/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/MapVector.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/MapVector.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/MapVector.java
index 66f7962..8b4b858 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/MapVector.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/MapVector.java
@@ -22,6 +22,7 @@ import com.google.common.primitives.Ints;
 
 import io.netty.buffer.DrillBuf;
 
+import java.util.Collection;
 import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
@@ -50,7 +51,7 @@ import org.apache.drill.exec.vector.complex.reader.FieldReader;
 import com.google.common.base.Preconditions;
 
 public class MapVector extends AbstractMapVector {
-  static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(MapVector.class);
+  //private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(MapVector.class);
 
   public final static MajorType TYPE = Types.required(MinorType.MAP);
 
@@ -112,7 +113,7 @@ public class MapVector extends AbstractMapVector {
       return 0;
     }
     long buffer = 0;
-    for (ValueVector v : (Iterable<ValueVector>)this) {
+    for (final ValueVector v : (Iterable<ValueVector>)this) {
       buffer += v.getBufferSize();
     }
 
@@ -149,7 +150,7 @@ public class MapVector extends AbstractMapVector {
 
   @Override
   public TransferPair makeTransferPair(ValueVector to) {
-    return new MapTransferPair(this, (MapVector)to);
+    return new MapTransferPair(this, (MapVector) to);
   }
 
   @Override
@@ -194,7 +195,7 @@ public class MapVector extends AbstractMapVector {
         // (This is similar to what happens in ScanBatch where the children cannot be added
till they are
         // read). To take care of this, we ensure that the hashCode of the MaterializedField
does not
         // include the hashCode of the children but is based only on MaterializedField$key.
-        ValueVector newVector = to.addOrGet(child, vector.getField().getType(), vector.getClass());
+        final ValueVector newVector = to.addOrGet(child, vector.getField().getType(), vector.getClass());
         if (allocate && to.size() != preSize) {
           newVector.allocateNew();
         }
@@ -202,10 +203,9 @@ public class MapVector extends AbstractMapVector {
       }
     }
 
-
     @Override
     public void transfer() {
-      for (TransferPair p : pairs) {
+      for (final TransferPair p : pairs) {
         p.transfer();
       }
       to.valueCount = from.valueCount;
@@ -231,7 +231,6 @@ public class MapVector extends AbstractMapVector {
       }
       to.getMutator().setValueCount(length);
     }
-
   }
 
   @Override
@@ -329,7 +328,6 @@ public class MapVector extends AbstractMapVector {
     public int getValueCount() {
       return valueCount;
     }
-
   }
 
   public ValueVector getVectorById(int id) {
@@ -340,7 +338,7 @@ public class MapVector extends AbstractMapVector {
 
     @Override
     public void setValueCount(int valueCount) {
-      for (ValueVector v : getChildren()) {
+      for (final ValueVector v : getChildren()) {
         v.getMutator().setValueCount(valueCount);
       }
       MapVector.this.valueCount = valueCount;
@@ -355,17 +353,19 @@ public class MapVector extends AbstractMapVector {
 
   @Override
   public void clear() {
-    valueCount = 0;
-    for (ValueVector v : getChildren()) {
+    for (final ValueVector v : getChildren()) {
       v.clear();
     }
+    valueCount = 0;
   }
 
   @Override
   public void close() {
-    for (final ValueVector v : getChildren()) {
+    final Collection<ValueVector> vectors = getChildren();
+    for (final ValueVector v : vectors) {
       v.close();
     }
+    vectors.clear();
     valueCount = 0;
 
     super.close();

http://git-wip-us.apache.org/repos/asf/drill/blob/27364122/exec/java-exec/src/test/java/org/apache/drill/exec/record/vector/TestValueVector.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/record/vector/TestValueVector.java
b/exec/java-exec/src/test/java/org/apache/drill/exec/record/vector/TestValueVector.java
index af4f2f0..0a9b470 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/record/vector/TestValueVector.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/record/vector/TestValueVector.java
@@ -20,12 +20,13 @@ package org.apache.drill.exec.record.vector;
 import static org.junit.Assert.assertArrayEquals;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertTrue;
+import io.netty.buffer.DrillBuf;
 
 import java.nio.charset.Charset;
 
 import com.google.common.base.Preconditions;
 import com.google.common.collect.ImmutableMap;
-import io.netty.buffer.DrillBuf;
+
 import org.apache.drill.common.AutoCloseables;
 import org.apache.drill.common.config.DrillConfig;
 import org.apache.drill.common.expression.SchemaPath;
@@ -41,6 +42,7 @@ import org.apache.drill.exec.expr.holders.NullableUInt4Holder;
 import org.apache.drill.exec.expr.holders.NullableVar16CharHolder;
 import org.apache.drill.exec.expr.holders.NullableVarCharHolder;
 import org.apache.drill.exec.expr.holders.RepeatedFloat4Holder;
+import org.apache.drill.exec.expr.holders.RepeatedIntHolder;
 import org.apache.drill.exec.expr.holders.RepeatedVarBinaryHolder;
 import org.apache.drill.exec.expr.holders.UInt1Holder;
 import org.apache.drill.exec.expr.holders.UInt4Holder;
@@ -54,6 +56,7 @@ import org.apache.drill.exec.vector.BitVector;
 import org.apache.drill.exec.vector.NullableFloat4Vector;
 import org.apache.drill.exec.vector.NullableUInt4Vector;
 import org.apache.drill.exec.vector.NullableVarCharVector;
+import org.apache.drill.exec.vector.RepeatedIntVector;
 import org.apache.drill.exec.vector.UInt4Vector;
 import org.apache.drill.exec.vector.ValueVector;
 import org.apache.drill.exec.vector.VarCharVector;
@@ -65,11 +68,9 @@ import org.junit.Before;
 import org.junit.Test;
 
 public class TestValueVector extends ExecTest {
-  private final static SchemaPath EMPTY_SCHEMA_PATH = SchemaPath.getSimplePath("");
+  //private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TestValueVector.class);
 
-  private final static byte[] STR1 = new String("AAAAA1").getBytes(Charset.forName("UTF-8"));
-  private final static byte[] STR2 = new String("BBBBBBBBB2").getBytes(Charset.forName("UTF-8"));
-  private final static byte[] STR3 = new String("CCCC3").getBytes(Charset.forName("UTF-8"));
+  private final static SchemaPath EMPTY_SCHEMA_PATH = SchemaPath.getSimplePath("");
 
   private DrillConfig drillConfig;
   private BufferAllocator allocator;
@@ -80,12 +81,16 @@ public class TestValueVector extends ExecTest {
     allocator = RootAllocatorFactory.newRoot(drillConfig);
   }
 
+  private final static Charset utf8Charset = Charset.forName("UTF-8");
+  private final static byte[] STR1 = new String("AAAAA1").getBytes(utf8Charset);
+  private final static byte[] STR2 = new String("BBBBBBBBB2").getBytes(utf8Charset);
+  private final static byte[] STR3 = new String("CCCC3").getBytes(utf8Charset);
+
   @After
-  public void terminate() {
+  public void terminate() throws Exception {
     allocator.close();
   }
 
-
   @Test(expected = OversizedAllocationException.class)
   public void testFixedVectorReallocation() {
     final MaterializedField field = MaterializedField.create(EMPTY_SCHEMA_PATH, UInt4Holder.TYPE);
@@ -172,11 +177,11 @@ public class TestValueVector extends ExecTest {
 
   @Test
   public void testFixedType() {
-    MaterializedField field = MaterializedField.create(EMPTY_SCHEMA_PATH, UInt4Holder.TYPE);
+    final MaterializedField field = MaterializedField.create(EMPTY_SCHEMA_PATH, UInt4Holder.TYPE);
 
-    // Create a new value vector for 1024 integers
-    try (UInt4Vector vector = new UInt4Vector(field, allocator)) {
-      UInt4Vector.Mutator m = vector.getMutator();
+    // Create a new value vector for 1024 integers.
+    try (final UInt4Vector vector = new UInt4Vector(field, allocator)) {
+      final UInt4Vector.Mutator m = vector.getMutator();
       vector.allocateNew(1024);
 
       // Put and set a few values
@@ -185,33 +190,36 @@ public class TestValueVector extends ExecTest {
       m.setSafe(100, 102);
       m.setSafe(1022, 103);
       m.setSafe(1023, 104);
-      assertEquals(100, vector.getAccessor().get(0));
-      assertEquals(101, vector.getAccessor().get(1));
-      assertEquals(102, vector.getAccessor().get(100));
-      assertEquals(103, vector.getAccessor().get(1022));
-      assertEquals(104, vector.getAccessor().get(1023));
+
+      final UInt4Vector.Accessor accessor = vector.getAccessor();
+      assertEquals(100, accessor.get(0));
+      assertEquals(101, accessor.get(1));
+      assertEquals(102, accessor.get(100));
+      assertEquals(103, accessor.get(1022));
+      assertEquals(104, accessor.get(1023));
     }
   }
 
   @Test
   public void testNullableVarLen2() {
-    MaterializedField field = MaterializedField.create(EMPTY_SCHEMA_PATH, NullableVarCharHolder.TYPE);
+    final MaterializedField field = MaterializedField.create(EMPTY_SCHEMA_PATH, NullableVarCharHolder.TYPE);
 
-    // Create a new value vector for 1024 integers
-    try (NullableVarCharVector vector = new NullableVarCharVector(field, allocator)) {
-      NullableVarCharVector.Mutator m = vector.getMutator();
+    // Create a new value vector for 1024 integers.
+    try (final NullableVarCharVector vector = new NullableVarCharVector(field, allocator))
{
+      final NullableVarCharVector.Mutator m = vector.getMutator();
       vector.allocateNew(1024 * 10, 1024);
 
       m.set(0, STR1);
       m.set(1, STR2);
       m.set(2, STR3);
 
-      // Check the sample strings
-      assertArrayEquals(STR1, vector.getAccessor().get(0));
-      assertArrayEquals(STR2, vector.getAccessor().get(1));
-      assertArrayEquals(STR3, vector.getAccessor().get(2));
+      // Check the sample strings.
+      final NullableVarCharVector.Accessor accessor = vector.getAccessor();
+      assertArrayEquals(STR1, accessor.get(0));
+      assertArrayEquals(STR2, accessor.get(1));
+      assertArrayEquals(STR3, accessor.get(2));
 
-      // Ensure null value throws
+      // Ensure null value throws.
       boolean b = false;
       try {
         vector.getAccessor().get(3);
@@ -223,14 +231,185 @@ public class TestValueVector extends ExecTest {
     }
   }
 
+  private static DrillBuf combineBuffers(final BufferAllocator allocator, final DrillBuf[]
buffers) {
+    // find the total size we'll need
+    int size = 0;
+    for(final DrillBuf buffer : buffers) {
+      size += buffer.readableBytes();
+    }
+
+    // create the new buffer
+    final DrillBuf newBuf = allocator.buffer(size);
+    final DrillBuf writeBuf = newBuf;
+    for(final DrillBuf buffer : buffers) {
+      final DrillBuf readBuf = (DrillBuf) buffer.slice();
+      final int nBytes = readBuf.readableBytes();
+      final byte[] bytes = new byte[nBytes];
+      readBuf.readBytes(bytes);
+      writeBuf.writeBytes(bytes);
+    }
+
+    return newBuf;
+  }
+
+  @Test
+  public void testRepeatedIntVector() {
+    final MaterializedField field = MaterializedField.create(EMPTY_SCHEMA_PATH, RepeatedIntHolder.TYPE);
+
+    // Create a new value vector.
+    final RepeatedIntVector vector1 = new RepeatedIntVector(field, allocator);
+
+    // Populate the vector.
+    final int[] values = {2, 3, 5, 7, 11, 13, 17, 19, 23, 27}; // some tricksy primes
+    final int nRecords = 7;
+    final int nElements = values.length;
+    vector1.allocateNew(nRecords, nRecords * nElements);
+    final RepeatedIntVector.Mutator mutator = vector1.getMutator();
+    for(int recordIndex = 0; recordIndex < nRecords; ++recordIndex) {
+      mutator.startNewValue(recordIndex);
+      for(int elementIndex = 0; elementIndex < nElements; ++elementIndex) {
+        mutator.add(recordIndex, recordIndex * values[elementIndex]);
+      }
+    }
+    mutator.setValueCount(nRecords);
+
+    // Verify the contents.
+    final RepeatedIntVector.Accessor accessor1 = vector1.getAccessor();
+    assertEquals(nRecords, accessor1.getValueCount());
+    for(int recordIndex = 0; recordIndex < nRecords; ++recordIndex) {
+      for(int elementIndex = 0; elementIndex < nElements; ++elementIndex) {
+        final int value = accessor1.get(recordIndex, elementIndex);
+        assertEquals(recordIndex * values[elementIndex], value);
+      }
+    }
+
+/* TODO(cwestin)
+the interface to load has changed
+    // Serialize, reify, and verify.
+    final DrillBuf[] buffers1 = vector1.getBuffers(false);
+    final DrillBuf buffer1 = combineBuffers(allocator, buffers1);
+    final RepeatedIntVector vector2 = new RepeatedIntVector(field, allocator);
+    vector2.load(nRecords, nRecords * nElements, buffer1);
+
+    final RepeatedIntVector.Accessor accessor2 = vector2.getAccessor();
+    for(int recordIndex = 0; recordIndex < nRecords; ++recordIndex) {
+      for(int elementIndex = 0; elementIndex < nElements; ++elementIndex) {
+        final int value = accessor2.get(recordIndex, elementIndex);
+        assertEquals(accessor1.get(recordIndex,  elementIndex), value);
+      }
+    }
+*/
+
+    vector1.close();
+/* TODO(cwestin)
+    vector2.close();
+    buffer1.release();
+*/
+  }
+
+  @Test
+  public void testVarCharVectorLoad() {
+    final MaterializedField field = MaterializedField.create(EMPTY_SCHEMA_PATH, VarCharHolder.TYPE);
+
+    // Create a new value vector for 1024 variable length strings.
+    final VarCharVector vector1 = new VarCharVector(field, allocator);
+    final VarCharVector.Mutator mutator = vector1.getMutator();
+    vector1.allocateNew(1024 * 10, 1024);
+
+    // Populate the vector.
+    final StringBuilder stringBuilder = new StringBuilder();
+    final int valueCount = 10;
+    for(int i = 0; i < valueCount; ++i) {
+      stringBuilder.append('x');
+      mutator.setSafe(i, stringBuilder.toString().getBytes(utf8Charset));
+    }
+    mutator.setValueCount(valueCount);
+    assertEquals(valueCount, vector1.getAccessor().getValueCount());
+
+    // Combine the backing buffers so we can load them into a new vector.
+    final DrillBuf[] buffers1 = vector1.getBuffers(false);
+    final DrillBuf buffer1 = combineBuffers(allocator, buffers1);
+    final VarCharVector vector2 = new VarCharVector(field, allocator);
+    vector2.load(vector1.getMetadata(), buffer1);
+
+    // Check the contents of the new vector.
+    final VarCharVector.Accessor accessor = vector2.getAccessor();
+    stringBuilder.setLength(0);
+    for(int i = 0; i < valueCount; ++i) {
+      stringBuilder.append('x');
+      final Object object = accessor.getObject(i);
+      assertEquals(stringBuilder.toString(), object.toString());
+    }
+
+    vector1.close();
+    vector2.close();
+    buffer1.release();
+  }
+
+  @Test
+  public void testNullableVarCharVectorLoad() {
+    final MaterializedField field = MaterializedField.create(EMPTY_SCHEMA_PATH, NullableVarCharHolder.TYPE);
+
+    // Create a new value vector for 1024 nullable variable length strings.
+    final NullableVarCharVector vector1 = new NullableVarCharVector(field, allocator);
+    final NullableVarCharVector.Mutator mutator = vector1.getMutator();
+    vector1.allocateNew(1024 * 10, 1024);
+
+    // Populate the vector.
+    final StringBuilder stringBuilder = new StringBuilder();
+    final int valueCount = 10;
+    for(int i = 0; i < valueCount; ++i) {
+      stringBuilder.append('x');
+      mutator.set(i, stringBuilder.toString().getBytes(utf8Charset));
+    }
+
+    // Check the contents.
+    final NullableVarCharVector.Accessor accessor1 = vector1.getAccessor();
+    stringBuilder.setLength(0);
+    for(int i = 0; i < valueCount; ++i) {
+      stringBuilder.append('x');
+      final Object object = accessor1.getObject(i);
+      assertEquals(stringBuilder.toString(), object.toString());
+    }
+
+    mutator.setValueCount(valueCount);
+    assertEquals(valueCount, vector1.getAccessor().getValueCount());
+
+    // Still ok after setting value count?
+    stringBuilder.setLength(0);
+    for(int i = 0; i < valueCount; ++i) {
+      stringBuilder.append('x');
+      final Object object = accessor1.getObject(i);
+      assertEquals(stringBuilder.toString(), object.toString());
+    }
+
+    // Combine into a single buffer so we can load it into a new vector.
+    final DrillBuf[] buffers1 = vector1.getBuffers(false);
+    final DrillBuf buffer1 = combineBuffers(allocator, buffers1);
+    final NullableVarCharVector vector2 = new NullableVarCharVector(field, allocator);
+    vector2.load(vector1.getMetadata(), buffer1);
+
+    // Check the vector's contents.
+    final NullableVarCharVector.Accessor accessor2 = vector2.getAccessor();
+    stringBuilder.setLength(0);
+    for(int i = 0; i < valueCount; ++i) {
+      stringBuilder.append('x');
+      final Object object = accessor2.getObject(i);
+      assertEquals(stringBuilder.toString(), object.toString());
+    }
+
+    vector1.close();
+    vector2.close();
+    buffer1.release();
+  }
 
   @Test
   public void testNullableFixedType() {
-    MaterializedField field = MaterializedField.create(EMPTY_SCHEMA_PATH, NullableUInt4Holder.TYPE);
+    final MaterializedField field = MaterializedField.create(EMPTY_SCHEMA_PATH, NullableUInt4Holder.TYPE);
 
-    // Create a new value vector for 1024 integers
-    try (NullableUInt4Vector vector = new NullableUInt4Vector(field, allocator)) {
-      NullableUInt4Vector.Mutator m = vector.getMutator();
+    // Create a new value vector for 1024 integers.
+    try (final NullableUInt4Vector vector = new NullableUInt4Vector(field, allocator)) {
+      final NullableUInt4Vector.Mutator m = vector.getMutator();
       vector.allocateNew(1024);
 
       // Put and set a few values
@@ -239,17 +418,19 @@ public class TestValueVector extends ExecTest {
       m.set(100, 102);
       m.set(1022, 103);
       m.set(1023, 104);
-      assertEquals(100, vector.getAccessor().get(0));
-      assertEquals(101, vector.getAccessor().get(1));
-      assertEquals(102, vector.getAccessor().get(100));
-      assertEquals(103, vector.getAccessor().get(1022));
-      assertEquals(104, vector.getAccessor().get(1023));
+
+      final NullableUInt4Vector.Accessor accessor = vector.getAccessor();
+      assertEquals(100, accessor.get(0));
+      assertEquals(101, accessor.get(1));
+      assertEquals(102, accessor.get(100));
+      assertEquals(103, accessor.get(1022));
+      assertEquals(104, accessor.get(1023));
 
       // Ensure null values throw
       {
         boolean b = false;
         try {
-          vector.getAccessor().get(3);
+          accessor.get(3);
         } catch (IllegalStateException e) {
           b = true;
         } finally {
@@ -257,12 +438,11 @@ public class TestValueVector extends ExecTest {
         }
       }
 
-
       vector.allocateNew(2048);
       {
         boolean b = false;
         try {
-          vector.getAccessor().get(0);
+          accessor.get(0);
         } catch (IllegalStateException e) {
           b = true;
         } finally {
@@ -275,14 +455,13 @@ public class TestValueVector extends ExecTest {
       m.set(100, 102);
       m.set(1022, 103);
       m.set(1023, 104);
-      assertEquals(100, vector.getAccessor().get(0));
-      assertEquals(101, vector.getAccessor().get(1));
-      assertEquals(102, vector.getAccessor().get(100));
-      assertEquals(103, vector.getAccessor().get(1022));
-      assertEquals(104, vector.getAccessor().get(1023));
-
-      // Ensure null values throw
+      assertEquals(100, accessor.get(0));
+      assertEquals(101, accessor.get(1));
+      assertEquals(102, accessor.get(100));
+      assertEquals(103, accessor.get(1022));
+      assertEquals(104, accessor.get(1023));
 
+      // Ensure null values throw.
       {
         boolean b = false;
         try {
@@ -294,31 +473,32 @@ public class TestValueVector extends ExecTest {
         }
       }
     }
-
   }
 
   @Test
   public void testNullableFloat() {
-    MaterializedField field = MaterializedField.create(EMPTY_SCHEMA_PATH, NullableFloat4Holder.TYPE);
+    final MaterializedField field = MaterializedField.create(EMPTY_SCHEMA_PATH, NullableFloat4Holder.TYPE);
 
     // Create a new value vector for 1024 integers
-    try (NullableFloat4Vector vector = (NullableFloat4Vector) TypeHelper.getNewVector(field,
allocator)) {
-      NullableFloat4Vector.Mutator m = vector.getMutator();
+    try (final NullableFloat4Vector vector = (NullableFloat4Vector) TypeHelper.getNewVector(field,
allocator)) {
+      final NullableFloat4Vector.Mutator m = vector.getMutator();
       vector.allocateNew(1024);
 
-      // Put and set a few values
+      // Put and set a few values.
       m.set(0, 100.1f);
       m.set(1, 101.2f);
       m.set(100, 102.3f);
       m.set(1022, 103.4f);
       m.set(1023, 104.5f);
-      assertEquals(100.1f, vector.getAccessor().get(0), 0);
-      assertEquals(101.2f, vector.getAccessor().get(1), 0);
-      assertEquals(102.3f, vector.getAccessor().get(100), 0);
-      assertEquals(103.4f, vector.getAccessor().get(1022), 0);
-      assertEquals(104.5f, vector.getAccessor().get(1023), 0);
 
-      // Ensure null values throw
+      final NullableFloat4Vector.Accessor accessor = vector.getAccessor();
+      assertEquals(100.1f, accessor.get(0), 0);
+      assertEquals(101.2f, accessor.get(1), 0);
+      assertEquals(102.3f, accessor.get(100), 0);
+      assertEquals(103.4f, accessor.get(1022), 0);
+      assertEquals(104.5f, accessor.get(1023), 0);
+
+      // Ensure null values throw.
       {
         boolean b = false;
         try {
@@ -334,7 +514,7 @@ public class TestValueVector extends ExecTest {
       {
         boolean b = false;
         try {
-          vector.getAccessor().get(0);
+          accessor.get(0);
         } catch (IllegalStateException e) {
           b = true;
         } finally {
@@ -346,11 +526,11 @@ public class TestValueVector extends ExecTest {
 
   @Test
   public void testBitVector() {
-    MaterializedField field = MaterializedField.create(EMPTY_SCHEMA_PATH, BitHolder.TYPE);
+    final MaterializedField field = MaterializedField.create(EMPTY_SCHEMA_PATH, BitHolder.TYPE);
 
     // Create a new value vector for 1024 integers
-    try (BitVector vector = new BitVector(field, allocator)) {
-      BitVector.Mutator m = vector.getMutator();
+    try (final BitVector vector = new BitVector(field, allocator)) {
+      final BitVector.Mutator m = vector.getMutator();
       vector.allocateNew(1024);
 
       // Put and set a few values
@@ -358,38 +538,39 @@ public class TestValueVector extends ExecTest {
       m.set(1, 0);
       m.set(100, 0);
       m.set(1022, 1);
-      assertEquals(1, vector.getAccessor().get(0));
-      assertEquals(0, vector.getAccessor().get(1));
-      assertEquals(0, vector.getAccessor().get(100));
-      assertEquals(1, vector.getAccessor().get(1022));
+
+      final BitVector.Accessor accessor = vector.getAccessor();
+      assertEquals(1, accessor.get(0));
+      assertEquals(0, accessor.get(1));
+      assertEquals(0, accessor.get(100));
+      assertEquals(1, accessor.get(1022));
 
       // test setting the same value twice
       m.set(0, 1);
       m.set(0, 1);
       m.set(1, 0);
       m.set(1, 0);
-      assertEquals(1, vector.getAccessor().get(0));
-      assertEquals(0, vector.getAccessor().get(1));
+      assertEquals(1, accessor.get(0));
+      assertEquals(0, accessor.get(1));
 
       // test toggling the values
       m.set(0, 0);
       m.set(1, 1);
-      assertEquals(0, vector.getAccessor().get(0));
-      assertEquals(1, vector.getAccessor().get(1));
+      assertEquals(0, accessor.get(0));
+      assertEquals(1, accessor.get(1));
 
       // Ensure unallocated space returns 0
-      assertEquals(0, vector.getAccessor().get(3));
+      assertEquals(0, accessor.get(3));
     }
   }
 
-
   @Test
-  public void testReAllocNullableFixedWidthVector() throws Exception {
-    MaterializedField field = MaterializedField.create(EMPTY_SCHEMA_PATH, NullableFloat4Holder.TYPE);
+  public void testReAllocNullableFixedWidthVector() {
+    final MaterializedField field = MaterializedField.create(EMPTY_SCHEMA_PATH, NullableFloat4Holder.TYPE);
 
     // Create a new value vector for 1024 integers
-    try (NullableFloat4Vector vector = (NullableFloat4Vector) TypeHelper.getNewVector(field,
allocator)) {
-      NullableFloat4Vector.Mutator m = vector.getMutator();
+    try (final NullableFloat4Vector vector = (NullableFloat4Vector) TypeHelper.getNewVector(field,
allocator)) {
+      final NullableFloat4Vector.Mutator m = vector.getMutator();
       vector.allocateNew(1024);
 
       assertEquals(1024, vector.getValueCapacity());
@@ -405,11 +586,11 @@ public class TestValueVector extends ExecTest {
       // Check valueCapacity is more than initial allocation
       assertEquals(1024 * 2, vector.getValueCapacity());
 
-      assertEquals(100.1f, vector.getAccessor().get(0), 0);
-      assertEquals(102.3f, vector.getAccessor().get(100), 0);
-      assertEquals(104.5f, vector.getAccessor().get(1023), 0);
-      assertEquals(105.5f, vector.getAccessor().get(2000), 0);
-
+      final NullableFloat4Vector.Accessor accessor = vector.getAccessor();
+      assertEquals(100.1f, accessor.get(0), 0);
+      assertEquals(102.3f, accessor.get(100), 0);
+      assertEquals(104.5f, accessor.get(1023), 0);
+      assertEquals(105.5f, accessor.get(2000), 0);
 
       // Set the valueCount to be more than valueCapacity of current allocation. This is
possible for NullableValueVectors
       // as we don't call setSafe for null values, but we do call setValueCount when all
values are inserted into the
@@ -419,12 +600,12 @@ public class TestValueVector extends ExecTest {
   }
 
   @Test
-  public void testReAllocNullableVariableWidthVector() throws Exception {
-    MaterializedField field = MaterializedField.create(EMPTY_SCHEMA_PATH, NullableVarCharHolder.TYPE);
+  public void testReAllocNullableVariableWidthVector() {
+    final MaterializedField field = MaterializedField.create(EMPTY_SCHEMA_PATH, NullableVarCharHolder.TYPE);
 
     // Create a new value vector for 1024 integers
-    try (NullableVarCharVector vector = (NullableVarCharVector) TypeHelper.getNewVector(field,
allocator)) {
-      NullableVarCharVector.Mutator m = vector.getMutator();
+    try (final NullableVarCharVector vector = (NullableVarCharVector) TypeHelper.getNewVector(field,
allocator)) {
+      final NullableVarCharVector.Mutator m = vector.getMutator();
       vector.allocateNew();
 
       int initialCapacity = vector.getValueCapacity();
@@ -439,9 +620,10 @@ public class TestValueVector extends ExecTest {
       // Check valueCapacity is more than initial allocation
       assertEquals((initialCapacity + 1) * 2 - 1, vector.getValueCapacity());
 
-      assertArrayEquals(STR1, vector.getAccessor().get(0));
-      assertArrayEquals(STR2, vector.getAccessor().get(initialCapacity - 1));
-      assertArrayEquals(STR3, vector.getAccessor().get(initialCapacity + 200));
+      final NullableVarCharVector.Accessor accessor = vector.getAccessor();
+      assertArrayEquals(STR1, accessor.get(0));
+      assertArrayEquals(STR2, accessor.get(initialCapacity - 1));
+      assertArrayEquals(STR3, accessor.get(initialCapacity + 200));
 
       // Set the valueCount to be more than valueCapacity of current allocation. This is
possible for NullableValueVectors
       // as we don't call setSafe for null values, but we do call setValueCount when the
current batch is processed.


Mime
View raw message