drill-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From par...@apache.org
Subject [2/2] drill git commit: DRILL-3313: Eliminate redundant #load methods and unit-test loading & exporting of vectors
Date Thu, 27 Aug 2015 23:20:21 GMT
DRILL-3313: Eliminate redundant #load methods and unit-test loading & exporting of vectors


Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/77e2b89f
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/77e2b89f
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/77e2b89f

Branch: refs/heads/master
Commit: 77e2b89fee7a823606070b6245095a8f3fa6834c
Parents: 21fbb28
Author: Hanifi Gunes <hgunes@maprtech.com>
Authored: Tue Jun 23 12:05:45 2015 -0700
Committer: Parth Chandra <parthc@apache.org>
Committed: Thu Aug 27 16:04:49 2015 -0700

----------------------------------------------------------------------
 .../codegen/templates/FixedValueVectors.java    | 147 ++++++++-------
 .../codegen/templates/NullableValueVectors.java | 181 ++++++-------------
 .../templates/VariableLengthVectors.java        | 117 ++++++------
 .../org/apache/drill/exec/vector/BitVector.java |  23 +--
 .../drill/exec/vector/FixedWidthVector.java     |  18 +-
 .../drill/exec/vector/NullableVector.java       |   2 +-
 .../drill/exec/vector/VariableWidthVector.java  |  27 +--
 .../vector/complex/BaseRepeatedValueVector.java |   2 +-
 .../drill/exec/vector/complex/MapVector.java    |   5 +-
 .../exec/vector/complex/RepeatedMapVector.java  |   2 +-
 .../exec/record/vector/TestValueVector.java     | 119 ++++++++++++
 11 files changed, 329 insertions(+), 314 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/drill/blob/77e2b89f/exec/java-exec/src/main/codegen/templates/FixedValueVectors.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/codegen/templates/FixedValueVectors.java b/exec/java-exec/src/main/codegen/templates/FixedValueVectors.java
index 7bb76fa..0189510 100644
--- a/exec/java-exec/src/main/codegen/templates/FixedValueVectors.java
+++ b/exec/java-exec/src/main/codegen/templates/FixedValueVectors.java
@@ -40,7 +40,7 @@ package org.apache.drill.exec.vector;
  *
  * NB: this class is automatically generated from ${.template_name} and ValueVectorTypes.tdd using FreeMarker.
  */
-public final class ${minor.class}Vector extends BaseDataValueVector implements FixedWidthVector {
+public final class ${minor.class}Vector extends BaseDataValueVector implements FixedWidthVector{
   private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(${minor.class}Vector.class);
 
   private final FieldReader reader = new ${minor.class}ReaderImpl(${minor.class}Vector.this);
@@ -55,22 +55,22 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F
   }
 
   @Override
-  public FieldReader getReader() {
+  public FieldReader getReader(){
     return reader;
   }
 
   @Override
-  public int getValueCapacity() {
-    return (int) (data.capacity() * 1.0 / ${type.width});
+  public int getValueCapacity(){
+    return (int) (data.capacity() *1.0 / ${type.width});
   }
 
   @Override
-  public Accessor getAccessor() {
+  public Accessor getAccessor(){
     return accessor;
   }
 
   @Override
-  public Mutator getMutator() {
+  public Mutator getMutator(){
     return mutator;
   }
 
@@ -85,7 +85,7 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F
 
   @Override
   public void allocateNew() {
-    if(!allocateNewSafe()) {
+    if(!allocateNewSafe()){
       throw new OutOfMemoryRuntimeException("Failure while allocating buffer.");
     }
   }
@@ -101,7 +101,7 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F
       allocationMonitor = 0;
     }
 
-    try {
+    try{
       allocateBytes(curAllocationSize);
     } catch (DrillRuntimeException ex) {
       return false;
@@ -147,14 +147,14 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F
     }
 
     logger.debug("Reallocating vector [{}]. # of bytes: [{}] -> [{}]", field, allocationSizeInBytes, newAllocationSize);
-    final DrillBuf newBuf = allocator.buffer((int) newAllocationSize);
+    final DrillBuf newBuf = allocator.buffer((int)newAllocationSize);
     newBuf.setBytes(0, data, 0, data.capacity());
     final int halfNewCapacity = newBuf.capacity() / 2;
     newBuf.setZero(halfNewCapacity, halfNewCapacity);
     newBuf.writerIndex(data.writerIndex());
     data.release(1);
     data = newBuf;
-    allocationSizeInBytes = (int) newAllocationSize;
+    allocationSizeInBytes = (int)newAllocationSize;
   }
 
   /**
@@ -166,27 +166,22 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F
   }
 
   @Override
-  public int load(int valueCount, DrillBuf buf) {
+  public void load(SerializedField metadata, DrillBuf buffer) {
+    final int actualLength = metadata.getBufferLength();
+    final int valueCount = metadata.getValueCount();
+    final int expectedLength = valueCount * ${type.width};
+    assert actualLength == expectedLength : String.format("Expected to load %d bytes but actually loaded %d bytes", expectedLength, actualLength);
+
     clear();
-    final int len = valueCount * ${type.width};
     if (data != null) {
       data.release(1);
     }
-    data = buf.slice(0, len);
+    data = buffer.slice(0, actualLength);
     data.retain(1);
-    data.writerIndex(len);
-    return len;
-  }
-
-  @Override
-  public void load(SerializedField metadata, DrillBuf buffer) {
-    assert this.field.matches(metadata) : String.format("The field %s doesn't match the provided metadata %s.", this.field, metadata);
-    final int loaded = load(metadata.getValueCount(), buffer);
-    assert metadata.getBufferLength() == loaded : String.format("Expected to load %d bytes but actually loaded %d bytes", metadata.getBufferLength(), loaded);
-  }
+    data.writerIndex(actualLength);
+    }
 
-  @Override
-  public TransferPair getTransferPair() {
+  public TransferPair getTransferPair(){
     return new TransferImpl(getField());
   }
 
@@ -200,7 +195,7 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F
     return new TransferImpl((${minor.class}Vector) to);
   }
 
-  public void transferTo(${minor.class}Vector target) {
+  public void transferTo(${minor.class}Vector target){
     target.clear();
     target.data = data;
     target.data.retain(1);
@@ -217,10 +212,10 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F
     target.data.writerIndex(sliceLength);
   }
 
-  private class TransferImpl implements TransferPair {
+  private class TransferImpl implements TransferPair{
     private ${minor.class}Vector to;
 
-    public TransferImpl(MaterializedField field) {
+    public TransferImpl(MaterializedField field){
       to = new ${minor.class}Vector(field, allocator);
     }
 
@@ -229,12 +224,12 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F
     }
 
     @Override
-    public ${minor.class}Vector getTo() {
+    public ${minor.class}Vector getTo(){
       return to;
     }
 
     @Override
-    public void transfer() {
+    public void transfer(){
       transferTo(to);
     }
 
@@ -249,7 +244,7 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F
     }
   }
 
-  public void copyFrom(int fromIndex, int thisIndex, ${minor.class}Vector from) {
+  public void copyFrom(int fromIndex, int thisIndex, ${minor.class}Vector from){
     <#if (type.width > 8)>
     from.data.getBytes(fromIndex * ${type.width}, data, thisIndex * ${type.width}, ${type.width});
     <#else> <#-- type.width <= 8 -->
@@ -259,7 +254,7 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F
     </#if> <#-- type.width -->
   }
 
-  public void copyFromSafe(int fromIndex, int thisIndex, ${minor.class}Vector from) {
+  public void copyFromSafe(int fromIndex, int thisIndex, ${minor.class}Vector from){
     while(thisIndex >= getValueCapacity()) {
         reAlloc();
     }
@@ -284,7 +279,7 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F
     }
 
     @Override
-    public boolean isNull(int index) {
+    public boolean isNull(int index){
       return false;
     }
 
@@ -295,7 +290,7 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F
     }
 
     <#if (minor.class == "Interval")>
-    public void get(int index, ${minor.class}Holder holder) {
+    public void get(int index, ${minor.class}Holder holder){
 
       final int offsetIndex = index * ${type.width};
       holder.months = data.getInt(offsetIndex);
@@ -303,7 +298,7 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F
       holder.milliseconds = data.getInt(offsetIndex + ${minor.millisecondsOffset});
     }
 
-    public void get(int index, Nullable${minor.class}Holder holder) {
+    public void get(int index, Nullable${minor.class}Holder holder){
       final int offsetIndex = index * ${type.width};
       holder.isSet = 1;
       holder.months = data.getInt(offsetIndex);
@@ -357,14 +352,14 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F
     }
 
     <#elseif (minor.class == "IntervalDay")>
-    public void get(int index, ${minor.class}Holder holder) {
+    public void get(int index, ${minor.class}Holder holder){
 
       final int offsetIndex = index * ${type.width};
       holder.days = data.getInt(offsetIndex);
       holder.milliseconds = data.getInt(offsetIndex + ${minor.millisecondsOffset});
     }
 
-    public void get(int index, Nullable${minor.class}Holder holder) {
+    public void get(int index, Nullable${minor.class}Holder holder){
       final int offsetIndex = index * ${type.width};
       holder.isSet = 1;
       holder.days = data.getInt(offsetIndex);
@@ -434,12 +429,12 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F
     }
 
     <#else>
-    public void get(int index, ${minor.class}Holder holder) {
+    public void get(int index, ${minor.class}Holder holder){
       holder.buffer = data;
       holder.start = index * ${type.width};
     }
 
-    public void get(int index, Nullable${minor.class}Holder holder) {
+    public void get(int index, Nullable${minor.class}Holder holder){
       holder.isSet = 1;
       holder.buffer = data;
       holder.start = index * ${type.width};
@@ -534,7 +529,7 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F
     }
     </#if>
 
-    public void get(int index, ${minor.class}Holder holder) {
+    public void get(int index, ${minor.class}Holder holder){
       <#if minor.class.startsWith("Decimal")>
       holder.scale = getField().getScale();
       holder.precision = getField().getPrecision();
@@ -543,7 +538,7 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F
       holder.value = data.get${(minor.javaType!type.javaType)?cap_first}(index * ${type.width});
     }
 
-    public void get(int index, Nullable${minor.class}Holder holder) {
+    public void get(int index, Nullable${minor.class}Holder holder){
       holder.isSet = 1;
       holder.value = data.get${(minor.javaType!type.javaType)?cap_first}(index * ${type.width});
     }
@@ -563,7 +558,7 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F
   */
   public final class Mutator extends BaseDataValueVector.BaseMutator {
 
-    private Mutator() {};
+    private Mutator(){};
    /**
     * Set the element at the given index to the given value.  Note that widths smaller than
     * 32 bits are handled by the DrillBuf interface.
@@ -584,122 +579,122 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F
    }
 
   <#if (minor.class == "Interval")>
-   public void set(int index, int months, int days, int milliseconds) {
+   public void set(int index, int months, int days, int milliseconds){
      final int offsetIndex = index * ${type.width};
      data.setInt(offsetIndex, months);
      data.setInt((offsetIndex + ${minor.daysOffset}), days);
      data.setInt((offsetIndex + ${minor.millisecondsOffset}), milliseconds);
    }
 
-   protected void set(int index, ${minor.class}Holder holder) {
+   protected void set(int index, ${minor.class}Holder holder){
      set(index, holder.months, holder.days, holder.milliseconds);
    }
 
-   protected void set(int index, Nullable${minor.class}Holder holder) {
+   protected void set(int index, Nullable${minor.class}Holder holder){
      set(index, holder.months, holder.days, holder.milliseconds);
    }
 
-   public void setSafe(int index, int months, int days, int milliseconds) {
+   public void setSafe(int index, int months, int days, int milliseconds){
      while(index >= getValueCapacity()) {
        reAlloc();
      }
      set(index, months, days, milliseconds);
    }
 
-   public void setSafe(int index, Nullable${minor.class}Holder holder) {
+   public void setSafe(int index, Nullable${minor.class}Holder holder){
      setSafe(index, holder.months, holder.days, holder.milliseconds);
    }
 
-   public void setSafe(int index, ${minor.class}Holder holder) {
+   public void setSafe(int index, ${minor.class}Holder holder){
      setSafe(index, holder.months, holder.days, holder.milliseconds);
    }
 
    <#elseif (minor.class == "IntervalDay")>
-   public void set(int index, int days, int milliseconds) {
+   public void set(int index, int days, int milliseconds){
      final int offsetIndex = index * ${type.width};
      data.setInt(offsetIndex, days);
      data.setInt((offsetIndex + ${minor.millisecondsOffset}), milliseconds);
    }
 
-   protected void set(int index, ${minor.class}Holder holder) {
+   protected void set(int index, ${minor.class}Holder holder){
      set(index, holder.days, holder.milliseconds);
    }
-   protected void set(int index, Nullable${minor.class}Holder holder) {
+   protected void set(int index, Nullable${minor.class}Holder holder){
      set(index, holder.days, holder.milliseconds);
    }
 
-   public void setSafe(int index, int days, int milliseconds) {
+   public void setSafe(int index, int days, int milliseconds){
      while(index >= getValueCapacity()) {
        reAlloc();
      }
      set(index, days, milliseconds);
    }
 
-   public void setSafe(int index, ${minor.class}Holder holder) {
+   public void setSafe(int index, ${minor.class}Holder holder){
      setSafe(index, holder.days, holder.milliseconds);
    }
 
-   public void setSafe(int index, Nullable${minor.class}Holder holder) {
+   public void setSafe(int index, Nullable${minor.class}Holder holder){
      setSafe(index, holder.days, holder.milliseconds);
    }
 
    <#elseif (minor.class == "Decimal28Sparse" || minor.class == "Decimal38Sparse") || (minor.class == "Decimal28Dense") || (minor.class == "Decimal38Dense")>
 
-   public void set(int index, ${minor.class}Holder holder) {
+   public void set(int index, ${minor.class}Holder holder){
      set(index, holder.start, holder.buffer);
    }
 
-   void set(int index, Nullable${minor.class}Holder holder) {
+   void set(int index, Nullable${minor.class}Holder holder){
      set(index, holder.start, holder.buffer);
    }
 
-   public void setSafe(int index,  Nullable${minor.class}Holder holder) {
+   public void setSafe(int index,  Nullable${minor.class}Holder holder){
      setSafe(index, holder.start, holder.buffer);
    }
-   public void setSafe(int index,  ${minor.class}Holder holder) {
+   public void setSafe(int index,  ${minor.class}Holder holder){
      setSafe(index, holder.start, holder.buffer);
    }
 
-   public void setSafe(int index, int start, DrillBuf buffer) {
+   public void setSafe(int index, int start, DrillBuf buffer){
      while(index >= getValueCapacity()) {
        reAlloc();
      }
      set(index, start, buffer);
    }
 
-   public void set(int index, int start, DrillBuf buffer) {
+   public void set(int index, int start, DrillBuf buffer){
      data.setBytes(index * ${type.width}, buffer, start, ${type.width});
    }
 
    <#else>
 
-   protected void set(int index, ${minor.class}Holder holder) {
+   protected void set(int index, ${minor.class}Holder holder){
      set(index, holder.start, holder.buffer);
    }
 
-   public void set(int index, Nullable${minor.class}Holder holder) {
+   public void set(int index, Nullable${minor.class}Holder holder){
      set(index, holder.start, holder.buffer);
    }
 
-   public void set(int index, int start, DrillBuf buffer) {
+   public void set(int index, int start, DrillBuf buffer){
      data.setBytes(index * ${type.width}, buffer, start, ${type.width});
    }
 
-   public void setSafe(int index, ${minor.class}Holder holder) {
+   public void setSafe(int index, ${minor.class}Holder holder){
      setSafe(index, holder.start, holder.buffer);
    }
-   public void setSafe(int index, Nullable${minor.class}Holder holder) {
+   public void setSafe(int index, Nullable${minor.class}Holder holder){
      setSafe(index, holder.start, holder.buffer);
    }
 
-   public void setSafe(int index, int start, DrillBuf buffer) {
+   public void setSafe(int index, int start, DrillBuf buffer){
      while(index >= getValueCapacity()) {
        reAlloc();
      }
      set(index, holder);
    }
 
-   public void set(int index, Nullable${minor.class}Holder holder) {
+   public void set(int index, Nullable${minor.class}Holder holder){
      data.setBytes(index * ${type.width}, holder.buffer, holder.start, ${type.width});
    }
    </#if>
@@ -711,7 +706,7 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F
      final int valueCount = getAccessor().getValueCount();
      for(int i = 0; i < valueCount; i++, even = !even) {
        final byte b = even ? Byte.MIN_VALUE : Byte.MAX_VALUE;
-       for(int w = 0; w < ${type.width}; w++) {
+       for(int w = 0; w < ${type.width}; w++){
          data.setByte(i + w, b);
        }
      }
@@ -729,22 +724,22 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F
      set(index, value);
    }
 
-   protected void set(int index, ${minor.class}Holder holder) {
+   protected void set(int index, ${minor.class}Holder holder){
      data.set${(minor.javaType!type.javaType)?cap_first}(index * ${type.width}, holder.value);
    }
 
-   public void setSafe(int index, ${minor.class}Holder holder) {
+   public void setSafe(int index, ${minor.class}Holder holder){
      while(index >= getValueCapacity()) {
        reAlloc();
      }
      set(index, holder);
    }
 
-   protected void set(int index, Nullable${minor.class}Holder holder) {
+   protected void set(int index, Nullable${minor.class}Holder holder){
      data.set${(minor.javaType!type.javaType)?cap_first}(index * ${type.width}, holder.value);
    }
 
-   public void setSafe(int index, Nullable${minor.class}Holder holder) {
+   public void setSafe(int index, Nullable${minor.class}Holder holder){
      while(index >= getValueCapacity()) {
        reAlloc();
      }
@@ -757,9 +752,9 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F
      boolean even = true;
      final int valueCount = getAccessor().getValueCount();
      for(int i = 0; i < valueCount; i++, even = !even) {
-       if (even) {
+       if(even){
          set(i, ${minor.boxedType!type.boxedType}.MIN_VALUE);
-       } else {
+       }else{
          set(i, ${minor.boxedType!type.boxedType}.MAX_VALUE);
        }
      }
@@ -770,9 +765,9 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F
      boolean even = true;
      final int valueCount = getAccessor().getValueCount();
      for(int i = 0; i < valueCount; i++, even = !even) {
-       if (even) {
+       if(even){
          set(i, (${(minor.javaType!type.javaType)}) 1);
-       } else {
+       }else{
          set(i, (${(minor.javaType!type.javaType)}) 0);
        }
      }

http://git-wip-us.apache.org/repos/asf/drill/blob/77e2b89f/exec/java-exec/src/main/codegen/templates/NullableValueVectors.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/codegen/templates/NullableValueVectors.java b/exec/java-exec/src/main/codegen/templates/NullableValueVectors.java
index 4bdde16..fe4f71f 100644
--- a/exec/java-exec/src/main/codegen/templates/NullableValueVectors.java
+++ b/exec/java-exec/src/main/codegen/templates/NullableValueVectors.java
@@ -45,13 +45,15 @@ package org.apache.drill.exec.vector;
  * NB: this class is automatically generated from ${.template_name} and ValueVectorTypes.tdd using FreeMarker.
  */
 @SuppressWarnings("unused")
-public final class ${className} extends BaseDataValueVector implements <#if type.major == "VarLen">VariableWidth<#else>FixedWidth</#if>Vector, NullableVector {
+public final class ${className} extends BaseDataValueVector implements <#if type.major == "VarLen">VariableWidth<#else>FixedWidth</#if>Vector, NullableVector{
   private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(${className}.class);
 
   private final FieldReader reader = new Nullable${minor.class}ReaderImpl(Nullable${minor.class}Vector.this);
 
-  private final UInt1Vector bits = new UInt1Vector(MaterializedField.create(field + "_bits", Types.required(MinorType.UINT1)), allocator);
+  private final MaterializedField bitsField = MaterializedField.create("$bits$", Types.required(MinorType.UINT1));
+  private final UInt1Vector bits = new UInt1Vector(bitsField, allocator);
   private final ${valuesName} values = new ${minor.class}Vector(field, allocator);
+
   private final Mutator mutator = new Mutator();
   private final Accessor accessor = new Accessor();
 
@@ -60,12 +62,12 @@ public final class ${className} extends BaseDataValueVector implements <#if type
   }
 
   @Override
-  public FieldReader getReader() {
+  public FieldReader getReader(){
     return reader;
   }
 
   @Override
-  public int getValueCapacity() {
+  public int getValueCapacity(){
     return Math.min(bits.getValueCapacity(), values.getValueCapacity());
   }
 
@@ -96,7 +98,7 @@ public final class ${className} extends BaseDataValueVector implements <#if type
   }
 
   @Override
-  public int getBufferSize() {
+  public int getBufferSize(){
     return values.getBufferSize() + bits.getBufferSize();
   }
 
@@ -116,19 +118,16 @@ public final class ${className} extends BaseDataValueVector implements <#if type
     values.setInitialCapacity(numRecords);
   }
 
-  <#if type.major == "VarLen">
   @Override
-  public SerializedField getMetadata() {
-    return getMetadataBuilder()
-             .setValueCount(getAccessor().getValueCount())
-             .setVarByteLength(values.getVarByteLength())
-             .setBufferLength(getBufferSize())
-             .build();
+  public SerializedField.Builder getMetadataBuilder() {
+    return super.getMetadataBuilder()
+      .addChild(bits.getMetadata())
+      .addChild(values.getMetadata());
   }
 
   @Override
   public void allocateNew() {
-    if(!allocateNewSafe()) {
+    if(!allocateNewSafe()){
       throw new OutOfMemoryRuntimeException("Failure while allocating buffer.");
     }
   }
@@ -154,6 +153,7 @@ public final class ${className} extends BaseDataValueVector implements <#if type
     return success;
   }
 
+  <#if type.major == "VarLen">
   @Override
   public void allocateNew(int totalBytes, int valueCount) {
     try {
@@ -169,78 +169,21 @@ public final class ${className} extends BaseDataValueVector implements <#if type
   }
 
   @Override
-  public int load(int dataBytes, int valueCount, DrillBuf buf) {
-    clear();
-    int loaded = bits.load(valueCount, buf);
-
-    // remove bits part of buffer.
-    buf = buf.slice(loaded, buf.capacity() - loaded);
-    dataBytes -= loaded;
-    loaded += values.load(dataBytes, valueCount, buf);
-    mutator.lastSet = valueCount;
-    return loaded;
-  }
-
-  @Override
-  public void load(SerializedField metadata, DrillBuf buffer) {
-    assert field.matches(metadata) : String.format("The field %s doesn't match the provided metadata %s.", field, metadata);
-    final int loaded = load(metadata.getBufferLength(), metadata.getValueCount(), buffer);
-    assert metadata.getBufferLength() == loaded : String.format("Expected to load %d bytes but actually loaded %d bytes", metadata.getBufferLength(), loaded);
-  }
-
-  @Override
-  public int getByteCapacity() {
+  public int getByteCapacity(){
     return values.getByteCapacity();
   }
 
   @Override
-  public int getCurrentSizeInBytes() {
+  public int getCurrentSizeInBytes(){
     return values.getCurrentSizeInBytes();
   }
 
   <#else>
-
-  @Override
-  public void allocateNew() {
-    try {
-      values.allocateNew();
-      bits.allocateNew();
-    } catch(DrillRuntimeException e) {
-      clear();
-      throw e;
-    }
-    bits.zeroVector();
-    mutator.reset();
-    accessor.reset();
-  }
-
-
-  @Override
-  public boolean allocateNewSafe() {
-    /* Boolean to keep track if all the memory allocations were successful
-     * Used in the case of composite vectors when we need to allocate multiple
-     * buffers for multiple vectors. If one of the allocations failed we need to
-     * clear all the memory that we allocated
-     */
-    boolean success = false;
-    try {
-      success = values.allocateNewSafe() && bits.allocateNewSafe();
-    } finally {
-      if (!success) {
-        clear();
-      }
-    }
-    bits.zeroVector();
-    mutator.reset();
-    accessor.reset();
-    return success;
-  }
-
   @Override
   public void allocateNew(int valueCount) {
     try {
       values.allocateNew(valueCount);
-      bits.allocateNew(valueCount);
+      bits.allocateNew(valueCount+1);
     } catch(OutOfMemoryRuntimeException e) {
       clear();
       throw e;
@@ -255,32 +198,26 @@ public final class ${className} extends BaseDataValueVector implements <#if type
    */
   @Override
   public void zeroVector() {
-    values.zeroVector();
     bits.zeroVector();
+    values.zeroVector();
   }
+  </#if>
 
-  @Override
-  public int load(int valueCount, DrillBuf buf) {
-    clear();
-    int loaded = bits.load(valueCount, buf);
-
-    // remove bits part of buffer.
-    buf = buf.slice(loaded, buf.capacity() - loaded);
-    loaded += values.load(valueCount, buf);
-    return loaded;
-  }
 
   @Override
   public void load(SerializedField metadata, DrillBuf buffer) {
-    assert field.matches(metadata);
-    final int loaded = load(metadata.getValueCount(), buffer);
-    assert metadata.getBufferLength() == loaded;
-  }
+    clear();
+    final SerializedField bitsField = metadata.getChild(0);
+    bits.load(bitsField, buffer);
 
-  </#if>
+    final int capacity = buffer.capacity();
+    final int bitsLength = bitsField.getBufferLength();
+    final SerializedField valuesField = metadata.getChild(1);
+    values.load(valuesField, buffer.slice(bitsLength, capacity - bitsLength));
+  }
 
   @Override
-  public TransferPair getTransferPair() {
+  public TransferPair getTransferPair(){
     return new TransferImpl(getField());
   }
 
@@ -294,7 +231,7 @@ public final class ${className} extends BaseDataValueVector implements <#if type
     return new TransferImpl((Nullable${minor.class}Vector) to);
   }
 
-  public void transferTo(Nullable${minor.class}Vector target) {
+  public void transferTo(Nullable${minor.class}Vector target){
     bits.transferTo(target.bits);
     values.transferTo(target.values);
     <#if type.major == "VarLen">
@@ -314,21 +251,21 @@ public final class ${className} extends BaseDataValueVector implements <#if type
   private class TransferImpl implements TransferPair {
     Nullable${minor.class}Vector to;
 
-    public TransferImpl(MaterializedField field) {
+    public TransferImpl(MaterializedField field){
       to = new Nullable${minor.class}Vector(field, allocator);
     }
 
-    public TransferImpl(Nullable${minor.class}Vector to) {
+    public TransferImpl(Nullable${minor.class}Vector to){
       this.to = to;
     }
 
     @Override
-    public Nullable${minor.class}Vector getTo() {
+    public Nullable${minor.class}Vector getTo(){
       return to;
     }
 
     @Override
-    public void transfer() {
+    public void transfer(){
       transferTo(to);
     }
 
@@ -344,16 +281,16 @@ public final class ${className} extends BaseDataValueVector implements <#if type
   }
 
   @Override
-  public Accessor getAccessor() {
+  public Accessor getAccessor(){
     return accessor;
   }
 
   @Override
-  public Mutator getMutator() {
+  public Mutator getMutator(){
     return mutator;
   }
 
-  public ${minor.class}Vector convertToRequiredVector() {
+  public ${minor.class}Vector convertToRequiredVector(){
     ${minor.class}Vector v = new ${minor.class}Vector(getField().getOtherNullableVersion(), allocator);
     if (v.data != null) {
       v.data.release(1);
@@ -364,14 +301,14 @@ public final class ${className} extends BaseDataValueVector implements <#if type
     return v;
   }
 
-  public void copyFrom(int fromIndex, int thisIndex, Nullable${minor.class}Vector from) {
+  public void copyFrom(int fromIndex, int thisIndex, Nullable${minor.class}Vector from){
     final Accessor fromAccessor = from.getAccessor();
     if (!fromAccessor.isNull(fromIndex)) {
       mutator.set(thisIndex, fromAccessor.get(fromIndex));
     }
   }
 
-  public void copyFromSafe(int fromIndex, int thisIndex, ${minor.class}Vector from) {
+  public void copyFromSafe(int fromIndex, int thisIndex, ${minor.class}Vector from){
     <#if type.major == "VarLen">
     mutator.fillEmpties(thisIndex);
     </#if>
@@ -379,7 +316,7 @@ public final class ${className} extends BaseDataValueVector implements <#if type
     bits.getMutator().setSafe(thisIndex, 1);
   }
 
-  public void copyFromSafe(int fromIndex, int thisIndex, Nullable${minor.class}Vector from) {
+  public void copyFromSafe(int fromIndex, int thisIndex, Nullable${minor.class}Vector from){
     <#if type.major == "VarLen">
     mutator.fillEmpties(thisIndex);
     </#if>
@@ -400,7 +337,7 @@ public final class ${className} extends BaseDataValueVector implements <#if type
      */
     public <#if type.major == "VarLen">byte[]<#else>${minor.javaType!type.javaType}</#if> get(int index) {
       if (isNull(index)) {
-        throw new IllegalStateException("Can't get a null value");
+          throw new IllegalStateException("Can't get a null value");
       }
       return vAccessor.get(index);
     }
@@ -410,12 +347,12 @@ public final class ${className} extends BaseDataValueVector implements <#if type
       return isSet(index) == 0;
     }
 
-    public int isSet(int index) {
+    public int isSet(int index){
       return bAccessor.get(index);
     }
 
     <#if type.major == "VarLen">
-    public long getStartEnd(int index) {
+    public long getStartEnd(int index){
       return vAccessor.getStartEnd(index);
     }
 
@@ -425,7 +362,7 @@ public final class ${className} extends BaseDataValueVector implements <#if type
     }
     </#if>
 
-    public void get(int index, Nullable${minor.class}Holder holder) {
+    public void get(int index, Nullable${minor.class}Holder holder){
       vAccessor.get(index, holder);
       holder.isSet = bAccessor.get(index);
 
@@ -439,7 +376,7 @@ public final class ${className} extends BaseDataValueVector implements <#if type
     public ${friendlyType} getObject(int index) {
       if (isNull(index)) {
           return null;
-      } else {
+      }else{
         return vAccessor.getObject(index);
       }
     }
@@ -448,33 +385,33 @@ public final class ${className} extends BaseDataValueVector implements <#if type
     public StringBuilder getAsStringBuilder(int index) {
       if (isNull(index)) {
           return null;
-      } else {
+      }else{
         return vAccessor.getAsStringBuilder(index);
       }
     }
     </#if>
 
     @Override
-    public int getValueCount() {
+    public int getValueCount(){
       return bits.getAccessor().getValueCount();
     }
 
-    public void reset() {}
+    public void reset(){}
   }
 
   public final class Mutator extends BaseDataValueVector.BaseMutator implements NullableVectorDefinitionSetter<#if type.major = "VarLen">, VariableWidthVector.VariableWidthMutator</#if> {
     private int setCount;
     <#if type.major = "VarLen"> private int lastSet = -1;</#if>
 
-    private Mutator() {
+    private Mutator(){
     }
 
-    public ${valuesName} getVectorWithValues() {
+    public ${valuesName} getVectorWithValues(){
       return values;
     }
 
     @Override
-    public void setIndexDefined(int index) {
+    public void setIndexDefined(int index){
       bits.getMutator().set(index, 1);
     }
 
@@ -500,9 +437,9 @@ public final class ${className} extends BaseDataValueVector implements <#if type
 
     <#if type.major == "VarLen">
 
-    private void fillEmpties(int index) {
+    private void fillEmpties(int index){
       final ${valuesName}.Mutator valuesMutator = values.getMutator();
-      for(int i = lastSet; i < index; i++) {
+      for (int i = lastSet; i < index; i++) {
         valuesMutator.setSafe(i + 1, emptyByteArray);
       }
       while(index > bits.getValueCapacity()) {
@@ -543,20 +480,20 @@ public final class ${className} extends BaseDataValueVector implements <#if type
       </#if>
     }
 
-    public void setNull(int index) {
+    public void setNull(int index){
       bits.getMutator().setSafe(index, 0);
     }
 
-    public void setSkipNull(int index, ${minor.class}Holder holder) {
+    public void setSkipNull(int index, ${minor.class}Holder holder){
       values.getMutator().set(index, holder);
     }
 
-    public void setSkipNull(int index, Nullable${minor.class}Holder holder) {
+    public void setSkipNull(int index, Nullable${minor.class}Holder holder){
       values.getMutator().set(index, holder);
     }
 
 
-    public void set(int index, Nullable${minor.class}Holder holder) {
+    public void set(int index, Nullable${minor.class}Holder holder){
       final ${valuesName}.Mutator valuesMutator = values.getMutator();
       <#if type.major == "VarLen">
       for (int i = lastSet + 1; i < index; i++) {
@@ -568,7 +505,7 @@ public final class ${className} extends BaseDataValueVector implements <#if type
       <#if type.major == "VarLen">lastSet = index;</#if>
     }
 
-    public void set(int index, ${minor.class}Holder holder) {
+    public void set(int index, ${minor.class}Holder holder){
       final ${valuesName}.Mutator valuesMutator = values.getMutator();
       <#if type.major == "VarLen">
       for (int i = lastSet + 1; i < index; i++) {
@@ -585,7 +522,7 @@ public final class ${className} extends BaseDataValueVector implements <#if type
     }
 
     <#assign fields = minor.fields!type.fields />
-    public void set(int index, int isSet<#list fields as field><#if field.include!true >, ${field.type} ${field.name}Field</#if></#list> ) {
+    public void set(int index, int isSet<#list fields as field><#if field.include!true >, ${field.type} ${field.name}Field</#if></#list> ){
       final ${valuesName}.Mutator valuesMutator = values.getMutator();
       <#if type.major == "VarLen">
       for (int i = lastSet + 1; i < index; i++) {
@@ -654,7 +591,7 @@ public final class ${className} extends BaseDataValueVector implements <#if type
     }
 
     @Override
-    public void generateTestData(int valueCount) {
+    public void generateTestData(int valueCount){
       bits.getMutator().generateTestDataAlt(valueCount);
       values.getMutator().generateTestData(valueCount);
       <#if type.major = "VarLen">lastSet = valueCount;</#if>
@@ -662,7 +599,7 @@ public final class ${className} extends BaseDataValueVector implements <#if type
     }
 
     @Override
-    public void reset() {
+    public void reset(){
       setCount = 0;
       <#if type.major = "VarLen">lastSet = -1;</#if>
     }

http://git-wip-us.apache.org/repos/asf/drill/blob/77e2b89f/exec/java-exec/src/main/codegen/templates/VariableLengthVectors.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/codegen/templates/VariableLengthVectors.java b/exec/java-exec/src/main/codegen/templates/VariableLengthVectors.java
index ac4d539..76fbed3 100644
--- a/exec/java-exec/src/main/codegen/templates/VariableLengthVectors.java
+++ b/exec/java-exec/src/main/codegen/templates/VariableLengthVectors.java
@@ -48,17 +48,16 @@ package org.apache.drill.exec.vector;
  *
  * NB: this class is automatically generated from ${.template_name} and ValueVectorTypes.tdd using FreeMarker.
  */
-public final class ${minor.class}Vector extends BaseDataValueVector implements VariableWidthVector {
+public final class ${minor.class}Vector extends BaseDataValueVector implements VariableWidthVector{
   private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(${minor.class}Vector.class);
 
   private static final int DEFAULT_RECORD_BYTE_COUNT = 8;
   private static final int INITIAL_BYTE_COUNT = 4096 * DEFAULT_RECORD_BYTE_COUNT;
   private static final int MIN_BYTE_COUNT = 4096;
 
-  public final static String OFFSETS_VECTOR_NAME = "offsets";
-  private final static MaterializedField offsetsField =
-    MaterializedField.create(OFFSETS_VECTOR_NAME, Types.required(MinorType.UINT4));
-  private final UInt${type.width}Vector offsetVector;
+  public final static String OFFSETS_VECTOR_NAME = "$offsets$";
+  private final MaterializedField offsetsField = MaterializedField.create(OFFSETS_VECTOR_NAME, Types.required(MinorType.UINT4));
+  private final UInt${type.width}Vector offsetVector = new UInt${type.width}Vector(offsetsField, allocator);
   private final FieldReader reader = new ${minor.class}ReaderImpl(${minor.class}Vector.this);
 
   private final Accessor accessor;
@@ -71,19 +70,18 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements V
 
   public ${minor.class}Vector(MaterializedField field, BufferAllocator allocator) {
     super(field, allocator);
-    this.offsetVector = new UInt${type.width}Vector(offsetsField, allocator);
     this.oAccessor = offsetVector.getAccessor();
     this.accessor = new Accessor();
     this.mutator = new Mutator();
   }
 
   @Override
-  public FieldReader getReader() {
+  public FieldReader getReader(){
     return reader;
   }
 
   @Override
-  public int getBufferSize() {
+  public int getBufferSize(){
     if (getAccessor().getValueCount() == 0) {
       return 0;
     }
@@ -91,12 +89,12 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements V
   }
 
   @Override
-  public int getValueCapacity() {
-    return offsetVector.getValueCapacity() - 1;
+  public int getValueCapacity(){
+    return Math.max(offsetVector.getValueCapacity() - 1, 0);
   }
 
   @Override
-  public int getByteCapacity() {
+  public int getByteCapacity(){
     return data.capacity();
   }
 
@@ -109,7 +107,7 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements V
    * Return the number of bytes contained in the current var len byte vector.
    * @return
    */
-  public int getVarByteLength() {
+  public int getVarByteLength(){
     final int valueCount = getAccessor().getValueCount();
     if(valueCount == 0) {
       return 0;
@@ -119,31 +117,22 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements V
 
   @Override
   public SerializedField getMetadata() {
-    return getMetadataBuilder()
-             .setValueCount(getAccessor().getValueCount())
-             .setVarByteLength(getVarByteLength())
-             .setBufferLength(getBufferSize())
+    return getMetadataBuilder() //
+             .addChild(offsetVector.getMetadata())
+             .setValueCount(getAccessor().getValueCount()) //
+             .setBufferLength(getBufferSize()) //
              .build();
   }
 
   @Override
-  public int load(int dataBytes, int valueCount, DrillBuf buf) {
-    if(valueCount == 0) {
-      allocateNew(0, 0);
-      return 0;
-    }
-    clear();
-    final int loaded = offsetVector.load(valueCount + 1, buf);
-    data = buf.slice(loaded, dataBytes - loaded);
-    data.retain(1);
-    return dataBytes;
-  }
-
-  @Override
   public void load(SerializedField metadata, DrillBuf buffer) {
-    assert this.field.matches(metadata) : String.format("The field %s doesn't match the provided metadata %s.", this.field, metadata);
-    int loaded = load(metadata.getBufferLength(), metadata.getValueCount(), buffer);
-    assert metadata.getBufferLength() == loaded : String.format("Expected to load %d bytes but actually loaded %d bytes", metadata.getBufferLength(), loaded);
+    final SerializedField offsetField = metadata.getChild(0);
+    offsetVector.load(offsetField, buffer);
+
+    final int capacity = buffer.capacity();
+    final int offsetsLength = offsetField.getBufferLength();
+    data = buffer.slice(offsetsLength, capacity - offsetsLength);
+    data.retain();
   }
 
   @Override
@@ -165,16 +154,16 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements V
     return buffers;
   }
 
-  public long getOffsetAddr() {
+  public long getOffsetAddr(){
     return offsetVector.getBuffer().memoryAddress();
   }
 
-  public UInt${type.width}Vector getOffsetVector() {
+  public UInt${type.width}Vector getOffsetVector(){
     return offsetVector;
   }
 
   @Override
-  public TransferPair getTransferPair() {
+  public TransferPair getTransferPair(){
     return new TransferImpl(getField());
   }
 
@@ -188,7 +177,7 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements V
     return new TransferImpl((${minor.class}Vector) to);
   }
 
-  public void transferTo(${minor.class}Vector target) {
+  public void transferTo(${minor.class}Vector target){
     target.clear();
     this.offsetVector.transferTo(target.offsetVector);
     target.data = data;
@@ -203,16 +192,16 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements V
     target.clear();
     target.offsetVector.allocateNew(length + 1);
     offsetVectorAccessor = this.offsetVector.getAccessor();
-    final UInt4Vector.Mutator offsetVectorMutator = target.offsetVector.getMutator();
+    final UInt4Vector.Mutator targetOffsetVectorMutator = target.offsetVector.getMutator();
     for (int i = 0; i < length + 1; i++) {
-      offsetVectorMutator.set(i, offsetVectorAccessor.get(startIndex + i) - startPoint);
+      targetOffsetVectorMutator.set(i, offsetVectorAccessor.get(startIndex + i) - startPoint);
     }
     target.data = data.slice(startPoint, sliceLength);
     target.data.retain(1);
     target.getMutator().setValueCount(length);
 }
 
-  protected void copyFrom(int fromIndex, int thisIndex, ${minor.class}Vector from) {
+  protected void copyFrom(int fromIndex, int thisIndex, ${minor.class}Vector from){
     final UInt4Vector.Accessor fromOffsetVectorAccessor = from.offsetVector.getAccessor();
     final int start = fromOffsetVectorAccessor.get(fromIndex);
     final int end = fromOffsetVectorAccessor.get(fromIndex + 1);
@@ -223,7 +212,7 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements V
     offsetVector.data.set${(minor.javaType!type.javaType)?cap_first}( (thisIndex+1) * ${type.width}, outputStart + len);
   }
 
-  public boolean copyFromSafe(int fromIndex, int thisIndex, ${minor.class}Vector from) {
+  public boolean copyFromSafe(int fromIndex, int thisIndex, ${minor.class}Vector from){
     final UInt${type.width}Vector.Accessor fromOffsetVectorAccessor = from.offsetVector.getAccessor();
     final int start = fromOffsetVectorAccessor.get(fromIndex);
     final int end =   fromOffsetVectorAccessor.get(fromIndex + 1);
@@ -231,7 +220,7 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements V
     final int outputStart = offsetVector.data.get${(minor.javaType!type.javaType)?cap_first}(thisIndex * ${type.width});
 
     while(data.capacity() < outputStart + len) {
-      reAlloc();
+        reAlloc();
     }
 
     offsetVector.getMutator().setSafe(thisIndex + 1, outputStart + len);
@@ -239,24 +228,24 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements V
     return true;
   }
 
-  private class TransferImpl implements TransferPair {
+  private class TransferImpl implements TransferPair{
     ${minor.class}Vector to;
 
-    public TransferImpl(MaterializedField field) {
+    public TransferImpl(MaterializedField field){
       to = new ${minor.class}Vector(field, allocator);
     }
 
-    public TransferImpl(${minor.class}Vector to) {
+    public TransferImpl(${minor.class}Vector to){
       this.to = to;
     }
 
     @Override
-    public ${minor.class}Vector getTo() {
+    public ${minor.class}Vector getTo(){
       return to;
     }
 
     @Override
-    public void transfer() {
+    public void transfer(){
       transferTo(to);
     }
 
@@ -283,7 +272,7 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements V
 
   @Override
   public void allocateNew() {
-    if(!allocateNewSafe()) {
+    if(!allocateNewSafe()){
       throw new OutOfMemoryRuntimeException("Failure while allocating buffer.");
     }
   }
@@ -310,7 +299,7 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements V
      * clear all the memory that we allocated
      */
     try {
-      final int requestedSize = (int) curAllocationSize;
+      final int requestedSize = (int)curAllocationSize;
       data = allocator.buffer(requestedSize);
       offsetVector.allocateNew();
     } catch (OutOfMemoryRuntimeException e) {
@@ -363,7 +352,7 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements V
   }
 
   @Override
-  public Accessor getAccessor() {
+  public Accessor getAccessor(){
     return accessor;
   }
 
@@ -374,7 +363,7 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements V
 
   public final class Accessor extends BaseValueVector.BaseAccessor implements VariableWidthAccessor {
     final UInt${type.width}Vector.Accessor oAccessor = offsetVector.getAccessor();
-    public long getStartEnd(int index) {
+    public long getStartEnd(int index){
       return oAccessor.getTwoAsLong(index);
     }
 
@@ -394,13 +383,13 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements V
       return offsetVectorAccessor.get(index + 1) - offsetVectorAccessor.get(index);
     }
 
-    public void get(int index, ${minor.class}Holder holder) {
+    public void get(int index, ${minor.class}Holder holder){
       holder.start = oAccessor.get(index);
       holder.end = oAccessor.get(index + 1);
       holder.buffer = data;
     }
 
-    public void get(int index, Nullable${minor.class}Holder holder) {
+    public void get(int index, Nullable${minor.class}Holder holder){
       holder.isSet = 1;
       holder.start = oAccessor.get(index);
       holder.end = oAccessor.get(index + 1);
@@ -436,11 +425,11 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements V
     }
 
     @Override
-    public boolean isNull(int index) {
+    public boolean isNull(int index){
       return false;
     }
 
-    public UInt${type.width}Vector getOffsetVector() {
+    public UInt${type.width}Vector getOffsetVector(){
       return offsetVector;
     }
   }
@@ -474,7 +463,7 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements V
       assert index >= 0;
 
       final int currentOffset = offsetVector.getAccessor().get(index);
-      while(data.capacity() < currentOffset + bytes.length) {
+      while (data.capacity() < currentOffset + bytes.length) {
         reAlloc();
       }
       offsetVector.getMutator().setSafe(index + 1, currentOffset + bytes.length);
@@ -530,7 +519,7 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements V
     }
 
 
-    public void setSafe(int index, int start, int end, DrillBuf buffer) {
+    public void setSafe(int index, int start, int end, DrillBuf buffer){
       final int len = end - start;
       final int outputStart = offsetVector.data.get${(minor.javaType!type.javaType)?cap_first}(index * ${type.width});
 
@@ -542,7 +531,7 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements V
       buffer.getBytes(start, data, outputStart, len);
     }
 
-    public void setSafe(int index, Nullable${minor.class}Holder holder) {
+    public void setSafe(int index, Nullable${minor.class}Holder holder){
       assert holder.isSet == 1;
 
       final int start = holder.start;
@@ -559,7 +548,7 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements V
       offsetVector.getMutator().setSafe( index+1,  outputStart + len);
     }
 
-    public void setSafe(int index, ${minor.class}Holder holder) {
+    public void setSafe(int index, ${minor.class}Holder holder){
       final int start = holder.start;
       final int end =   holder.end;
       final int len = end - start;
@@ -573,7 +562,7 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements V
       offsetVector.getMutator().setSafe( index+1,  outputStart + len);
     }
 
-    protected void set(int index, int start, int length, DrillBuf buffer) {
+    protected void set(int index, int start, int length, DrillBuf buffer){
       assert index >= 0;
       final int currentOffset = offsetVector.getAccessor().get(index);
       offsetVector.getMutator().set(index + 1, currentOffset + length);
@@ -581,14 +570,14 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements V
       data.setBytes(currentOffset, bb);
     }
 
-    protected void set(int index, Nullable${minor.class}Holder holder) {
+    protected void set(int index, Nullable${minor.class}Holder holder){
       final int length = holder.end - holder.start;
       final int currentOffset = offsetVector.getAccessor().get(index);
       offsetVector.getMutator().set(index + 1, currentOffset + length);
       data.setBytes(currentOffset, holder.buffer, holder.start, length);
     }
 
-    protected void set(int index, ${minor.class}Holder holder) {
+    protected void set(int index, ${minor.class}Holder holder){
       final int length = holder.end - holder.start;
       final int currentOffset = offsetVector.getAccessor().get(index);
       offsetVector.getMutator().set(index + 1, currentOffset + length);
@@ -610,7 +599,7 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements V
     }
 
     @Override
-    public void generateTestData(int size) {
+    public void generateTestData(int size){
       boolean even = true;
       <#switch minor.class>
       <#case "Var16Char">
@@ -622,9 +611,9 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements V
       </#switch>
       final byte[] evenValue = new String("aaaaa").getBytes(charset);
       final byte[] oddValue = new String("bbbbbbbbbb").getBytes(charset);
-      for(int i =0; i < size; i++, even = !even) {
+      for(int i =0; i < size; i++, even = !even){
         set(i, even ? evenValue : oddValue);
-      }
+        }
       setValueCount(size);
     }
   }

http://git-wip-us.apache.org/repos/asf/drill/blob/77e2b89f/exec/java-exec/src/main/java/org/apache/drill/exec/vector/BitVector.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/BitVector.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/BitVector.java
index 054ef82..65e88e6 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/BitVector.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/BitVector.java
@@ -152,16 +152,6 @@ public final class BitVector extends BaseDataValueVector implements FixedWidthVe
     data.setZero(0, data.capacity());
   }
 
-  @Override
-  public int load(int valueCount, DrillBuf buf) {
-    clear();
-    this.valueCount = valueCount;
-    int len = getSizeFromCount(valueCount);
-    data = buf.slice(0, len);
-    data.retain();
-    return len;
-  }
-
   public void copyFrom(int inIndex, int outIndex, BitVector from) {
     this.mutator.set(outIndex, from.accessor.get(inIndex));
   }
@@ -177,9 +167,16 @@ public final class BitVector extends BaseDataValueVector implements FixedWidthVe
 
   @Override
   public void load(SerializedField metadata, DrillBuf buffer) {
-    assert this.field.matches(metadata);
-    int loaded = load(metadata.getValueCount(), buffer);
-    assert metadata.getBufferLength() == loaded;
+    assert field.matches(metadata);
+    final int valueCount = metadata.getValueCount();
+    final int expectedLength = getSizeFromCount(valueCount);
+    final int actualLength = metadata.getBufferLength();
+    assert expectedLength == actualLength: "expected and actual buffer sizes do not match";
+
+    clear();
+    data = buffer.slice(0, actualLength);
+    data.retain();
+    this.valueCount = valueCount;
   }
 
   public Mutator getMutator() {

http://git-wip-us.apache.org/repos/asf/drill/blob/77e2b89f/exec/java-exec/src/main/java/org/apache/drill/exec/vector/FixedWidthVector.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/FixedWidthVector.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/FixedWidthVector.java
index b546b9e..529c054 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/FixedWidthVector.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/FixedWidthVector.java
@@ -17,7 +17,6 @@
  */
 package org.apache.drill.exec.vector;
 
-import io.netty.buffer.DrillBuf;
 
 public interface FixedWidthVector extends ValueVector{
 
@@ -26,21 +25,8 @@ public interface FixedWidthVector extends ValueVector{
    *
    * @param valueCount   Number of values in the vector.
    */
-  public void allocateNew(int valueCount);
-
-  /**
-   * Load the records in the provided buffer based on the given number of values.
-   * @param valueCount Number of values the buffer contains.
-   * @param buf Incoming buffer.
-   * @return The number of bytes of the buffer that were consumed.
-   */
-  public int load(int valueCount, DrillBuf buf);
+  void allocateNew(int valueCount);
 
+  void zeroVector();
 
-  public abstract Mutator getMutator();
-
-  /**
-   * Zero out the underlying buffer backing this vector.
-   */
-  public void zeroVector();
 }

http://git-wip-us.apache.org/repos/asf/drill/blob/77e2b89f/exec/java-exec/src/main/java/org/apache/drill/exec/vector/NullableVector.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/NullableVector.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/NullableVector.java
index 92f60d6..8091c4c 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/NullableVector.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/NullableVector.java
@@ -19,5 +19,5 @@ package org.apache.drill.exec.vector;
 
 public interface NullableVector extends ValueVector{
 
-  public ValueVector getValuesVector();
+  ValueVector getValuesVector();
 }

http://git-wip-us.apache.org/repos/asf/drill/blob/77e2b89f/exec/java-exec/src/main/java/org/apache/drill/exec/vector/VariableWidthVector.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/VariableWidthVector.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/VariableWidthVector.java
index 10ddcc9..ee9c039 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/VariableWidthVector.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/VariableWidthVector.java
@@ -27,34 +27,25 @@ public interface VariableWidthVector extends ValueVector{
    * @param totalBytes   Desired size of the underlying data buffer.
    * @param valueCount   Number of values in the vector.
    */
-  public void allocateNew(int totalBytes, int valueCount);
+  void allocateNew(int totalBytes, int valueCount);
 
   /**
    * Provide the maximum amount of variable width bytes that can be stored int his vector.
    * @return
    */
-  public int getByteCapacity();
+  int getByteCapacity();
 
-  /**
-   * Load the records in the provided buffer based on the given number of values.
-   * @param dataBytes   The number of bytes associated with the data array.
-   * @param valueCount Number of values the buffer contains.
-   * @param buf Incoming buffer.
-   * @return The number of bytes of the buffer that were consumed.
-   */
-  public int load(int dataBytes, int valueCount, DrillBuf buf);
-
-  public abstract VariableWidthMutator getMutator();
+  VariableWidthMutator getMutator();
 
-  public abstract VariableWidthAccessor getAccessor();
+  VariableWidthAccessor getAccessor();
 
-  public interface VariableWidthAccessor extends Accessor {
-    public int getValueLength(int index);
+  interface VariableWidthAccessor extends Accessor {
+    int getValueLength(int index);
   }
 
-  public int getCurrentSizeInBytes();
+  int getCurrentSizeInBytes();
 
-  public interface VariableWidthMutator extends Mutator {
-    public void setValueLengthSafe(int index, int length);
+  interface VariableWidthMutator extends Mutator {
+    void setValueLengthSafe(int index, int length);
   }
 }

http://git-wip-us.apache.org/repos/asf/drill/blob/77e2b89f/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/BaseRepeatedValueVector.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/BaseRepeatedValueVector.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/BaseRepeatedValueVector.java
index 9bf6d85..fcec2ab 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/BaseRepeatedValueVector.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/BaseRepeatedValueVector.java
@@ -100,7 +100,7 @@ public abstract class BaseRepeatedValueVector extends BaseValueVector implements
 
   @Override
   public int getValueCapacity() {
-    final int offsetValueCapacity = offsets.getValueCapacity() - 1;
+    final int offsetValueCapacity = Math.max(offsets.getValueCapacity() - 1, 0);
     if (vector == DEFAULT_DATA_VECTOR) {
       return offsetValueCapacity;
     }

http://git-wip-us.apache.org/repos/asf/drill/blob/77e2b89f/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/MapVector.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/MapVector.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/MapVector.java
index 1e30ea2..a06ce66 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/MapVector.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/MapVector.java
@@ -33,6 +33,7 @@ import org.apache.drill.common.expression.SchemaPath;
 import org.apache.drill.common.types.TypeProtos.DataMode;
 import org.apache.drill.common.types.TypeProtos.MajorType;
 import org.apache.drill.common.types.TypeProtos.MinorType;
+import org.apache.drill.common.types.Types;
 import org.apache.drill.exec.expr.TypeHelper;
 import org.apache.drill.exec.expr.holders.ComplexHolder;
 import org.apache.drill.exec.memory.BufferAllocator;
@@ -52,7 +53,7 @@ import com.google.common.base.Preconditions;
 public class MapVector extends AbstractMapVector {
   static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(MapVector.class);
 
-  public final static MajorType TYPE = MajorType.newBuilder().setMinorType(MinorType.MAP).setMode(DataMode.REQUIRED).build();
+  public final static MajorType TYPE = Types.required(MinorType.MAP);
 
   private final SingleMapReaderImpl reader = new SingleMapReaderImpl(MapVector.this);
   private final Accessor accessor = new Accessor();
@@ -224,7 +225,7 @@ public class MapVector extends AbstractMapVector {
   @Override
   public int getValueCapacity() {
     if (size() == 0) {
-      return Integer.MAX_VALUE;
+      return 0;
     }
 
     final Ordering<ValueVector> natural = new Ordering<ValueVector>() {

http://git-wip-us.apache.org/repos/asf/drill/blob/77e2b89f/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedMapVector.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedMapVector.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedMapVector.java
index 644e5db..40d0be4 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedMapVector.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedMapVector.java
@@ -415,7 +415,7 @@ public class RepeatedMapVector extends AbstractMapVector implements RepeatedValu
 
   @Override
   public int getValueCapacity() {
-    return offsets.getValueCapacity()-1;
+    return Math.max(offsets.getValueCapacity() - 1, 0);
   }
 
   @Override

http://git-wip-us.apache.org/repos/asf/drill/blob/77e2b89f/exec/java-exec/src/test/java/org/apache/drill/exec/record/vector/TestValueVector.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/record/vector/TestValueVector.java b/exec/java-exec/src/test/java/org/apache/drill/exec/record/vector/TestValueVector.java
index 4eeb8f2..339dbe8 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/record/vector/TestValueVector.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/record/vector/TestValueVector.java
@@ -23,8 +23,13 @@ import static org.junit.Assert.assertTrue;
 
 import java.nio.charset.Charset;
 
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableMap;
+import io.netty.buffer.DrillBuf;
 import org.apache.drill.common.AutoCloseables;
 import org.apache.drill.common.expression.SchemaPath;
+import org.apache.drill.common.types.TypeProtos;
+import org.apache.drill.common.types.Types;
 import org.apache.drill.exec.ExecTest;
 import org.apache.drill.exec.exception.OversizedAllocationException;
 import org.apache.drill.exec.expr.TypeHelper;
@@ -36,9 +41,11 @@ import org.apache.drill.exec.expr.holders.NullableVar16CharHolder;
 import org.apache.drill.exec.expr.holders.NullableVarCharHolder;
 import org.apache.drill.exec.expr.holders.RepeatedFloat4Holder;
 import org.apache.drill.exec.expr.holders.RepeatedVarBinaryHolder;
+import org.apache.drill.exec.expr.holders.UInt1Holder;
 import org.apache.drill.exec.expr.holders.UInt4Holder;
 import org.apache.drill.exec.expr.holders.VarCharHolder;
 import org.apache.drill.exec.memory.TopLevelAllocator;
+import org.apache.drill.exec.proto.UserBitShared;
 import org.apache.drill.exec.record.MaterializedField;
 import org.apache.drill.exec.vector.BaseValueVector;
 import org.apache.drill.exec.vector.BitVector;
@@ -480,4 +487,116 @@ public class TestValueVector extends ExecTest {
       AutoCloseables.close(valueVectors);
     }
   }
+
+  protected interface VectorVerifier {
+    void verify(ValueVector vector) throws Exception;
+  }
+
+  protected static class ChildVerifier implements VectorVerifier {
+    public final TypeProtos.MajorType[] types;
+
+    public ChildVerifier(TypeProtos.MajorType... childTypes) {
+      this.types = Preconditions.checkNotNull(childTypes);
+    }
+
+    @Override
+    public void verify(ValueVector vector) throws Exception {
+      final String hint = String.format("%s failed the test case", vector.getClass().getSimpleName());
+
+      final UserBitShared.SerializedField metadata = vector.getMetadata();
+      final int actual = metadata.getChildCount();
+      assertEquals(hint, types.length, actual);
+
+      for (int i = 0; i < types.length; i++) {
+        final UserBitShared.SerializedField child = metadata.getChild(i);
+
+        assertEquals(hint, types[i], child.getMajorType());
+      }
+    }
+  }
+
+  /**
+   * Convenience method that allows running tests on various {@link ValueVector vector} instances.
+   *
+   * @param test test function to execute
+   */
+  private void testVectors(VectorVerifier test) throws Exception {
+    final MaterializedField[] fields = {
+        MaterializedField.create(EMPTY_SCHEMA_PATH, UInt1Holder.TYPE),
+        MaterializedField.create(EMPTY_SCHEMA_PATH, BitHolder.TYPE),
+        MaterializedField.create(EMPTY_SCHEMA_PATH, VarCharHolder.TYPE),
+        MaterializedField.create(EMPTY_SCHEMA_PATH, NullableVarCharHolder.TYPE),
+        MaterializedField.create(EMPTY_SCHEMA_PATH, RepeatedListVector.TYPE),
+        MaterializedField.create(EMPTY_SCHEMA_PATH, MapVector.TYPE),
+        MaterializedField.create(EMPTY_SCHEMA_PATH, RepeatedMapVector.TYPE)
+    };
+
+    final ValueVector[] vectors = {
+        new UInt4Vector(fields[0], allocator),
+        new BitVector(fields[1], allocator),
+        new VarCharVector(fields[2], allocator),
+        new NullableVarCharVector(fields[3], allocator),
+        new RepeatedListVector(fields[4], allocator, null),
+        new MapVector(fields[5], allocator, null),
+        new RepeatedMapVector(fields[6], allocator, null)
+    };
+
+    try {
+      for (final ValueVector vector : vectors) {
+        test.verify(vector);
+      }
+    } finally {
+      AutoCloseables.close(vectors);
+    }
+  }
+
+  @Test
+  public void testVectorMetadataIsAccurate() throws Exception {
+    final VectorVerifier noChild = new ChildVerifier();
+    final VectorVerifier offsetChild = new ChildVerifier(UInt4Holder.TYPE);
+
+    final ImmutableMap.Builder<Class, VectorVerifier> builder = ImmutableMap.builder();
+    builder.put(UInt4Vector.class, noChild);
+    builder.put(BitVector.class, noChild);
+    builder.put(VarCharVector.class, offsetChild);
+    builder.put(NullableVarCharVector.class, new ChildVerifier(UInt1Holder.TYPE, Types.optional(TypeProtos.MinorType.VARCHAR)));
+    builder.put(RepeatedListVector.class, new ChildVerifier(UInt4Holder.TYPE, Types.LATE_BIND_TYPE));
+    builder.put(MapVector.class, noChild);
+    builder.put(RepeatedMapVector.class, offsetChild);
+    final ImmutableMap<Class, VectorVerifier> children = builder.build();
+
+    testVectors(new VectorVerifier() {
+
+      @Override
+      public void verify(ValueVector vector) throws Exception {
+
+        final Class klazz = vector.getClass();
+        final VectorVerifier verifier = children.get(klazz);
+        verifier.verify(vector);
+      }
+    });
+  }
+
+  @Test
+  public void testVectorCanLoadEmptyBuffer() throws Exception {
+    final DrillBuf empty = allocator.getEmpty();
+
+    testVectors(new VectorVerifier() {
+
+      @Override
+      public void verify(ValueVector vector) {
+        final String hint = String.format("%s failed the test case", vector.getClass().getSimpleName());
+        final UserBitShared.SerializedField metadata = vector.getMetadata();
+        assertEquals(hint, 0, metadata.getBufferLength());
+        assertEquals(hint, 0, metadata.getValueCount());
+
+        vector.load(metadata, empty);
+
+        assertEquals(hint, 0, vector.getValueCapacity());
+        assertEquals(hint, 0, vector.getAccessor().getValueCount());
+
+        vector.clear();
+      }
+    });
+  }
 }


Mime
View raw message