drill-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From prog...@apache.org
Subject [1/3] drill git commit: DRILL-5517: Size-aware set methods in value vectors
Date Mon, 03 Jul 2017 19:49:34 GMT
Repository: drill
Updated Branches:
  refs/heads/master 6446e56f2 -> 63e243378


http://git-wip-us.apache.org/repos/asf/drill/blob/92c9304f/exec/vector/src/main/codegen/templates/NullableValueVectors.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/codegen/templates/NullableValueVectors.java b/exec/vector/src/main/codegen/templates/NullableValueVectors.java
index 170c606..0f8d90c 100644
--- a/exec/vector/src/main/codegen/templates/NullableValueVectors.java
+++ b/exec/vector/src/main/codegen/templates/NullableValueVectors.java
@@ -251,8 +251,8 @@ public final class ${className} extends BaseDataValueVector implements <#if type
     bits.zeroVector();
     values.zeroVector();
   }
-  </#if>
 
+  </#if>
   @Override
   public void load(SerializedField metadata, DrillBuf buffer) {
     clear();
@@ -374,6 +374,27 @@ public final class ${className} extends BaseDataValueVector implements <#if type
     values.copyFromSafe(fromIndex, thisIndex, from.values);
   }
 
+  @Override
+  public void copyEntry(int toIndex, ValueVector from, int fromIndex) {
+    Nullable${minor.class}Vector fromVector = (Nullable${minor.class}Vector) from;
+    <#if type.major == "VarLen">
+
+    // This method is to be called only for loading the vector
+    // sequentially, so there should be no empties to fill.
+
+    </#if>
+    bits.copyFromSafe(fromIndex, toIndex, fromVector.bits);
+    values.copyFromSafe(fromIndex, toIndex, fromVector.values);
+  }
+
+  @Override
+  public void exchange(ValueVector other) {
+    ${className} target = (${className}) other;
+    bits.exchange(target.bits);
+    values.exchange(target.values);
+    mutator.exchange(other.getMutator());
+  }
+
   public final class Accessor extends BaseDataValueVector.BaseAccessor <#if type.major = "VarLen">implements VariableWidthVector.VariableWidthAccessor</#if> {
     final UInt1Vector.Accessor bAccessor = bits.getAccessor();
     final ${valuesName}.Accessor vAccessor = values.getAccessor();
@@ -383,7 +404,7 @@ public final class ${className} extends BaseDataValueVector implements <#if type
      *
      * @param   index   position of the value
      * @return  value of the element, if not null
-     * @throws  NullValueException if the value is null
+     * @throws  IllegalStateException if the value is null
      */
     public <#if type.major == "VarLen">byte[]<#else>${minor.javaType!type.javaType}</#if> get(int index) {
       if (isNull(index)) {
@@ -410,8 +431,8 @@ public final class ${className} extends BaseDataValueVector implements <#if type
     public int getValueLength(int index) {
       return values.getAccessor().getValueLength(index);
     }
-    </#if>
 
+    </#if>
     public void get(int index, Nullable${minor.class}Holder holder){
       vAccessor.get(index, holder);
       holder.isSet = bAccessor.get(index);
@@ -439,22 +460,21 @@ public final class ${className} extends BaseDataValueVector implements <#if type
         return vAccessor.getAsStringBuilder(index);
       }
     }
-    </#if>
 
+    </#if>
     @Override
-    public int getValueCount(){
+    public int getValueCount() {
       return bits.getAccessor().getValueCount();
     }
 
-    public void reset(){}
+    public void reset() {}
   }
 
   public final class Mutator extends BaseDataValueVector.BaseMutator implements NullableVectorDefinitionSetter<#if type.major = "VarLen">, VariableWidthVector.VariableWidthMutator</#if> {
     private int setCount;
-    <#if type.major = "VarLen"> private int lastSet = -1;</#if>
+    <#if type.major = "VarLen">private int lastSet = -1;</#if>
 
-    private Mutator(){
-    }
+    private Mutator() { }
 
     public ${valuesName} getVectorWithValues(){
       return values;
@@ -466,11 +486,12 @@ public final class ${className} extends BaseDataValueVector implements <#if type
     }
 
     /**
-     * Set the variable length element at the specified index to the supplied byte array.
+     * Set the variable length element at the specified index to the supplied value.
      *
      * @param index   position of the bit to set
-     * @param bytes   array of bytes to write
+     * @param value   value to write
      */
+
     public void set(int index, <#if type.major == "VarLen">byte[]<#elseif (type.width < 4)>int<#else>${minor.javaType!type.javaType}</#if> value) {
       setCount++;
       final ${valuesName}.Mutator valuesMutator = values.getMutator();
@@ -486,7 +507,7 @@ public final class ${className} extends BaseDataValueVector implements <#if type
     }
 
     <#if type.major == "VarLen">
-    private void fillEmpties(int index){
+    private void fillEmpties(int index) {
       final ${valuesName}.Mutator valuesMutator = values.getMutator();
       for (int i = lastSet; i < index; i++) {
         valuesMutator.setSafe(i + 1, emptyByteArray);
@@ -502,27 +523,29 @@ public final class ${className} extends BaseDataValueVector implements <#if type
       values.getMutator().setValueLengthSafe(index, length);
       lastSet = index;
     }
-    </#if>
 
     public void setSafe(int index, byte[] value, int start, int length) {
-      <#if type.major != "VarLen">
-      throw new UnsupportedOperationException();
-      <#else>
-      if (index > lastSet + 1) {
+       if (index > lastSet + 1) {
         fillEmpties(index);
       }
 
       bits.getMutator().setSafe(index, 1);
       values.getMutator().setSafe(index, value, start, length);
       setCount++;
-      <#if type.major == "VarLen">lastSet = index;</#if>
-      </#if>
+      lastSet = index;
+    }
+
+    public void setScalar(int index, byte[] value, int start, int length) throws VectorOverflowException {
+      if (index > lastSet + 1) {
+        fillEmpties(index); // Filling empties cannot overflow the vector
+      }
+      values.getMutator().setScalar(index, value, start, length);
+      bits.getMutator().setSafe(index, 1);
+      setCount++;
+      lastSet = index;
     }
 
     public void setSafe(int index, ByteBuffer value, int start, int length) {
-      <#if type.major != "VarLen">
-      throw new UnsupportedOperationException();
-      <#else>
       if (index > lastSet + 1) {
         fillEmpties(index);
       }
@@ -530,23 +553,38 @@ public final class ${className} extends BaseDataValueVector implements <#if type
       bits.getMutator().setSafe(index, 1);
       values.getMutator().setSafe(index, value, start, length);
       setCount++;
-      <#if type.major == "VarLen">lastSet = index;</#if>
-      </#if>
+      lastSet = index;
     }
 
-    public void setNull(int index){
+    public void setScalar(int index, DrillBuf value, int start, int length) throws VectorOverflowException {
+      if (index > lastSet + 1) {
+        fillEmpties(index); // Filling empties cannot overflow the vector
+      }
+
+      values.getMutator().setScalar(index, value, start, length);
+      bits.getMutator().setSafe(index, 1);
+      setCount++;
+      lastSet = index;
+    }
+
+    </#if>
+    public void setNull(int index) {
       bits.getMutator().setSafe(index, 0);
     }
 
-    public void setSkipNull(int index, ${minor.class}Holder holder){
+    public void setSkipNull(int index, ${minor.class}Holder holder) {
       values.getMutator().set(index, holder);
     }
 
-    public void setSkipNull(int index, Nullable${minor.class}Holder holder){
+    public void setSkipNull(int index, Nullable${minor.class}Holder holder) {
       values.getMutator().set(index, holder);
     }
 
-    public void set(int index, Nullable${minor.class}Holder holder){
+    public void setNullBounded(int index) throws VectorOverflowException {
+      bits.getMutator().setScalar(index, 0);
+    }
+
+    public void set(int index, Nullable${minor.class}Holder holder) {
       final ${valuesName}.Mutator valuesMutator = values.getMutator();
       <#if type.major == "VarLen">
       for (int i = lastSet + 1; i < index; i++) {
@@ -558,7 +596,7 @@ public final class ${className} extends BaseDataValueVector implements <#if type
       <#if type.major == "VarLen">lastSet = index;</#if>
     }
 
-    public void set(int index, ${minor.class}Holder holder){
+    public void set(int index, ${minor.class}Holder holder) {
       final ${valuesName}.Mutator valuesMutator = values.getMutator();
       <#if type.major == "VarLen">
       for (int i = lastSet + 1; i < index; i++) {
@@ -575,7 +613,7 @@ public final class ${className} extends BaseDataValueVector implements <#if type
     }
 
     <#assign fields = minor.fields!type.fields />
-    public void set(int index, int isSet<#list fields as field><#if field.include!true >, ${field.type} ${field.name}Field</#if></#list> ){
+    public void set(int index, int isSet<#list fields as field><#if field.include!true >, ${field.type} ${field.name}Field</#if></#list> ) {
       final ${valuesName}.Mutator valuesMutator = values.getMutator();
       <#if type.major == "VarLen">
       for (int i = lastSet + 1; i < index; i++) {
@@ -593,11 +631,22 @@ public final class ${className} extends BaseDataValueVector implements <#if type
         fillEmpties(index);
       }
       </#if>
-
       bits.getMutator().setSafe(index, isSet);
       values.getMutator().setSafe(index<#list fields as field><#if field.include!true >, ${field.name}Field</#if></#list>);
       setCount++;
       <#if type.major == "VarLen">lastSet = index;</#if>
+   }
+
+    public void setScalar(int index, int isSet<#list fields as field><#if field.include!true >, ${field.type} ${field.name}Field</#if></#list> ) throws VectorOverflowException {
+      <#if type.major == "VarLen">
+      if (index > lastSet + 1) {
+        fillEmpties(index);
+      }
+      </#if>
+      values.getMutator().setScalar(index<#list fields as field><#if field.include!true >, ${field.name}Field</#if></#list>);
+      bits.getMutator().setSafe(index, isSet);
+      setCount++;
+      <#if type.major == "VarLen">lastSet = index;</#if>
     }
 
     public void setSafe(int index, Nullable${minor.class}Holder value) {
@@ -612,6 +661,18 @@ public final class ${className} extends BaseDataValueVector implements <#if type
       <#if type.major == "VarLen">lastSet = index;</#if>
     }
 
+    public void setScalar(int index, Nullable${minor.class}Holder value) throws VectorOverflowException {
+      <#if type.major == "VarLen">
+      if (index > lastSet + 1) {
+        fillEmpties(index);
+      }
+      </#if>
+      values.getMutator().setScalar(index, value);
+      bits.getMutator().setSafe(index, value.isSet);
+      setCount++;
+      <#if type.major == "VarLen">lastSet = index;</#if>
+    }
+
     public void setSafe(int index, ${minor.class}Holder value) {
       <#if type.major == "VarLen">
       if (index > lastSet + 1) {
@@ -624,6 +685,18 @@ public final class ${className} extends BaseDataValueVector implements <#if type
       <#if type.major == "VarLen">lastSet = index;</#if>
     }
 
+    public void setScalar(int index, ${minor.class}Holder value) throws VectorOverflowException {
+      <#if type.major == "VarLen">
+      if (index > lastSet + 1) {
+        fillEmpties(index);
+      }
+      </#if>
+      values.getMutator().setScalar(index, value);
+      bits.getMutator().setSafe(index, 1);
+      setCount++;
+      <#if type.major == "VarLen">lastSet = index;</#if>
+    }
+
     <#if !(type.major == "VarLen" || minor.class == "Decimal28Sparse" || minor.class == "Decimal38Sparse" || minor.class == "Decimal28Dense" || minor.class == "Decimal38Dense" || minor.class == "Interval" || minor.class == "IntervalDay")>
     public void setSafe(int index, ${minor.javaType!type.javaType} value) {
       <#if type.major == "VarLen">
@@ -636,6 +709,17 @@ public final class ${className} extends BaseDataValueVector implements <#if type
       setCount++;
     }
 
+    public void setScalar(int index, ${minor.javaType!type.javaType} value) throws VectorOverflowException {
+      <#if type.major == "VarLen">
+      if (index > lastSet + 1) {
+        fillEmpties(index);
+      }
+      </#if>
+      values.getMutator().setScalar(index, value);
+      bits.getMutator().setSafe(index, 1);
+      setCount++;
+    }
+
     </#if>
     <#if minor.class == "Decimal28Sparse" || minor.class == "Decimal38Sparse">
     public void set(int index, BigDecimal value) {
@@ -650,6 +734,12 @@ public final class ${className} extends BaseDataValueVector implements <#if type
       setCount++;
     }
 
+    public void setScalar(int index, BigDecimal value) throws VectorOverflowException {
+      values.getMutator().setScalar(index, value);
+      bits.getMutator().setSafe(index, 1);
+      setCount++;
+    }
+
     </#if>
     @Override
     public void setValueCount(int valueCount) {
@@ -674,6 +764,17 @@ public final class ${className} extends BaseDataValueVector implements <#if type
       setCount = 0;
       <#if type.major = "VarLen">lastSet = -1;</#if>
     }
+
+    // For nullable vectors, exchanging buffers (done elsewhere)
+    // requires also exchanging mutator state (done here.)
+
+    @Override
+    public void exchange(ValueVector.Mutator other) {
+      final Mutator target = (Mutator) other;
+      int temp = setCount;
+      setCount = target.setCount;
+      target.setCount = temp;
+    }
   }
 }
 </#list>

http://git-wip-us.apache.org/repos/asf/drill/blob/92c9304f/exec/vector/src/main/codegen/templates/RepeatedValueVectors.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/codegen/templates/RepeatedValueVectors.java b/exec/vector/src/main/codegen/templates/RepeatedValueVectors.java
index 159a8e7..9780b7d 100644
--- a/exec/vector/src/main/codegen/templates/RepeatedValueVectors.java
+++ b/exec/vector/src/main/codegen/templates/RepeatedValueVectors.java
@@ -1,4 +1,4 @@
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
@@ -36,10 +36,10 @@ package org.apache.drill.exec.vector;
 <#include "/@includes/vv_imports.ftl" />
 
 /**
- * Repeated${minor.class} implements a vector with multple values per row (e.g. JSON array or
- * repeated protobuf field).  The implementation uses two additional value vectors; one to convert
- * the index offset to the underlying element offset, and another to store the number of values
- * in the vector.
+ * Repeated${minor.class} implements a vector with multiple values per row (e.g. JSON array or
+ * repeated protobuf field).  The implementation uses an additional value vectors to convert
+ * the index offset to the underlying element offset. The count of values comes from subtracting
+ * two successive offsets.
  *
  * NB: this class is automatically generated from ${.template_name} and ValueVectorTypes.tdd using FreeMarker.
  */
@@ -178,11 +178,16 @@ public final class Repeated${minor.class}Vector extends BaseRepeatedValueVector
     }
   }
 
+  @Override
+  public void copyEntry(int toIndex, ValueVector from, int fromIndex) {
+    copyFromSafe(fromIndex, toIndex, (Repeated${minor.class}Vector) from);
+  }
+
   public boolean allocateNewSafe() {
-    /* boolean to keep track if all the memory allocation were successful
+    /* boolean to keep track if all the memory allocations were successful.
      * Used in the case of composite vectors when we need to allocate multiple
      * buffers for multiple vectors. If one of the allocations failed we need to
-     * clear all the memory that we allocated
+     * clear all the memory that we allocated.
      */
     boolean success = false;
     try {
@@ -239,12 +244,6 @@ public final class Repeated${minor.class}Vector extends BaseRepeatedValueVector
   @Override
   public void allocateNew(int valueCount, int innerValueCount) {
     clear();
-    /* boolean to keep track if all the memory allocation were successful
-     * Used in the case of composite vectors when we need to allocate multiple
-     * buffers for multiple vectors. If one of the allocations failed we need to//
-     * clear all the memory that we allocated
-     */
-    boolean success = false;
     try {
       offsets.allocateNew(valueCount + 1);
       values.allocateNew(innerValueCount);
@@ -258,9 +257,9 @@ public final class Repeated${minor.class}Vector extends BaseRepeatedValueVector
 
   </#if>
   // This is declared a subclass of the accessor declared inside of FixedWidthVector, this is also used for
-  // variable length vectors, as they should ahve consistent interface as much as possible, if they need to diverge
+  // variable length vectors, as they should have a consistent interface as much as possible, if they need to diverge
   // in the future, the interface shold be declared in the respective value vector superclasses for fixed and variable
-  // and we should refer to each in the generation template
+  // and we should refer to each in the generation template.
   public final class Accessor extends BaseRepeatedValueVector.BaseRepeatedAccessor {
     @Override
     public List<${friendlyType}> getObject(int index) {
@@ -281,7 +280,7 @@ public final class Repeated${minor.class}Vector extends BaseRepeatedValueVector
     }
 
     /**
-     * Get a value for the given record.  Each element in the repeated field is accessed by
+     * Get a value for the given record. Each element in the repeated field is accessed by
      * the positionIndex param.
      *
      * @param  index           record containing the repeated field
@@ -329,7 +328,7 @@ public final class Repeated${minor.class}Vector extends BaseRepeatedValueVector
      * @param value   value to add to the given row
      */
     public void add(int index, <#if type.major == "VarLen">byte[]<#elseif (type.width < 4)>int<#else>${minor.javaType!type.javaType}</#if> value) {
-      int nextOffset = offsets.getAccessor().get(index+1);
+      final int nextOffset = offsets.getAccessor().get(index+1);
       values.getMutator().set(nextOffset, value);
       offsets.getMutator().set(index+1, nextOffset+1);
     }
@@ -339,12 +338,25 @@ public final class Repeated${minor.class}Vector extends BaseRepeatedValueVector
       addSafe(index, bytes, 0, bytes.length);
     }
 
+    public void addEntry(int index, byte[] bytes) throws VectorOverflowException {
+      addEntry(index, bytes, 0, bytes.length);
+    }
+
     public void addSafe(int index, byte[] bytes, int start, int length) {
       final int nextOffset = offsets.getAccessor().get(index+1);
       values.getMutator().setSafe(nextOffset, bytes, start, length);
       offsets.getMutator().setSafe(index+1, nextOffset+1);
     }
 
+    public void addEntry(int index, byte[] bytes, int start, int length) throws VectorOverflowException {
+      if (index >= MAX_ROW_COUNT) {
+        throw new VectorOverflowException();
+      }
+      final int nextOffset = offsets.getAccessor().get(index+1);
+      values.getMutator().setArrayItem(nextOffset, bytes, start, length);
+      offsets.getMutator().setSafe(index+1, nextOffset+1);
+    }
+
     <#else>
     public void addSafe(int index, ${minor.javaType!type.javaType} srcValue) {
       final int nextOffset = offsets.getAccessor().get(index+1);
@@ -352,6 +364,15 @@ public final class Repeated${minor.class}Vector extends BaseRepeatedValueVector
       offsets.getMutator().setSafe(index+1, nextOffset+1);
     }
 
+    public void addEntry(int index, ${minor.javaType!type.javaType} srcValue) throws VectorOverflowException {
+      if (index >= MAX_ROW_COUNT) {
+        throw new VectorOverflowException();
+      }
+      final int nextOffset = offsets.getAccessor().get(index+1);
+      values.getMutator().setArrayItem(nextOffset, srcValue);
+      offsets.getMutator().setSafe(index+1, nextOffset+1);
+    }
+
     </#if>
     public void setSafe(int index, Repeated${minor.class}Holder h) {
       final ${minor.class}Holder ih = new ${minor.class}Holder();
@@ -364,43 +385,104 @@ public final class Repeated${minor.class}Vector extends BaseRepeatedValueVector
     }
 
     public void addSafe(int index, ${minor.class}Holder holder) {
-      int nextOffset = offsets.getAccessor().get(index+1);
+      final int nextOffset = offsets.getAccessor().get(index+1);
       values.getMutator().setSafe(nextOffset, holder);
       offsets.getMutator().setSafe(index+1, nextOffset+1);
     }
 
+    public void addEntry(int index, ${minor.class}Holder holder) throws VectorOverflowException {
+      if (index >= MAX_ROW_COUNT) {
+        throw new VectorOverflowException();
+      }
+      final int nextOffset = offsets.getAccessor().get(index+1);
+      values.getMutator().setArrayItem(nextOffset, holder);
+      offsets.getMutator().setSafe(index+1, nextOffset+1);
+    }
+
     public void addSafe(int index, Nullable${minor.class}Holder holder) {
       final int nextOffset = offsets.getAccessor().get(index+1);
       values.getMutator().setSafe(nextOffset, holder);
       offsets.getMutator().setSafe(index+1, nextOffset+1);
     }
 
+    public void addEntry(int index, Nullable${minor.class}Holder holder) throws VectorOverflowException {
+      if (index >= MAX_ROW_COUNT) {
+        throw new VectorOverflowException();
+      }
+      final int nextOffset = offsets.getAccessor().get(index+1);
+      values.getMutator().setArrayItem(nextOffset, holder);
+      offsets.getMutator().setSafe(index+1, nextOffset+1);
+    }
+
+    /**
+     * Backfill missing offsets from the given last written position to the
+     * given current write position. Used by the "new" size-safe column
+     * writers to allow skipping values. The <tt>set()</tt> and <tt>setSafe()</tt>
+     * <b>do not</b> fill empties. See DRILL-5529.
+     * @param lastWrite the position of the last valid write: the offset
+     * to be copied forward
+     * @param index the current write position to be initialized
+     */
+
+    public void fillEmptiesBounded(int lastWrite, int index)
+            throws VectorOverflowException {
+      if (index >= UInt4Vector.MAX_ROW_COUNT) {
+        throw new VectorOverflowException();
+      }
+      // If last write was 2, offsets are [0, 3, 6]
+      // If next write is 4, offsets must be: [0, 3, 6, 6, 6]
+      // Remember the offsets are one more than row count.
+      final int fillOffset = offsets.getAccessor().get(lastWrite+1);
+      final UInt4Vector.Mutator offsetMutator = offsets.getMutator();
+      for (int i = lastWrite; i < index; i++) {
+        offsetMutator.setSafe(i + 1, fillOffset);
+      }
+    }
+
     <#if (fields?size > 1) && !(minor.class == "Decimal9" || minor.class == "Decimal18" || minor.class == "Decimal28Sparse" || minor.class == "Decimal38Sparse" || minor.class == "Decimal28Dense" || minor.class == "Decimal38Dense")>
-    public void addSafe(int arrayIndex, <#list fields as field>${field.type} ${field.name}<#if field_has_next>, </#if></#list>) {
-      int nextOffset = offsets.getAccessor().get(arrayIndex+1);
+    public void addSafe(int rowIndex, <#list fields as field>${field.type} ${field.name}<#if field_has_next>, </#if></#list>) {
+      final int nextOffset = offsets.getAccessor().get(rowIndex+1);
       values.getMutator().setSafe(nextOffset, <#list fields as field>${field.name}<#if field_has_next>, </#if></#list>);
-      offsets.getMutator().setSafe(arrayIndex+1, nextOffset+1);
+      offsets.getMutator().setSafe(rowIndex+1, nextOffset+1);
+    }
+
+    public void addEntry(int rowIndex, <#list fields as field>${field.type} ${field.name}<#if field_has_next>, </#if></#list>) throws VectorOverflowException {
+      if (rowIndex >= MAX_ROW_COUNT) {
+        throw new VectorOverflowException();
+      }
+      final int nextOffset = offsets.getAccessor().get(rowIndex+1);
+      values.getMutator().setArrayItem(nextOffset, <#list fields as field>${field.name}<#if field_has_next>, </#if></#list>);
+      offsets.getMutator().setSafe(rowIndex+1, nextOffset+1);
     }
 
     </#if>
     <#if minor.class == "Decimal28Sparse" || minor.class == "Decimal38Sparse">
     public void addSafe(int index, BigDecimal value) {
-      int nextOffset = offsets.getAccessor().get(index+1);
+      final int nextOffset = offsets.getAccessor().get(index+1);
       values.getMutator().setSafe(nextOffset, value);
       offsets.getMutator().setSafe(index+1, nextOffset+1);
     }
 
+    public void addEntry(int index, BigDecimal value) throws VectorOverflowException {
+      if (index >= MAX_ROW_COUNT) {
+        throw new VectorOverflowException();
+      }
+      final int nextOffset = offsets.getAccessor().get(index+1);
+      values.getMutator().setArrayItem(nextOffset, value);
+      offsets.getMutator().setSafe(index+1, nextOffset+1);
+    }
+
     </#if>
     protected void add(int index, ${minor.class}Holder holder) {
-      int nextOffset = offsets.getAccessor().get(index+1);
+      final int nextOffset = offsets.getAccessor().get(index+1);
       values.getMutator().set(nextOffset, holder);
       offsets.getMutator().set(index+1, nextOffset+1);
     }
 
     public void add(int index, Repeated${minor.class}Holder holder) {
 
-      ${minor.class}Vector.Accessor accessor = holder.vector.getAccessor();
-      ${minor.class}Holder innerHolder = new ${minor.class}Holder();
+      final ${minor.class}Vector.Accessor accessor = holder.vector.getAccessor();
+      final ${minor.class}Holder innerHolder = new ${minor.class}Holder();
 
       for(int i = holder.start; i < holder.end; i++) {
         accessor.get(i, innerHolder);

http://git-wip-us.apache.org/repos/asf/drill/blob/92c9304f/exec/vector/src/main/codegen/templates/UnionVector.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/codegen/templates/UnionVector.java b/exec/vector/src/main/codegen/templates/UnionVector.java
index 93854e7..207e55a 100644
--- a/exec/vector/src/main/codegen/templates/UnionVector.java
+++ b/exec/vector/src/main/codegen/templates/UnionVector.java
@@ -1,4 +1,4 @@
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
@@ -80,6 +80,7 @@ public class UnionVector implements ValueVector {
     this.callBack = callBack;
   }
 
+  @Override
   public BufferAllocator getAllocator() {
     return allocator;
   }
@@ -248,6 +249,11 @@ public class UnionVector implements ValueVector {
     copyFrom(inIndex, outIndex, from);
   }
 
+  @Override
+  public void copyEntry(int toIndex, ValueVector from, int fromIndex) {
+    copyFromSafe(fromIndex, toIndex, (UnionVector) from);
+  }
+
   public ValueVector addVector(ValueVector v) {
     String name = v.getField().getType().getMinorType().name().toLowerCase();
     MajorType type = v.getField().getType();
@@ -485,4 +491,9 @@ public class UnionVector implements ValueVector {
     @Override
     public void generateTestData(int values) { }
   }
+
+  @Override
+  public void exchange(ValueVector other) {
+    throw new UnsupportedOperationException("Union vector does not yet support exchange()");
+  }
 }

http://git-wip-us.apache.org/repos/asf/drill/blob/92c9304f/exec/vector/src/main/codegen/templates/VariableLengthVectors.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/codegen/templates/VariableLengthVectors.java b/exec/vector/src/main/codegen/templates/VariableLengthVectors.java
index 581a9f8..9a9e178 100644
--- a/exec/vector/src/main/codegen/templates/VariableLengthVectors.java
+++ b/exec/vector/src/main/codegen/templates/VariableLengthVectors.java
@@ -42,20 +42,23 @@ package org.apache.drill.exec.vector;
  * ${minor.class}Vector implements a vector of variable width values.  Elements in the vector
  * are accessed by position from the logical start of the vector.  A fixed width offsetVector
  * is used to convert an element's position to it's offset from the start of the (0-based)
- * DrillBuf.  Size is inferred by adjacent elements.
- *   The width of each element is ${type.width} byte(s)
- *   The equivalent Java primitive is '${minor.javaType!type.javaType}'
- *
+ * DrillBuf. Size is inferred from adjacent elements.
+ * <ul>
+ * <li>The width of each element is ${type.width} byte(s). Note that the actual width is
+ * variable, this width is used as a guess for certain calculations.</li>
+ * <li>The equivalent Java primitive is '${minor.javaType!type.javaType}'<li>
+ * </ul>
  * NB: this class is automatically generated from ${.template_name} and ValueVectorTypes.tdd using FreeMarker.
  */
-public final class ${minor.class}Vector extends BaseDataValueVector implements VariableWidthVector{
+
+public final class ${minor.class}Vector extends BaseDataValueVector implements VariableWidthVector {
   private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(${minor.class}Vector.class);
 
   private static final int DEFAULT_RECORD_BYTE_COUNT = 8;
-  private static final int INITIAL_BYTE_COUNT = 4096 * DEFAULT_RECORD_BYTE_COUNT;
+  private static final int INITIAL_BYTE_COUNT = Math.min(INITIAL_VALUE_ALLOCATION * DEFAULT_RECORD_BYTE_COUNT, MAX_BUFFER_SIZE);
   private static final int MIN_BYTE_COUNT = 4096;
-
   public final static String OFFSETS_VECTOR_NAME = "$offsets$";
+
   private final MaterializedField offsetsField = MaterializedField.create(OFFSETS_VECTOR_NAME, Types.required(MinorType.UINT4));
   private final UInt${type.width}Vector offsetVector = new UInt${type.width}Vector(offsetsField, allocator);
   private final FieldReader reader = new ${minor.class}ReaderImpl(${minor.class}Vector.this);
@@ -239,6 +242,11 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements V
   }
 
   @Override
+  public void copyEntry(int toIndex, ValueVector from, int fromIndex) {
+    copyFromSafe(fromIndex, toIndex, (${minor.class}Vector) from);
+  }
+
+  @Override
   public int getAllocatedByteCount() {
     return offsetVector.getAllocatedByteCount() + super.getAllocatedByteCount();
   }
@@ -406,6 +414,13 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements V
     return mutator;
   }
 
+  @Override
+  public void exchange(ValueVector other) {
+    super.exchange(other);
+    ${minor.class}Vector target = (${minor.class}Vector) other;
+    offsetVector.exchange(target.offsetVector);
+  }
+
   public final class Accessor extends BaseValueVector.BaseAccessor implements VariableWidthAccessor {
     final UInt${type.width}Vector.Accessor oAccessor = offsetVector.getAccessor();
     public long getStartEnd(int index){
@@ -519,6 +534,10 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements V
       }
     }
 
+    public void setScalar(int index, byte[] bytes) throws VectorOverflowException {
+      setScalar(index, bytes, 0, bytes.length);
+    }
+
     /**
      * Set the variable length element at the specified index to the supplied byte array.
      *
@@ -537,7 +556,7 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements V
     public void setSafe(int index, ByteBuffer bytes, int start, int length) {
       assert index >= 0;
 
-      int currentOffset = offsetVector.getAccessor().get(index);
+      final int currentOffset = offsetVector.getAccessor().get(index);
       offsetVector.getMutator().setSafe(index + 1, currentOffset + length);
       try {
         data.setBytes(currentOffset, bytes, start, length);
@@ -549,6 +568,23 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements V
       }
     }
 
+    public void setScalar(int index, DrillBuf bytes, int start, int length) throws VectorOverflowException {
+      assert index >= 0;
+
+      if (index >= MAX_ROW_COUNT) {
+        throw new VectorOverflowException();
+      }
+      int currentOffset = offsetVector.getAccessor().get(index);
+      final int newSize = currentOffset + length;
+      if (newSize > MAX_BUFFER_SIZE) {
+        throw new VectorOverflowException();
+      }
+      while (! data.setBytesBounded(currentOffset, bytes, start, length)) {
+        reAlloc();
+      }
+      offsetVector.getMutator().setSafe(index + 1, newSize);
+    }
+
     public void setSafe(int index, byte[] bytes, int start, int length) {
       assert index >= 0;
 
@@ -565,6 +601,28 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements V
       }
     }
 
+    public void setScalar(int index, byte[] bytes, int start, int length) throws VectorOverflowException {
+      if (index >= MAX_ROW_COUNT) {
+        throw new VectorOverflowException();
+      }
+      setArrayItem(index, bytes, start, length);
+    }
+
+    public void setArrayItem(int index, byte[] bytes, int start, int length) throws VectorOverflowException {
+      assert index >= 0;
+
+      final int currentOffset = offsetVector.getAccessor().get(index);
+      final int newSize = currentOffset + length;
+      if (newSize > MAX_BUFFER_SIZE) {
+        throw new VectorOverflowException();
+      }
+
+      while (! data.setBytesBounded(currentOffset, bytes, start, length)) {
+        reAlloc();
+      }
+      offsetVector.getMutator().setSafe(index + 1, newSize);
+    }
+
     @Override
     public void setValueLengthSafe(int index, int length) {
       final int offset = offsetVector.getAccessor().get(index);
@@ -574,12 +632,11 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements V
       offsetVector.getMutator().setSafe(index + 1, offsetVector.getAccessor().get(index) + length);
     }
 
-
-    public void setSafe(int index, int start, int end, DrillBuf buffer){
+    public void setSafe(int index, int start, int end, DrillBuf buffer) {
       final int len = end - start;
       final int outputStart = offsetVector.data.get${(minor.javaType!type.javaType)?cap_first}(index * ${type.width});
 
-      offsetVector.getMutator().setSafe( index+1,  outputStart + len);
+      offsetVector.getMutator().setSafe(index+1,  outputStart + len);
       try{
         buffer.getBytes(start, data, outputStart, len);
       } catch (IndexOutOfBoundsException e) {
@@ -588,17 +645,42 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements V
         }
         buffer.getBytes(start, data, outputStart, len);
       }
+    }
+
+    public void setScalar(int index, int start, int end, DrillBuf buffer) throws VectorOverflowException {
+      if (index >= MAX_ROW_COUNT) {
+        throw new VectorOverflowException();
+      }
+      setArrayItem(index, start, end, buffer);
+    }
 
+    public void setArrayItem(int index, int start, int end, DrillBuf buffer) throws VectorOverflowException {
+      final int len = end - start;
+      final int outputStart = offsetVector.data.get${(minor.javaType!type.javaType)?cap_first}(index * ${type.width});
+      final int newSize = outputStart + len;
+      if (newSize > MAX_BUFFER_SIZE) {
+        throw new VectorOverflowException();
+      }
+
+      offsetVector.getMutator().setSafe(index+1, newSize);
+      try{
+        buffer.getBytes(start, data, outputStart, len);
+      } catch (IndexOutOfBoundsException e) {
+        while (data.capacity() < newSize) {
+          reAlloc();
+        }
+        buffer.getBytes(start, data, outputStart, len);
+      }
     }
 
-    public void setSafe(int index, Nullable${minor.class}Holder holder){
+    public void setSafe(int index, Nullable${minor.class}Holder holder) {
       assert holder.isSet == 1;
 
       final int start = holder.start;
       final int end =   holder.end;
       final int len = end - start;
 
-      int outputStart = offsetVector.data.get${(minor.javaType!type.javaType)?cap_first}(index * ${type.width});
+      final int outputStart = offsetVector.data.get${(minor.javaType!type.javaType)?cap_first}(index * ${type.width});
 
       try {
         holder.buffer.getBytes(start, data, outputStart, len);
@@ -608,15 +690,45 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements V
         }
         holder.buffer.getBytes(start, data, outputStart, len);
       }
-      offsetVector.getMutator().setSafe( index+1,  outputStart + len);
+      offsetVector.getMutator().setSafe(index+1,  outputStart + len);
+    }
+
+    public void setScalar(int index, Nullable${minor.class}Holder holder) throws VectorOverflowException {
+      if (index >= MAX_ROW_COUNT) {
+        throw new VectorOverflowException();
+      }
+      setArrayItem(index, holder);
     }
 
-    public void setSafe(int index, ${minor.class}Holder holder){
+    public void setArrayItem(int index, Nullable${minor.class}Holder holder) throws VectorOverflowException {
+      assert holder.isSet == 1;
+
       final int start = holder.start;
       final int end =   holder.end;
       final int len = end - start;
+
       final int outputStart = offsetVector.data.get${(minor.javaType!type.javaType)?cap_first}(index * ${type.width});
+      final int newSize = outputStart + len;
+      if (newSize > MAX_BUFFER_SIZE) {
+        throw new VectorOverflowException();
+      }
+
+      try {
+        holder.buffer.getBytes(start, data, outputStart, len);
+      } catch (IndexOutOfBoundsException e) {
+        while (data.capacity() < newSize) {
+          reAlloc();
+        }
+        holder.buffer.getBytes(start, data, outputStart, len);
+      }
+      offsetVector.getMutator().setSafe(index+1, newSize);
+    }
 
+    public void setSafe(int index, ${minor.class}Holder holder) {
+      final int start = holder.start;
+      final int end =   holder.end;
+      final int len = end - start;
+      final int outputStart = offsetVector.data.get${(minor.javaType!type.javaType)?cap_first}(index * ${type.width});
 
       try {
         holder.buffer.getBytes(start, data, outputStart, len);
@@ -629,6 +741,68 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements V
       offsetVector.getMutator().setSafe( index+1,  outputStart + len);
     }
 
+    public void setScalar(int index, ${minor.class}Holder holder) throws VectorOverflowException {
+      if (index >= MAX_ROW_COUNT) {
+        throw new VectorOverflowException();
+      }
+      setArrayItem(index, holder);
+   }
+
+    public void setArrayItem(int index, ${minor.class}Holder holder) throws VectorOverflowException {
+      final int start = holder.start;
+      final int end =   holder.end;
+      final int len = end - start;
+      final int outputStart = offsetVector.data.get${(minor.javaType!type.javaType)?cap_first}(index * ${type.width});
+      final int newSize = outputStart + len;
+      if (newSize > MAX_BUFFER_SIZE) {
+        throw new VectorOverflowException();
+      }
+
+      try {
+        holder.buffer.getBytes(start, data, outputStart, len);
+      } catch (IndexOutOfBoundsException e) {
+        while(data.capacity() < newSize) {
+          reAlloc();
+        }
+        holder.buffer.getBytes(start, data, outputStart, len);
+      }
+      offsetVector.getMutator().setSafe( index+1, newSize);
+    }
+
+    /**
+     * Backfill missing offsets from the given last written position to the
+     * given current write position. Used by the "new" size-safe column
+     * writers to allow skipping values. The <tt>set()</tt> and <tt>setSafe()</tt>
+     * <b>do not</b> fill empties. See DRILL-5529.
+     * @param lastWrite the position of the last valid write: the offset
+     * to be copied forward
+     * @param index the current write position filling occurs up to,
+     * but not including, this position
+     * @throws VectorOverflowException if the item was written, false if the index would
+     * overfill the vector
+     */
+
+    public void fillEmptiesBounded(int lastWrite, int index)
+            throws VectorOverflowException {
+
+      // Index is the next write index, which might be "virtual",
+      // that is, past the last row at EOF. This check only protects
+      // the actual data written here, which is up to index-1.
+
+      if (index > UInt4Vector.MAX_ROW_COUNT) {
+        throw new VectorOverflowException();
+      }
+      // If last write was 2, offsets are [0, 3, 6]
+      // If next write is 4, offsets must be: [0, 3, 6, 6, 6]
+      // Remember the offsets are one more than row count.
+
+      final int fillOffset = offsetVector.getAccessor().get(lastWrite+1);
+      final UInt4Vector.Mutator offsetMutator = offsetVector.getMutator();
+      for (int i = lastWrite; i < index; i++) {
+        offsetMutator.setSafe(i + 1, fillOffset);
+      }
+    }
+
     protected void set(int index, int start, int length, DrillBuf buffer){
       assert index >= 0;
       final int currentOffset = offsetVector.getAccessor().get(index);
@@ -651,6 +825,19 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements V
       data.setBytes(currentOffset, holder.buffer, holder.start, length);
     }
 
+  <#if (minor.class == "VarChar")>
+    public void setScalar(int index, String value) throws VectorOverflowException {
+      if (index >= MAX_ROW_COUNT) {
+        throw new VectorOverflowException();
+      }
+      // Treat a null string as an empty string.
+      if (value != null) {
+        byte encoded[] = value.getBytes(Charsets.UTF_8);
+        setScalar(index, encoded, 0, encoded.length);
+      }
+    }
+
+  </#if>
     @Override
     public void setValueCount(int valueCount) {
       final int currentByteCapacity = getByteCapacity();
@@ -685,7 +872,6 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements V
     }
   }
 }
-
 </#if> <#-- type.major -->
 </#list>
 </#list>

http://git-wip-us.apache.org/repos/asf/drill/blob/92c9304f/exec/vector/src/main/java/org/apache/drill/exec/vector/BaseDataValueVector.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/java/org/apache/drill/exec/vector/BaseDataValueVector.java b/exec/vector/src/main/java/org/apache/drill/exec/vector/BaseDataValueVector.java
index 4def5b8..5ce58ed 100644
--- a/exec/vector/src/main/java/org/apache/drill/exec/vector/BaseDataValueVector.java
+++ b/exec/vector/src/main/java/org/apache/drill/exec/vector/BaseDataValueVector.java
@@ -1,4 +1,4 @@
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
@@ -24,7 +24,7 @@ import org.apache.drill.exec.record.MaterializedField;
 
 public abstract class BaseDataValueVector extends BaseValueVector {
 
-  protected final static byte[] emptyByteArray = new byte[]{}; // Nullable vectors use this
+  protected final static byte[] emptyByteArray = new byte[0]; // Nullable vectors use this
 
   protected DrillBuf data;
 
@@ -78,9 +78,7 @@ public abstract class BaseDataValueVector extends BaseValueVector {
     return data.writerIndex();
   }
 
-  public DrillBuf getBuffer() {
-    return data;
-  }
+  public DrillBuf getBuffer() { return data; }
 
   /**
    * This method has a similar effect of allocateNew() without actually clearing and reallocating
@@ -89,7 +87,16 @@ public abstract class BaseDataValueVector extends BaseValueVector {
   public void reset() {}
 
   @Override
-  public int getAllocatedByteCount() {
-    return data.capacity();
+  public int getAllocatedByteCount() { return data.capacity(); }
+
+  @Override
+  public void exchange(ValueVector other) {
+    BaseDataValueVector target = (BaseDataValueVector) other;
+    DrillBuf temp = data;
+    data = target.data;
+    target.data = temp;
+    getReader().reset();
+    getMutator().exchange(target.getMutator());
+    // No state in an Accessor to reset
   }
 }

http://git-wip-us.apache.org/repos/asf/drill/blob/92c9304f/exec/vector/src/main/java/org/apache/drill/exec/vector/BaseValueVector.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/java/org/apache/drill/exec/vector/BaseValueVector.java b/exec/vector/src/main/java/org/apache/drill/exec/vector/BaseValueVector.java
index a0d5f65..f4a5847 100644
--- a/exec/vector/src/main/java/org/apache/drill/exec/vector/BaseValueVector.java
+++ b/exec/vector/src/main/java/org/apache/drill/exec/vector/BaseValueVector.java
@@ -1,4 +1,4 @@
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
@@ -32,6 +32,13 @@ import org.apache.drill.exec.record.TransferPair;
 public abstract class BaseValueVector implements ValueVector {
 //  private static final Logger logger = LoggerFactory.getLogger(BaseValueVector.class);
 
+  /**
+   * Physical maximum allocation. This is the value prior to Drill 1.11.
+   * This size causes memory fragmentation. Please use
+   * {@link ValueVector#MAX_BUFFER_SIZE} in new code.
+   */
+
+  @Deprecated
   public static final int MAX_ALLOCATION_SIZE = Integer.MAX_VALUE;
   public static final int INITIAL_VALUE_ALLOCATION = 4096;
 
@@ -101,6 +108,10 @@ public abstract class BaseValueVector implements ValueVector {
     //TODO: consider making mutator stateless(if possible) on another issue.
     @Override
     public void reset() {}
+
+    // TODO: If mutator becomes stateless, remove this method.
+    @Override
+    public void exchange(ValueVector.Mutator other) { }
   }
 
   @Override

http://git-wip-us.apache.org/repos/asf/drill/blob/92c9304f/exec/vector/src/main/java/org/apache/drill/exec/vector/BitVector.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/java/org/apache/drill/exec/vector/BitVector.java b/exec/vector/src/main/java/org/apache/drill/exec/vector/BitVector.java
index a6c0cea..0062e77 100644
--- a/exec/vector/src/main/java/org/apache/drill/exec/vector/BitVector.java
+++ b/exec/vector/src/main/java/org/apache/drill/exec/vector/BitVector.java
@@ -1,4 +1,4 @@
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
@@ -39,6 +39,38 @@ import org.apache.drill.exec.vector.complex.reader.FieldReader;
 public final class BitVector extends BaseDataValueVector implements FixedWidthVector {
   static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(BitVector.class);
 
+  /**
+   * Width of each fixed-width value.
+   */
+
+  public static final int VALUE_WIDTH = 1;
+
+  /**
+   * Maximum number of values that this fixed-width vector can hold
+   * and stay below the maximum vector size limit. This is the limit
+   * enforced when the vector is used to hold values in a repeated
+   * vector.
+   */
+
+  public static final int MAX_CAPACITY = MAX_BUFFER_SIZE / VALUE_WIDTH;
+
+  /**
+   * Maximum number of values that this fixed-width vector can hold
+   * and stay below the maximum vector size limit and/or stay below
+   * the maximum item count. This lis the limit enforced when the
+   * vector is used to hold required or nullable values.
+   */
+
+  public static final int MAX_COUNT = Math.min(MAX_ROW_COUNT, MAX_CAPACITY);
+
+  /**
+   * Actual maximum vector size, in bytes, given the number of fixed-width
+   * values that either fit in the maximum overall vector size, or that
+   * is no larger than the maximum vector item count.
+   */
+
+  public static final int NET_MAX_SIZE = VALUE_WIDTH * MAX_COUNT;
+
   private final FieldReader reader = new BitReaderImpl(BitVector.this);
   private final Accessor accessor = new Accessor();
   private final Mutator mutator = new Mutator();
@@ -72,7 +104,7 @@ public final class BitVector extends BaseDataValueVector implements FixedWidthVe
 
   @Override
   public int getValueCapacity() {
-    return (int)Math.min((long)Integer.MAX_VALUE, data.capacity() * 8L);
+    return (int) Math.min((long)Integer.MAX_VALUE, data.capacity() * 8L);
   }
 
   private int getByteIndex(int index) {
@@ -183,6 +215,11 @@ public final class BitVector extends BaseDataValueVector implements FixedWidthVe
   }
 
   @Override
+  public void copyEntry(int toIndex, ValueVector from, int fromIndex) {
+    copyFrom(fromIndex, toIndex, (BitVector) from);
+  }
+
+  @Override
   public void load(SerializedField metadata, DrillBuf buffer) {
     Preconditions.checkArgument(this.field.getPath().equals(metadata.getNamePart().getName()), "The field %s doesn't match the provided metadata %s.", this.field, metadata);
     final int valueCount = metadata.getValueCount();
@@ -221,7 +258,6 @@ public final class BitVector extends BaseDataValueVector implements FixedWidthVe
     return new TransferImpl((BitVector) to);
   }
 
-
   public void transferTo(BitVector target) {
     target.clear();
     if (target.data != null) {
@@ -401,6 +437,20 @@ public final class BitVector extends BaseDataValueVector implements FixedWidthVe
       set(index, value);
     }
 
+    public void setScalar(int index, int value) throws VectorOverflowException {
+      if (index >= MAX_COUNT) {
+        throw new VectorOverflowException();
+      }
+      setSafe(index, value);
+    }
+
+    public void setArrayItem(int index, int value) throws VectorOverflowException {
+      if (index >= MAX_CAPACITY) {
+        throw new VectorOverflowException();
+      }
+      setSafe(index, value);
+    }
+
     public void setSafe(int index, BitHolder holder) {
       while(index >= getValueCapacity()) {
         reAlloc();
@@ -408,6 +458,20 @@ public final class BitVector extends BaseDataValueVector implements FixedWidthVe
       set(index, holder.value);
     }
 
+    public void setScalar(int index, BitHolder holder) throws VectorOverflowException {
+      if (index >= MAX_COUNT) {
+        throw new VectorOverflowException();
+      }
+      setSafe(index, holder);
+    }
+
+    public void setArrayItem(int index, BitHolder holder) throws VectorOverflowException {
+      if (index >= MAX_CAPACITY) {
+        throw new VectorOverflowException();
+      }
+      setSafe(index, holder);
+    }
+
     public void setSafe(int index, NullableBitHolder holder) {
       while(index >= getValueCapacity()) {
         reAlloc();
@@ -415,6 +479,20 @@ public final class BitVector extends BaseDataValueVector implements FixedWidthVe
       set(index, holder.value);
     }
 
+    public void setScalar(int index, NullableBitHolder holder) throws VectorOverflowException {
+      if (index >= MAX_COUNT) {
+        throw new VectorOverflowException();
+      }
+      setSafe(index, holder);
+    }
+
+    public void setArrayItem(int index, NullableBitHolder holder) throws VectorOverflowException {
+      if (index >= MAX_CAPACITY) {
+        throw new VectorOverflowException();
+      }
+      setSafe(index, holder);
+    }
+
     @Override
     public final void setValueCount(int valueCount) {
       int currentValueCapacity = getValueCapacity();
@@ -441,7 +519,6 @@ public final class BitVector extends BaseDataValueVector implements FixedWidthVe
       }
       setValueCount(values);
     }
-
   }
 
   @Override

http://git-wip-us.apache.org/repos/asf/drill/blob/92c9304f/exec/vector/src/main/java/org/apache/drill/exec/vector/FixedWidthVector.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/java/org/apache/drill/exec/vector/FixedWidthVector.java b/exec/vector/src/main/java/org/apache/drill/exec/vector/FixedWidthVector.java
index c2781eb..a9a1631 100644
--- a/exec/vector/src/main/java/org/apache/drill/exec/vector/FixedWidthVector.java
+++ b/exec/vector/src/main/java/org/apache/drill/exec/vector/FixedWidthVector.java
@@ -1,4 +1,4 @@
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
@@ -18,7 +18,7 @@
 package org.apache.drill.exec.vector;
 
 
-public interface FixedWidthVector extends ValueVector{
+public interface FixedWidthVector extends ValueVector {
 
   /**
    * Allocate a new memory space for this vector.  Must be called prior to using the ValueVector.
@@ -31,5 +31,4 @@ public interface FixedWidthVector extends ValueVector{
  * Zero out the underlying buffer backing this vector.
  */
   void zeroVector();
-
 }

http://git-wip-us.apache.org/repos/asf/drill/blob/92c9304f/exec/vector/src/main/java/org/apache/drill/exec/vector/ObjectVector.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/java/org/apache/drill/exec/vector/ObjectVector.java b/exec/vector/src/main/java/org/apache/drill/exec/vector/ObjectVector.java
index f69dc98..bd8566d 100644
--- a/exec/vector/src/main/java/org/apache/drill/exec/vector/ObjectVector.java
+++ b/exec/vector/src/main/java/org/apache/drill/exec/vector/ObjectVector.java
@@ -1,4 +1,4 @@
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
@@ -32,11 +32,12 @@ import org.apache.drill.exec.record.TransferPair;
 import org.apache.drill.exec.vector.complex.reader.FieldReader;
 
 public class ObjectVector extends BaseValueVector {
+  private final int ALLOCATION_SIZE = 4096;
+
   private final Accessor accessor = new Accessor();
   private final Mutator mutator = new Mutator();
   private int maxCount = 0;
   private int count = 0;
-  private int allocationSize = 4096;
 
   private List<Object[]> objectArrayList = new ArrayList<>();
 
@@ -45,8 +46,8 @@ public class ObjectVector extends BaseValueVector {
   }
 
   public void addNewArray() {
-    objectArrayList.add(new Object[allocationSize]);
-    maxCount += allocationSize;
+    objectArrayList.add(new Object[ALLOCATION_SIZE]);
+    maxCount += ALLOCATION_SIZE;
   }
 
   @Override
@@ -57,11 +58,11 @@ public class ObjectVector extends BaseValueVector {
   public final class Mutator implements ValueVector.Mutator {
 
     public void set(int index, Object obj) {
-      int listOffset = index / allocationSize;
+      int listOffset = index / ALLOCATION_SIZE;
       if (listOffset >= objectArrayList.size()) {
         addNewArray();
       }
-      objectArrayList.get(listOffset)[index % allocationSize] = obj;
+      objectArrayList.get(listOffset)[index % ALLOCATION_SIZE] = obj;
     }
 
     public boolean setSafe(int index, long value) {
@@ -94,6 +95,9 @@ public class ObjectVector extends BaseValueVector {
     @Override
     public void generateTestData(int values) {
     }
+
+    @Override
+    public void exchange(ValueVector.Mutator other) { }
   }
 
   @Override
@@ -161,6 +165,11 @@ public class ObjectVector extends BaseValueVector {
   }
 
   @Override
+  public void copyEntry(int toIndex, ValueVector from, int fromIndex) {
+    throw new UnsupportedOperationException("ObjectVector does not support this");
+  }
+
+  @Override
   public int getValueCapacity() {
     return maxCount;
   }
@@ -198,11 +207,11 @@ public class ObjectVector extends BaseValueVector {
   public final class Accessor extends BaseAccessor {
     @Override
     public Object getObject(int index) {
-      int listOffset = index / allocationSize;
+      int listOffset = index / ALLOCATION_SIZE;
       if (listOffset >= objectArrayList.size()) {
         addNewArray();
       }
-      return objectArrayList.get(listOffset)[index % allocationSize];
+      return objectArrayList.get(listOffset)[index % ALLOCATION_SIZE];
     }
 
     @Override
@@ -230,4 +239,18 @@ public class ObjectVector extends BaseValueVector {
     // Values not stored in direct memory?
     return 0;
   }
+
+  @Override
+  public void exchange(ValueVector other) {
+    ObjectVector target = (ObjectVector) other;
+    List<Object[]> tempList = objectArrayList;
+    objectArrayList = target.objectArrayList;
+    target.objectArrayList = tempList;
+    int tempCount = count;
+    count = target.count;
+    target.count = tempCount;
+    tempCount = maxCount;
+    maxCount = target.maxCount;
+    target.maxCount = tempCount;
+  }
 }

http://git-wip-us.apache.org/repos/asf/drill/blob/92c9304f/exec/vector/src/main/java/org/apache/drill/exec/vector/ValueVector.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/java/org/apache/drill/exec/vector/ValueVector.java b/exec/vector/src/main/java/org/apache/drill/exec/vector/ValueVector.java
index f4c7935..3bc43fa 100644
--- a/exec/vector/src/main/java/org/apache/drill/exec/vector/ValueVector.java
+++ b/exec/vector/src/main/java/org/apache/drill/exec/vector/ValueVector.java
@@ -1,4 +1,4 @@
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
@@ -40,14 +40,17 @@ import org.apache.drill.exec.vector.complex.reader.FieldReader;
  * There are a few "rules" around vectors:
  *
  * <ul>
- *   <li>values need to be written in order (e.g. index 0, 1, 2, 5)</li>
- *   <li>null vectors start with all values as null before writing anything</li>
- *   <li>for variable width types, the offset vector should be all zeros before writing</li>
- *   <li>you must call setValueCount before a vector can be read</li>
- *   <li>you should never write to a vector once it has been read.</li>
+ *   <li>Values need to be written in order (e.g. index 0, 1, 2, 5).</li>
+ *   <li>Null vectors start with all values as null before writing anything.</li>
+ *   <li>For variable width types, the offset vector should be all zeros before writing.</li>
+ *   <li>You must call setValueCount before a vector can be read.</li>
+ *   <li>You should never write to a vector once it has been read.</li>
+ *   <li>Vectors may not grow larger than the number of bytes specified
+ *   in {@link #MAX_BUFFER_SIZE} to prevent memory fragmentation. Use the
+ *   <tt>setBounded()</tt> methods in the mutator to enforce this rule.</li>
  * </ul>
  *
- * Please note that the current implementation doesn't enfore those rules, hence we may find few places that
+ * Please note that the current implementation doesn't enforce those rules, hence we may find few places that
  * deviate from these rules (e.g. offset vectors in Variable Length and Repeated vector)
  *
  * This interface "should" strive to guarantee this order of operation:
@@ -56,6 +59,29 @@ import org.apache.drill.exec.vector.complex.reader.FieldReader;
  * </blockquote>
  */
 public interface ValueVector extends Closeable, Iterable<ValueVector> {
+
+  /**
+   * Maximum allowed size of the buffer backing a value vector.
+   */
+
+  int MAX_BUFFER_SIZE = VectorUtils.maxSize();
+
+  /**
+   * Debug-time system option that artificially limits vector lengths
+   * for testing. Must be set prior to the first reference to this
+   * class. (Made deliberately difficult to prevent misuse...)
+   */
+
+  String MAX_BUFFER_SIZE_KEY = "drill.max_vector";
+
+  /**
+   * Maximum allowed row count in a vector. Repeated vectors
+   * may have more items, but can have no more than this number
+   * or arrays. Limited by 2-byte length in SV2: 65536 = 2<sup>16</sup>.
+   */
+
+  int MAX_ROW_COUNT = Character.MAX_VALUE + 1;
+
   /**
    * Allocate new buffers. ValueVector implements logic to determine how much to allocate.
    * @throws OutOfMemoryException Thrown if no memory can be allocated.
@@ -64,7 +90,7 @@ public interface ValueVector extends Closeable, Iterable<ValueVector> {
 
   /**
    * Allocates new buffers. ValueVector implements logic to determine how much to allocate.
-   * @return Returns true if allocation was succesful.
+   * @return Returns true if allocation was successful.
    */
   boolean allocateNewSafe();
 
@@ -175,6 +201,8 @@ public interface ValueVector extends Closeable, Iterable<ValueVector> {
    */
   void load(SerializedField metadata, DrillBuf buffer);
 
+  void copyEntry(int toIndex, ValueVector from, int fromIndex);
+
   /**
    * Return the total memory consumed by all buffers within this vector.
    */
@@ -188,6 +216,13 @@ public interface ValueVector extends Closeable, Iterable<ValueVector> {
   int getPayloadByteCount();
 
   /**
+   * Exchange state with another value vector of the same type.
+   * Used to implement look-ahead writers.
+   */
+
+  void exchange(ValueVector other);
+
+  /**
    * An abstraction that is used to read from this vector instance.
    */
   interface Accessor {
@@ -211,7 +246,7 @@ public interface ValueVector extends Closeable, Iterable<ValueVector> {
   }
 
   /**
-   * An abstractiong that is used to write into this vector instance.
+   * An abstraction that is used to write into this vector instance.
    */
   interface Mutator {
     /**
@@ -231,5 +266,12 @@ public interface ValueVector extends Closeable, Iterable<ValueVector> {
      */
     @Deprecated
     void generateTestData(int values);
+
+    /**
+     * Exchanges state with the mutator of another mutator. Used when exchanging
+     * state with another vector.
+     */
+
+    void exchange(Mutator other);
   }
 }

http://git-wip-us.apache.org/repos/asf/drill/blob/92c9304f/exec/vector/src/main/java/org/apache/drill/exec/vector/VectorOverflowException.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/java/org/apache/drill/exec/vector/VectorOverflowException.java b/exec/vector/src/main/java/org/apache/drill/exec/vector/VectorOverflowException.java
new file mode 100644
index 0000000..8b54966
--- /dev/null
+++ b/exec/vector/src/main/java/org/apache/drill/exec/vector/VectorOverflowException.java
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.vector;
+
+/**
+ * Indicates that an attempt to write to a vector overflowed the vector
+ * bounds: either the limit on values or the size of the buffer backing
+ * the vector. This is an expected exception: code must catch it and
+ * properly handle the partially-written, incomplete last row.
+ */
+
+@SuppressWarnings("serial")
+public class VectorOverflowException extends Exception {
+
+}

http://git-wip-us.apache.org/repos/asf/drill/blob/92c9304f/exec/vector/src/main/java/org/apache/drill/exec/vector/VectorUtils.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/java/org/apache/drill/exec/vector/VectorUtils.java b/exec/vector/src/main/java/org/apache/drill/exec/vector/VectorUtils.java
new file mode 100644
index 0000000..6b29eb2
--- /dev/null
+++ b/exec/vector/src/main/java/org/apache/drill/exec/vector/VectorUtils.java
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.vector;
+
+public class VectorUtils {
+
+  /**
+   * Vectors cannot be any larger than the Netty memory allocation
+   * block size.
+   */
+
+  private static final int ABSOLUTE_MAX_SIZE = 16 * 1024 * 1024;
+
+  /**
+   * Minimum size selected to prevent pathological performance if vectors
+   * are limited to an unusably small size. This limit is a judgment call,
+   * not based on any known limits.
+   */
+
+  private static final int ABSOLUTE_MIN_SIZE = 16 * 1024;
+
+  private VectorUtils() { }
+
+  /**
+   * Static function called once per run to compute the maximum
+   * vector size, in bytes. Normally uses the hard-coded limit,
+   * but allows setting a system property to override the limit
+   * for testing. The configured value must be within reasonable
+   * bounds.
+   * @return the maximum vector size, in bytes
+   */
+
+  static int maxSize() {
+    String prop = System.getProperty( ValueVector.MAX_BUFFER_SIZE_KEY );
+    int value = ABSOLUTE_MAX_SIZE;
+    if (prop != null) {
+      try {
+        value = Integer.parseInt(prop);
+        value = Math.max(value, ABSOLUTE_MIN_SIZE);
+        value = Math.min(value, ABSOLUTE_MAX_SIZE);
+      } catch (NumberFormatException e) {
+        // Ignore
+      }
+    }
+    return value;
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/drill/blob/92c9304f/exec/vector/src/main/java/org/apache/drill/exec/vector/ZeroVector.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/java/org/apache/drill/exec/vector/ZeroVector.java b/exec/vector/src/main/java/org/apache/drill/exec/vector/ZeroVector.java
index 9181f20..9a0b6be 100644
--- a/exec/vector/src/main/java/org/apache/drill/exec/vector/ZeroVector.java
+++ b/exec/vector/src/main/java/org/apache/drill/exec/vector/ZeroVector.java
@@ -1,4 +1,4 @@
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
@@ -53,19 +53,13 @@ public class ZeroVector implements ValueVector {
 
   private final Accessor defaultAccessor = new Accessor() {
     @Override
-    public Object getObject(int index) {
-      return null;
-    }
+    public Object getObject(int index) { return null; }
 
     @Override
-    public int getValueCount() {
-      return 0;
-    }
+    public int getValueCount() { return 0; }
 
     @Override
-    public boolean isNull(int index) {
-      return true;
-    }
+    public boolean isNull(int index) { return true; }
   };
 
   private final Mutator defaultMutator = new Mutator() {
@@ -77,6 +71,9 @@ public class ZeroVector implements ValueVector {
 
     @Override
     public void generateTestData(int values) { }
+
+    @Override
+    public void exchange(Mutator other) { }
   };
 
   public ZeroVector() { }
@@ -88,9 +85,7 @@ public class ZeroVector implements ValueVector {
   public void clear() { }
 
   @Override
-  public MaterializedField getField() {
-    return field;
-  }
+  public MaterializedField getField() { return field; }
 
   @Override
   public TransferPair getTransferPair(BufferAllocator allocator) {
@@ -112,14 +107,10 @@ public class ZeroVector implements ValueVector {
   }
 
   @Override
-  public int getBufferSize() {
-    return 0;
-  }
+  public int getBufferSize() { return 0; }
 
   @Override
-  public int getBufferSizeFor(final int valueCount) {
-    return 0;
-  }
+  public int getBufferSizeFor(final int valueCount) { return 0; }
 
   @Override
   public DrillBuf[] getBuffers(boolean clear) {
@@ -132,9 +123,7 @@ public class ZeroVector implements ValueVector {
   }
 
   @Override
-  public boolean allocateNewSafe() {
-    return true;
-  }
+  public boolean allocateNewSafe() { return true; }
 
   @Override
   public BufferAllocator getAllocator() {
@@ -145,45 +134,35 @@ public class ZeroVector implements ValueVector {
   public void setInitialCapacity(int numRecords) { }
 
   @Override
-  public int getValueCapacity() {
-    return 0;
-  }
+  public int getValueCapacity() { return 0; }
 
   @Override
-  public TransferPair getTransferPair(String ref, BufferAllocator allocator) {
-    return defaultPair;
-  }
+  public TransferPair getTransferPair(String ref, BufferAllocator allocator) { return defaultPair; }
 
   @Override
-  public TransferPair makeTransferPair(ValueVector target) {
-    return defaultPair;
-  }
+  public TransferPair makeTransferPair(ValueVector target) { return defaultPair; }
 
   @Override
-  public Accessor getAccessor() {
-    return defaultAccessor;
-  }
+  public Accessor getAccessor() { return defaultAccessor; }
 
   @Override
-  public Mutator getMutator() {
-    return defaultMutator;
-  }
+  public Mutator getMutator() { return defaultMutator; }
 
   @Override
-  public FieldReader getReader() {
-    return NullReader.INSTANCE;
-  }
+  public FieldReader getReader() { return NullReader.INSTANCE; }
 
   @Override
   public void load(UserBitShared.SerializedField metadata, DrillBuf buffer) { }
 
   @Override
-  public int getAllocatedByteCount() {
-    return 0;
-  }
+  public void copyEntry(int toIndex, ValueVector from, int fromIndex) { }
 
   @Override
-  public int getPayloadByteCount() {
-    return 0;
-  }
+  public int getAllocatedByteCount() { return 0; }
+
+  @Override
+  public int getPayloadByteCount() { return 0; }
+
+  @Override
+  public void exchange(ValueVector other) { }
 }

http://git-wip-us.apache.org/repos/asf/drill/blob/92c9304f/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/BaseRepeatedValueVector.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/BaseRepeatedValueVector.java b/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/BaseRepeatedValueVector.java
index 8a54535..5b8f44d 100644
--- a/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/BaseRepeatedValueVector.java
+++ b/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/BaseRepeatedValueVector.java
@@ -1,4 +1,4 @@
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
@@ -209,7 +209,6 @@ public abstract class BaseRepeatedValueVector extends BaseValueVector implements
     vector = v;
   }
 
-
   @Override
   public int getAllocatedByteCount() {
     return offsets.getAllocatedByteCount() + vector.getAllocatedByteCount();
@@ -220,6 +219,13 @@ public abstract class BaseRepeatedValueVector extends BaseValueVector implements
     return offsets.getPayloadByteCount() + vector.getPayloadByteCount();
   }
 
+  @Override
+  public void exchange(ValueVector other) {
+    BaseRepeatedValueVector target = (BaseRepeatedValueVector) other;
+    vector.exchange(target.vector);
+    offsets.exchange(target.offsets);
+  }
+
   public abstract class BaseRepeatedAccessor extends BaseValueVector.BaseAccessor implements RepeatedAccessor {
 
     @Override
@@ -259,6 +265,14 @@ public abstract class BaseRepeatedValueVector extends BaseValueVector implements
       setValueCount(index+1);
     }
 
+    public boolean startNewValueBounded(int index) {
+      if (index >= MAX_ROW_COUNT) {
+        return false;
+      }
+      startNewValue(index);
+      return true;
+    }
+
     @Override
     public void setValueCount(int valueCount) {
       // TODO: populate offset end points

http://git-wip-us.apache.org/repos/asf/drill/blob/92c9304f/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/ListVector.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/ListVector.java b/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/ListVector.java
index f71baa7..7f0e939 100644
--- a/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/ListVector.java
+++ b/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/ListVector.java
@@ -1,5 +1,4 @@
-/*******************************************************************************
-
+/*
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
@@ -95,6 +94,11 @@ public class ListVector extends BaseRepeatedValueVector {
   }
 
   @Override
+  public void copyEntry(int toIndex, ValueVector from, int fromIndex) {
+    copyFromSafe(fromIndex, toIndex, (ListVector) from);
+  }
+
+  @Override
   public ValueVector getDataVector() {
     return vector;
   }

http://git-wip-us.apache.org/repos/asf/drill/blob/92c9304f/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/MapVector.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/MapVector.java b/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/MapVector.java
index af1ec8e..f755081 100644
--- a/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/MapVector.java
+++ b/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/MapVector.java
@@ -1,4 +1,4 @@
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
@@ -90,6 +90,11 @@ public class MapVector extends AbstractMapVector {
   }
 
   @Override
+  public void copyEntry(int toIndex, ValueVector from, int fromIndex) {
+    copyFromSafe(fromIndex, toIndex, (MapVector) from);
+  }
+
+  @Override
   protected boolean supportsDirectRead() {
     return true;
   }
@@ -302,6 +307,13 @@ public class MapVector extends AbstractMapVector {
     return mutator;
   }
 
+  @Override
+  public void exchange(ValueVector other) {
+    // Exchange is used for look-ahead writers, but writers manage
+    // map member vectors directly.
+    throw new UnsupportedOperationException("Exchange() not supported for maps");
+  }
+
   public class Accessor extends BaseValueVector.BaseAccessor {
 
     @Override

http://git-wip-us.apache.org/repos/asf/drill/blob/92c9304f/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/RepeatedListVector.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/RepeatedListVector.java b/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/RepeatedListVector.java
index b5c97bf..969c141 100644
--- a/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/RepeatedListVector.java
+++ b/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/RepeatedListVector.java
@@ -1,4 +1,4 @@
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
@@ -118,7 +118,6 @@ public class RepeatedListVector extends AbstractContainerVector
       }
     }
 
-
     public class DelegateTransferPair implements TransferPair {
       private final DelegateRepeatedVector target;
       private final TransferPair[] children;
@@ -218,6 +217,10 @@ public class RepeatedListVector extends AbstractContainerVector
       ephPair.copyValueSafe(fromIndex, thisIndex);
     }
 
+    @Override
+    public void copyEntry(int toIndex, ValueVector from, int fromIndex) {
+      copyFromSafe(fromIndex, toIndex, (DelegateRepeatedVector) from);
+    }
   }
 
   protected class RepeatedListTransferPair implements TransferPair {
@@ -428,6 +431,11 @@ public class RepeatedListVector extends AbstractContainerVector
   }
 
   @Override
+  public void copyEntry(int toIndex, ValueVector from, int fromIndex) {
+    copyFromSafe(fromIndex, toIndex, (RepeatedListVector) from);
+  }
+
+  @Override
   public int getAllocatedByteCount() {
     return delegate.getAllocatedByteCount();
   }
@@ -436,4 +444,11 @@ public class RepeatedListVector extends AbstractContainerVector
   public int getPayloadByteCount() {
     return delegate.getPayloadByteCount();
   }
+
+  @Override
+  public void exchange(ValueVector other) {
+    // TODO: Figure out how to test this scenario, then what to do...
+    throw new UnsupportedOperationException("Exchange() not yet supported for repeated lists");
+  }
+
 }

http://git-wip-us.apache.org/repos/asf/drill/blob/92c9304f/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/RepeatedMapVector.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/RepeatedMapVector.java b/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/RepeatedMapVector.java
index d930728..7ff36a7 100644
--- a/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/RepeatedMapVector.java
+++ b/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/RepeatedMapVector.java
@@ -1,4 +1,4 @@
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
@@ -390,7 +390,6 @@ public class RepeatedMapVector extends AbstractMapVector
     }
   }
 
-
   transient private RepeatedMapTransferPair ephPair;
 
   public void copyFromSafe(int fromIndex, int thisIndex, RepeatedMapVector from) {
@@ -401,6 +400,11 @@ public class RepeatedMapVector extends AbstractMapVector
   }
 
   @Override
+  public void copyEntry(int toIndex, ValueVector from, int fromIndex) {
+    copyFromSafe(fromIndex, toIndex, (RepeatedMapVector) from);
+  }
+
+  @Override
   public int getValueCapacity() {
     return Math.max(offsets.getValueCapacity() - 1, 0);
   }
@@ -411,6 +415,13 @@ public class RepeatedMapVector extends AbstractMapVector
   }
 
   @Override
+  public void exchange(ValueVector other) {
+    // Exchange is used for look-ahead writers, but writers manage
+    // map member vectors directly.
+    throw new UnsupportedOperationException("Exchange() not supported for maps");
+  }
+
+  @Override
   public DrillBuf[] getBuffers(boolean clear) {
     //final int expectedBufferSize = getBufferSize();
     //final int actualBufferSize = super.getBufferSize();
@@ -418,7 +429,6 @@ public class RepeatedMapVector extends AbstractMapVector
     return ArrayUtils.addAll(offsets.getBuffers(clear), super.getBuffers(clear));
   }
 
-
   @Override
   public void load(SerializedField metadata, DrillBuf buffer) {
     final List<SerializedField> children = metadata.getChildList();
@@ -572,6 +582,9 @@ public class RepeatedMapVector extends AbstractMapVector
       offsets.getMutator().setSafe(index + 1, prevEnd + 1);
       return prevEnd;
     }
+
+    @Override
+    public void exchange(ValueVector.Mutator other) { }
   }
 
   @Override

http://git-wip-us.apache.org/repos/asf/drill/blob/92c9304f/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/impl/AbstractBaseReader.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/impl/AbstractBaseReader.java b/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/impl/AbstractBaseReader.java
index 05fe63f..fdcfcd6 100644
--- a/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/impl/AbstractBaseReader.java
+++ b/exec/vector/src/main/java/org/apache/drill/exec/vector/complex/impl/AbstractBaseReader.java
@@ -38,18 +38,12 @@ abstract class AbstractBaseReader implements FieldReader{
     super();
   }
 
-  public void setPosition(int index){
-    this.index = index;
-  }
+  public void setPosition(int index) { this.index = index; }
 
-  int idx(){
-    return index;
-  }
+  int idx() { return index; }
 
   @Override
-  public void reset() {
-    index = 0;
-  }
+  public void reset() { index = 0; }
 
   @Override
   public Iterator<String> iterator() {


Mime
View raw message