drill-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From prog...@apache.org
Subject [2/3] drill git commit: DRILL-5517: Size-aware set methods in value vectors
Date Mon, 03 Jul 2017 19:49:35 GMT
DRILL-5517: Size-aware set methods in value vectors

Please see DRILL-5517 for an explanation.

Also includes a workaround for DRILL-5529.
Implements a setEmpties method for repeated and non-nullable
variable-width types in support of the revised column accessors.

Unit test included. Without the setEmpties call, the tests fail with
vector corruption. With the call, things work properly.

closes #840


Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/92c9304f
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/92c9304f
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/92c9304f

Branch: refs/heads/master
Commit: 92c9304f77a515d8f6747514bd295613738f611e
Parents: 6446e56
Author: Paul Rogers <progers@maprtech.com>
Authored: Tue May 16 13:20:32 2017 -0700
Committer: Paul Rogers <progers@maprtech.com>
Committed: Mon Jul 3 11:39:16 2017 -0700

----------------------------------------------------------------------
 .../exec/record/vector/TestValueVector.java     |   7 +-
 .../apache/drill/vector/TestFillEmpties.java    | 242 ++++++++
 .../apache/drill/vector/TestVectorLimits.java   | 484 ++++++++++++++++
 .../org/apache/drill/vector/package-info.java   |  22 +
 .../src/main/java/io/netty/buffer/DrillBuf.java |  19 +
 .../netty/buffer/UnsafeDirectLittleEndian.java  |  75 ++-
 .../codegen/templates/FixedValueVectors.java    | 577 +++++++++++++------
 .../codegen/templates/NullableValueVectors.java | 163 +++++-
 .../codegen/templates/RepeatedValueVectors.java | 132 ++++-
 .../src/main/codegen/templates/UnionVector.java |  13 +-
 .../templates/VariableLengthVectors.java        | 218 ++++++-
 .../drill/exec/vector/BaseDataValueVector.java  |  21 +-
 .../drill/exec/vector/BaseValueVector.java      |  13 +-
 .../org/apache/drill/exec/vector/BitVector.java |  85 ++-
 .../drill/exec/vector/FixedWidthVector.java     |   5 +-
 .../apache/drill/exec/vector/ObjectVector.java  |  39 +-
 .../apache/drill/exec/vector/ValueVector.java   |  60 +-
 .../exec/vector/VectorOverflowException.java    |  30 +
 .../apache/drill/exec/vector/VectorUtils.java   |  63 ++
 .../apache/drill/exec/vector/ZeroVector.java    |  71 +--
 .../vector/complex/BaseRepeatedValueVector.java |  18 +-
 .../drill/exec/vector/complex/ListVector.java   |   8 +-
 .../drill/exec/vector/complex/MapVector.java    |  14 +-
 .../exec/vector/complex/RepeatedListVector.java |  19 +-
 .../exec/vector/complex/RepeatedMapVector.java  |  19 +-
 .../vector/complex/impl/AbstractBaseReader.java |  12 +-
 26 files changed, 2059 insertions(+), 370 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/drill/blob/92c9304f/exec/java-exec/src/test/java/org/apache/drill/exec/record/vector/TestValueVector.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/record/vector/TestValueVector.java b/exec/java-exec/src/test/java/org/apache/drill/exec/record/vector/TestValueVector.java
index f6044e1..5191f5f 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/record/vector/TestValueVector.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/record/vector/TestValueVector.java
@@ -1,4 +1,4 @@
-/**
+/*
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
@@ -25,7 +25,6 @@ import java.nio.charset.Charset;
 
 import org.apache.drill.common.AutoCloseables;
 import org.apache.drill.common.config.DrillConfig;
-import org.apache.drill.common.types.MinorType;
 import org.apache.drill.common.types.TypeProtos;
 import org.apache.drill.common.types.Types;
 import org.apache.drill.exec.ExecTest;
@@ -47,7 +46,6 @@ import org.apache.drill.exec.memory.BufferAllocator;
 import org.apache.drill.exec.memory.RootAllocatorFactory;
 import org.apache.drill.exec.proto.UserBitShared;
 import org.apache.drill.exec.record.MaterializedField;
-import org.apache.drill.exec.record.VectorWrapper;
 import org.apache.drill.exec.vector.BaseValueVector;
 import org.apache.drill.exec.vector.BitVector;
 import org.apache.drill.exec.vector.NullableFloat4Vector;
@@ -97,6 +95,7 @@ public class TestValueVector extends ExecTest {
   @Test(expected = OversizedAllocationException.class)
   public void testFixedVectorReallocation() {
     final MaterializedField field = MaterializedField.create(EMPTY_SCHEMA_PATH, UInt4Holder.TYPE);
+    @SuppressWarnings("resource")
     final UInt4Vector vector = new UInt4Vector(field, allocator);
     // edge case 1: buffer size = max value capacity
     final int expectedValueCapacity = BaseValueVector.MAX_ALLOCATION_SIZE / 4;
@@ -149,10 +148,10 @@ public class TestValueVector extends ExecTest {
     }
   }
 
-
   @Test(expected = OversizedAllocationException.class)
   public void testVariableVectorReallocation() {
     final MaterializedField field = MaterializedField.create(EMPTY_SCHEMA_PATH, UInt4Holder.TYPE);
+    @SuppressWarnings("resource")
     final VarCharVector vector = new VarCharVector(field, allocator);
     // edge case 1: value count = MAX_VALUE_ALLOCATION
     final int expectedAllocationInBytes = BaseValueVector.MAX_ALLOCATION_SIZE;

http://git-wip-us.apache.org/repos/asf/drill/blob/92c9304f/exec/java-exec/src/test/java/org/apache/drill/vector/TestFillEmpties.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/vector/TestFillEmpties.java b/exec/java-exec/src/test/java/org/apache/drill/vector/TestFillEmpties.java
new file mode 100644
index 0000000..266bff2
--- /dev/null
+++ b/exec/java-exec/src/test/java/org/apache/drill/vector/TestFillEmpties.java
@@ -0,0 +1,242 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ ******************************************************************************/
+
+package org.apache.drill.vector;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
+
+import org.apache.drill.common.types.TypeProtos.DataMode;
+import org.apache.drill.common.types.TypeProtos.MajorType;
+import org.apache.drill.common.types.TypeProtos.MinorType;
+import org.apache.drill.exec.record.MaterializedField;
+import org.apache.drill.exec.vector.BaseDataValueVector;
+import org.apache.drill.exec.vector.IntVector;
+import org.apache.drill.exec.vector.NullableVarCharVector;
+import org.apache.drill.exec.vector.RepeatedVarCharVector;
+import org.apache.drill.exec.vector.UInt4Vector;
+import org.apache.drill.exec.vector.VarCharVector;
+import org.apache.drill.exec.vector.VectorOverflowException;
+import org.apache.drill.test.DrillTest;
+import org.apache.drill.test.OperatorFixture;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import io.netty.buffer.DrillBuf;
+
+public class TestFillEmpties extends DrillTest {
+
+  public static OperatorFixture fixture;
+
+  @BeforeClass
+  public static void setUpBeforeClass() throws Exception {
+    fixture = OperatorFixture.builder().build();
+  }
+
+  @AfterClass
+  public static void tearDownAfterClass() throws Exception {
+    fixture.close();
+  }
+
+  // To be replaced by a test method in a separate commit.
+
+  public static MaterializedField makeField(String name, MinorType dataType, DataMode mode) {
+    MajorType type = MajorType.newBuilder()
+        .setMinorType(dataType)
+        .setMode(mode)
+        .build();
+
+    return MaterializedField.create(name, type);
+  }
+
+  @Test
+  public void testNullableVarChar() {
+    @SuppressWarnings("resource")
+    NullableVarCharVector vector = new NullableVarCharVector(makeField("a", MinorType.VARCHAR, DataMode.OPTIONAL), fixture.allocator());
+    vector.allocateNew( );
+
+    // Create "foo", null, "bar", but omit the null.
+
+    NullableVarCharVector.Mutator mutator = vector.getMutator();
+    byte[] value = makeValue( "foo" );
+    mutator.setSafe(0, value, 0, value.length);
+
+    value = makeValue("bar");
+    mutator.setSafe(2, value, 0, value.length);
+
+    visualize(vector, 3);
+    verifyOffsets(vector.getValuesVector().getOffsetVector(), new int[] {0, 3, 3, 6});
+    vector.close();
+  }
+
+  @Test
+  public void testVarChar() {
+    @SuppressWarnings("resource")
+    VarCharVector vector = new VarCharVector(makeField("a", MinorType.VARCHAR, DataMode.REQUIRED), fixture.allocator());
+    vector.allocateNew( );
+
+    // Create "foo", null, "bar", but omit the null.
+
+    VarCharVector.Mutator mutator = vector.getMutator();
+    byte[] value = makeValue( "foo" );
+    mutator.setSafe(0, value, 0, value.length);
+
+    // Work around: test fails without this. But, only the new column writers
+    // call this method.
+
+    try {
+      mutator.fillEmptiesBounded(0, 2);
+    } catch (VectorOverflowException e) {
+      fail();
+    }
+    value = makeValue("bar");
+    mutator.setSafe(2, value, 0, value.length);
+
+    visualize(vector, 3);
+    verifyOffsets(vector.getOffsetVector(), new int[] {0, 3, 3, 6});
+    vector.close();
+  }
+
+  @Test
+  public void testInt() {
+    @SuppressWarnings("resource")
+    IntVector vector = new IntVector(makeField("a", MinorType.INT, DataMode.REQUIRED), fixture.allocator());
+    vector.allocateNew( );
+
+    // Create 1, 0, 2, but omit the 0.
+
+    IntVector.Mutator mutator = vector.getMutator();
+    mutator.setSafe(0, 1);
+
+    mutator.setSafe(2, 3);
+
+    visualize(vector, 3);
+    vector.close();
+  }
+
+  @Test
+  public void testRepeatedVarChar() {
+    @SuppressWarnings("resource")
+    RepeatedVarCharVector vector = new RepeatedVarCharVector(makeField("a", MinorType.VARCHAR, DataMode.REPEATED), fixture.allocator());
+    vector.allocateNew( );
+
+    // Create "foo", null, "bar", but omit the null.
+
+    RepeatedVarCharVector.Mutator mutator = vector.getMutator();
+    mutator.startNewValue(0);
+    byte[] value = makeValue( "a" );
+    mutator.addSafe(0, value, 0, value.length);
+    value = makeValue( "b" );
+    mutator.addSafe(0, value, 0, value.length);
+
+    // Work around: test fails without this. But, only the new column writers
+    // call this method.
+
+    try {
+      mutator.fillEmptiesBounded(0, 2);
+    } catch (VectorOverflowException e) {
+      fail();
+    }
+    mutator.startNewValue(2);
+    value = makeValue( "c" );
+    mutator.addSafe(2, value, 0, value.length);
+    value = makeValue( "d" );
+    mutator.addSafe(2, value, 0, value.length);
+
+    visualize(vector, 3);
+    verifyOffsets(vector.getOffsetVector(), new int[] {0, 2, 2, 4});
+    verifyOffsets(vector.getDataVector().getOffsetVector(), new int[] {0, 1, 2, 3, 4});
+    vector.close();
+  }
+
+  private void visualize(RepeatedVarCharVector vector, int valueCount) {
+    visualize("Array Offsets", vector.getOffsetVector(), valueCount + 1);
+    visualize(vector.getDataVector(), vector.getOffsetVector().getAccessor().get(valueCount));
+  }
+
+  private void visualize(IntVector vector, int valueCount) {
+    System.out.print("Values: [");
+    IntVector.Accessor accessor = vector.getAccessor();
+    for (int i = 0; i < valueCount; i++) {
+      if (i > 0) { System.out.print(" "); }
+      System.out.print(accessor.get(i));
+    }
+    System.out.println("]");
+  }
+
+  private void visualize(NullableVarCharVector vector, int valueCount) {
+    visualize("Is-set", vector.getAccessor(), valueCount);
+    visualize(vector.getValuesVector(), valueCount);
+  }
+
+  private void visualize(VarCharVector vector, int valueCount) {
+    visualize("Offsets", vector.getOffsetVector(), valueCount + 1);
+    visualize("Data", vector.getBuffer(), vector.getOffsetVector().getAccessor().get(valueCount));
+  }
+
+  private void visualize(String label, UInt4Vector offsetVector,
+      int valueCount) {
+    System.out.print(label + ": [");
+    UInt4Vector.Accessor accessor = offsetVector.getAccessor();
+    for (int i = 0; i < valueCount; i++) {
+      if (i > 0) { System.out.print(" "); }
+      System.out.print(accessor.get(i));
+    }
+    System.out.println("]");
+  }
+
+  private void visualize(String label, DrillBuf buffer, int valueCount) {
+    System.out.print(label + ": [");
+    for (int i = 0; i < valueCount; i++) {
+      if (i > 0) { System.out.print(" "); }
+      System.out.print((char) buffer.getByte(i));
+    }
+    System.out.println("]");
+  }
+
+  private void visualize(String label, BaseDataValueVector.BaseAccessor accessor, int valueCount) {
+    System.out.print(label + ": [");
+    for (int i = 0; i < valueCount; i++) {
+      if (i > 0) { System.out.print(" "); }
+      System.out.print(accessor.isNull(i) ? 0 : 1);
+    }
+    System.out.println("]");
+  }
+
+  private void verifyOffsets(UInt4Vector offsetVector, int[] expected) {
+    UInt4Vector.Accessor accessor = offsetVector.getAccessor();
+    for (int i = 0; i < expected.length; i++) {
+      assertEquals(expected[i], accessor.get(i));
+    }
+  }
+
+  /**
+   * Create a test value. Works only for the ASCII subset of characters, obviously.
+   * @param string
+   * @return
+   */
+  private byte[] makeValue(String string) {
+    byte value[] = new byte[string.length()];
+    for (int i = 0; i < value.length; i++) {
+      value[i] = (byte) string.charAt(i);
+    }
+    return value;
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/drill/blob/92c9304f/exec/java-exec/src/test/java/org/apache/drill/vector/TestVectorLimits.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/vector/TestVectorLimits.java b/exec/java-exec/src/test/java/org/apache/drill/vector/TestVectorLimits.java
new file mode 100644
index 0000000..86bd206
--- /dev/null
+++ b/exec/java-exec/src/test/java/org/apache/drill/vector/TestVectorLimits.java
@@ -0,0 +1,484 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.vector;
+
+import static org.junit.Assert.*;
+
+import org.apache.drill.common.types.TypeProtos.DataMode;
+import org.apache.drill.common.types.TypeProtos.MajorType;
+import org.apache.drill.common.types.TypeProtos.MinorType;
+import org.apache.drill.exec.record.MaterializedField;
+import org.apache.drill.exec.vector.IntVector;
+import org.apache.drill.exec.vector.NullableIntVector;
+import org.apache.drill.exec.vector.NullableVarCharVector;
+import org.apache.drill.exec.vector.RepeatedIntVector;
+import org.apache.drill.exec.vector.ValueVector;
+import org.apache.drill.exec.vector.VarCharVector;
+import org.apache.drill.exec.vector.VectorOverflowException;
+import org.apache.drill.test.DrillTest;
+import org.apache.drill.test.OperatorFixture;
+import org.bouncycastle.util.Arrays;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import io.netty.buffer.DrillBuf;
+
+/**
+ * Test the setScalar() methods in the various generated vector
+ * classes. Rather than test all 100+ vectors, we sample a few and
+ * rely on the fact that code is generated from a common template.
+ */
+
+public class TestVectorLimits extends DrillTest {
+
+  public static OperatorFixture fixture;
+
+  @BeforeClass
+  public static void setUpBeforeClass() throws Exception {
+    fixture = OperatorFixture.builder().build();
+  }
+
+  @AfterClass
+  public static void tearDownAfterClass() throws Exception {
+    fixture.close();
+  }
+
+  /**
+   * Test a vector directly using the vector mutator to ensure
+   * that the <tt>setScalar</tt> method works for the maximum
+   * row count.
+   * <p>
+   * This test is a proxy for all the other fixed types, since all
+   * share the same code template.
+   */
+
+  @Test
+  public void testFixedVector() {
+
+    // Create a non-nullable int vector: a typical fixed-size vector
+
+    @SuppressWarnings("resource")
+    IntVector vector = new IntVector(makeField(MinorType.INT, DataMode.REQUIRED), fixture.allocator() );
+
+    // Sanity test of generated constants.
+
+    assertTrue( IntVector.MAX_SCALAR_COUNT <= ValueVector.MAX_ROW_COUNT );
+    assertEquals( 4, IntVector.VALUE_WIDTH );
+    assertTrue( IntVector.NET_MAX_SCALAR_SIZE <= ValueVector.MAX_BUFFER_SIZE );
+
+    // Allocate a default size, small vector. Forces test of
+    // the auto-grow (setSafe()) aspect of setScalar().
+
+    vector.allocateNew( );
+
+    // Write to the vector until it complains. At that point,
+    // we should have written up to the static fixed value count
+    // (which is computed to stay below the capacity limit.)
+
+    IntVector.Mutator mutator = vector.getMutator();
+    for (int i = 0; i < 2 * ValueVector.MAX_ROW_COUNT; i++) {
+      try {
+        mutator.setScalar(i, i);
+      } catch (VectorOverflowException e) {
+        assertEquals(IntVector.MAX_SCALAR_COUNT, i);
+        break;
+      }
+    }
+
+    // The vector should be below the allocation limit. Since this
+    // is an int vector, in practice the size will be far below
+    // the overall limit (if the limit stays at 16 MB.) But, it should
+    // be at the type-specific limit since we filled up the vector.
+
+    assertEquals(IntVector.NET_MAX_SCALAR_SIZE, vector.getBuffer().getActualMemoryConsumed());
+    vector.close();
+  }
+
+  @Test
+  public void testNullableFixedVector() {
+
+    @SuppressWarnings("resource")
+    NullableIntVector vector = new NullableIntVector(makeField(MinorType.INT, DataMode.OPTIONAL), fixture.allocator() );
+    vector.allocateNew( );
+
+    NullableIntVector.Mutator mutator = vector.getMutator();
+    for (int i = 0; i < 2 * ValueVector.MAX_ROW_COUNT; i++) {
+      try {
+        mutator.setScalar(i, i);
+      } catch (VectorOverflowException e) {
+        assertEquals(IntVector.MAX_SCALAR_COUNT, i);
+        break;
+      }
+    }
+
+    vector.close();
+  }
+
+  /**
+   * Repeated fixed vector. Using an int vector, each column array can hold
+   * 256 / 4 = 64 values. We write only 10. The vector becomes full when we
+   * exceed 64K items.
+   */
+
+  @Test
+  public void testRepeatedFixedVectorCountLimit() {
+
+    @SuppressWarnings("resource")
+    RepeatedIntVector vector = new RepeatedIntVector(makeField(MinorType.INT, DataMode.REPEATED), fixture.allocator() );
+    vector.allocateNew( );
+
+    RepeatedIntVector.Mutator mutator = vector.getMutator();
+    top:
+    for (int i = 0; i < 2 * ValueVector.MAX_ROW_COUNT; i++) {
+      if (! mutator.startNewValueBounded(i)) {
+        assertEquals(ValueVector.MAX_ROW_COUNT, i);
+        // Continue, let's check the addBounded method also
+      }
+      for (int j = 0; j < 10; j++) {
+        try {
+          mutator.addEntry(i, i * 100 + j);
+        } catch (VectorOverflowException e) {
+          assertEquals(ValueVector.MAX_ROW_COUNT, i);
+          mutator.setValueCount(i);
+          break top;
+        }
+      }
+    }
+
+    vector.close();
+  }
+
+  /**
+   * Repeated fixed vector. Using an int vector, each column array can hold
+   * 256 / 4 = 64 values. We write 100. The vector becomes full when we
+   * exceed the 16 MB size limit.
+   */
+
+  @Test
+  public void testRepeatedFixedVectorBufferLimit() {
+
+    @SuppressWarnings("resource")
+    RepeatedIntVector vector = new RepeatedIntVector(makeField(MinorType.INT, DataMode.REPEATED), fixture.allocator() );
+    vector.allocateNew( );
+
+    RepeatedIntVector.Mutator mutator = vector.getMutator();
+    top:
+    for (int i = 0; i < 2 * ValueVector.MAX_ROW_COUNT; i++) {
+      // We'll never hit the value count limit
+      assertTrue(mutator.startNewValueBounded(i));
+      for (int j = 0; j < 100; j++) {
+        try {
+          mutator.addEntry(i, i * 100 + j);
+        } catch (VectorOverflowException e) {
+          // We should have hit the buffer limit before the value limit.
+          assertTrue(i < ValueVector.MAX_ROW_COUNT);
+          mutator.setValueCount(i);
+          break top;
+        }
+      }
+    }
+
+    vector.close();
+  }
+
+  // To be replaced by a test method in a separate commit.
+
+  public static MaterializedField makeField(MinorType dataType, DataMode mode) {
+    MajorType type = MajorType.newBuilder()
+        .setMinorType(dataType)
+        .setMode(mode)
+        .build();
+
+    return MaterializedField.create("foo", type);
+  }
+
+  /**
+   * Baseline test for a variable-width vector using <tt>setSafe</tt> and
+   * loading the vector up to the maximum size. Doing so will cause the vector
+   * to have a buffer that exceeds the maximum size, demonstrating the
+   * need for <tt>setScalar()</tt>.
+   */
+
+  @Test
+  public void variableVectorBaseline() {
+
+    // Create a non-nullable VarChar vector: a typical variable-size vector
+
+    @SuppressWarnings("resource")
+    VarCharVector vector = new VarCharVector(makeField(MinorType.VARCHAR, DataMode.REQUIRED), fixture.allocator() );
+    vector.allocateNew( );
+
+    // A 16 MB value can hold 64K values of up to 256 bytes each.
+    // To force a size overflow, write values much larger.
+    // Write the maximum number of values which will silently
+    // allow the vector to grow beyond the critical size of 16 MB.
+    // Doing this in production would lead to memory fragmentation.
+    // So, this is what the setScalar() method assures we don't do.
+
+    byte dummyValue[] = new byte[512];
+    Arrays.fill(dummyValue, (byte) 'X');
+    VarCharVector.Mutator mutator = vector.getMutator();
+    for (int i = 0; i < 2 * ValueVector.MAX_ROW_COUNT; i++) {
+      mutator.setSafe(i, dummyValue, 0, dummyValue.length);
+    }
+
+    // The vector should be above the allocation limit.
+    // This is why code must migrate to the setScalar() call
+    // away from the setSafe() call.
+
+    assertTrue(ValueVector.MAX_BUFFER_SIZE < vector.getBuffer().getActualMemoryConsumed());
+    vector.close();
+  }
+
+  /**
+   * Test a vector directly using the vector mutator to ensure
+   * that the <tt>setScalar</tt> method works for the maximum
+   * vector size.
+   */
+
+  @Test
+  public void testWideVariableVector() {
+
+    @SuppressWarnings("resource")
+    VarCharVector vector = new VarCharVector(makeField(MinorType.VARCHAR, DataMode.REQUIRED), fixture.allocator() );
+    vector.allocateNew( );
+
+    // A 16 MB value can hold 64K values of up to 256 bytes each.
+    // To force a size overflow, write values much larger.
+    // Write to the vector until it complains. At that point,
+    // we should have written up to the maximum buffer size.
+
+    byte dummyValue[] = makeVarCharValue(512);
+    VarCharVector.Mutator mutator = vector.getMutator();
+    int count = 0;
+    for ( ; count < 2 * ValueVector.MAX_ROW_COUNT; count++) {
+      try {
+        mutator.setScalar(count, dummyValue, 0, dummyValue.length);
+      } catch (VectorOverflowException e) {
+        break;
+      }
+    }
+
+    // The vector should be at the allocation limit. If it wasn't, we
+    // should have grown it to hold more data. The value count will
+    // be below the maximum.
+
+    mutator.setValueCount(count);
+    assertEquals(ValueVector.MAX_BUFFER_SIZE, vector.getBuffer().getActualMemoryConsumed());
+    assertTrue(count < ValueVector.MAX_ROW_COUNT);
+    vector.close();
+  }
+
+  private byte[] makeVarCharValue(int n) {
+    byte dummyValue[] = new byte[n];
+    Arrays.fill(dummyValue, (byte) 'X');
+    return dummyValue;
+  }
+
+  @Test
+  public void testNullableWideVariableVector() {
+
+    @SuppressWarnings("resource")
+    NullableVarCharVector vector = new NullableVarCharVector(makeField(MinorType.VARCHAR, DataMode.OPTIONAL), fixture.allocator() );
+    vector.allocateNew( );
+
+    byte dummyValue[] = makeVarCharValue(512);
+    NullableVarCharVector.Mutator mutator = vector.getMutator();
+    int count = 0;
+    for ( ; count < 2 * ValueVector.MAX_ROW_COUNT; count++) {
+      try {
+        mutator.setScalar(count, dummyValue, 0, dummyValue.length);
+      } catch (VectorOverflowException e) {
+        break;
+      }
+    }
+
+    mutator.setValueCount(count);
+    assertEquals(ValueVector.MAX_BUFFER_SIZE, vector.getValuesVector().getBuffer().getActualMemoryConsumed());
+    assertTrue(count < ValueVector.MAX_ROW_COUNT);
+    vector.close();
+  }
+
+  /**
+   * Test a vector directly using the vector mutator to ensure
+   * that the <tt>setScalar</tt> method works for the maximum
+   * value count.
+   */
+
+  @Test
+  public void testNarrowVariableVector() {
+
+    @SuppressWarnings("resource")
+    VarCharVector vector = new VarCharVector(makeField(MinorType.VARCHAR, DataMode.REQUIRED), fixture.allocator() );
+    vector.allocateNew( );
+
+    // Write small values that fit into 16 MB. We should stop writing
+    // when we reach the value count limit.
+
+    byte dummyValue[] = makeVarCharValue(254);
+    VarCharVector.Mutator mutator = vector.getMutator();
+    int count = 0;
+    for (; count < 2 * ValueVector.MAX_ROW_COUNT; count++) {
+      try {
+        mutator.setScalar(count, dummyValue, 0, dummyValue.length);
+      } catch (VectorOverflowException e) {
+        break;
+      }
+    }
+
+    // Buffer size should be at or below the maximum, with count
+    // at the maximum.
+
+    mutator.setValueCount(count);
+    assertTrue(vector.getBuffer().getActualMemoryConsumed() <= ValueVector.MAX_BUFFER_SIZE);
+    assertEquals(ValueVector.MAX_ROW_COUNT, count);
+    vector.close();
+  }
+
+  /**
+   * Test a vector directly using the vector mutator to ensure
+   * that the <tt>setScalar</tt> method works for the maximum
+   * value count. Uses a DrillBuf as input.
+   */
+
+  @Test
+  public void testDirectVariableVector() {
+
+    @SuppressWarnings("resource")
+    VarCharVector vector = new VarCharVector(makeField(MinorType.VARCHAR, DataMode.REQUIRED), fixture.allocator() );
+    vector.allocateNew( );
+
+    // Repeat the big-value test, but with data coming from a DrillBuf
+    // (direct memory) rather than a heap array.
+
+    @SuppressWarnings("resource")
+    DrillBuf drillBuf = makeVarCharValueDirect(260);
+    VarCharVector.Mutator mutator = vector.getMutator();
+    int count = 0;
+    for (; count < 2 * ValueVector.MAX_ROW_COUNT; count++) {
+      try {
+        mutator.setScalar(count, drillBuf, 0, 260);
+      } catch (VectorOverflowException e) {
+        break;
+      }
+    }
+    drillBuf.close();
+
+    // Again, vector should be at the size limit, count below the
+    // value limit.
+
+    mutator.setValueCount(count);
+    assertEquals(ValueVector.MAX_BUFFER_SIZE, vector.getBuffer().getActualMemoryConsumed());
+    assertTrue(count < ValueVector.MAX_ROW_COUNT);
+    vector.close();
+  }
+
+  private DrillBuf makeVarCharValueDirect(int n) {
+    byte dummyValue[] = makeVarCharValue(n);
+    DrillBuf drillBuf = fixture.allocator().buffer(dummyValue.length);
+    drillBuf.setBytes(0, dummyValue);
+    return drillBuf;
+  }
+
+  @Test
+  public void testDirectNullableVariableVector() {
+
+    @SuppressWarnings("resource")
+    NullableVarCharVector vector = new NullableVarCharVector(makeField(MinorType.VARCHAR, DataMode.OPTIONAL), fixture.allocator() );
+    vector.allocateNew( );
+
+    @SuppressWarnings("resource")
+    DrillBuf drillBuf = makeVarCharValueDirect(260);
+    NullableVarCharVector.Mutator mutator = vector.getMutator();
+    int count = 0;
+    for (; count < 2 * ValueVector.MAX_ROW_COUNT; count++) {
+      try {
+        mutator.setScalar(count, drillBuf, 0, 260);
+      } catch (VectorOverflowException e) {
+        break;
+      }
+    }
+    drillBuf.close();
+
+    mutator.setValueCount(count);
+    assertEquals(ValueVector.MAX_BUFFER_SIZE, vector.getValuesVector().getBuffer().getActualMemoryConsumed());
+    assertTrue(count < ValueVector.MAX_ROW_COUNT);
+    vector.close();
+  }
+
+  public static void main(String args[]) {
+    try {
+      setUpBeforeClass();
+      new TestVectorLimits().performanceTest();
+      tearDownAfterClass();
+    } catch (Exception e) {
+      // TODO Auto-generated catch block
+      e.printStackTrace();
+    }
+  }
+
+  private void performanceTest() {
+    @SuppressWarnings("resource")
+    VarCharVector vector = new VarCharVector(makeField(MinorType.VARCHAR, DataMode.OPTIONAL), fixture.allocator() );
+    byte value[] = makeVarCharValue(1);
+    int warmCount = 100;
+    timeSetSafe(vector, value, warmCount);
+    runSetBounded(vector, value, warmCount);
+    int runCount = 1000;
+    timeSetSafe(vector, value, runCount);
+    runSetBounded(vector, value, runCount);
+    timeSetSafe(vector, value, runCount);
+    vector.close();
+  }
+
+  private void timeSetSafe(VarCharVector vector, byte[] value, int iterCount) {
+    long start = System.currentTimeMillis();
+    for (int i = 0; i < iterCount; i++) {
+      vector.clear();
+      vector.allocateNew( );
+
+      VarCharVector.Mutator mutator = vector.getMutator();
+      for (int j = 0; j < ValueVector.MAX_ROW_COUNT; j++) {
+        mutator.setSafe(j, value, 0, value.length);
+      }
+    }
+    long elapsed = System.currentTimeMillis() - start;
+    System.out.println( iterCount + " runs of setSafe: " + elapsed + " ms." );
+  }
+
+  private void runSetBounded(VarCharVector vector, byte[] value, int iterCount) {
+    long start = System.currentTimeMillis();
+    for (int i = 0; i < iterCount; i++) {
+      vector.clear();
+      vector.allocateNew( );
+
+      VarCharVector.Mutator mutator = vector.getMutator();
+      int posn = 0;
+      for (;;) {
+        try {
+          mutator.setScalar(posn++, value, 0, value.length);
+        } catch (VectorOverflowException e) {
+          break;
+        }
+      }
+    }
+    long elapsed = System.currentTimeMillis() - start;
+    System.out.println( iterCount + " runs of setScalar: " + elapsed + " ms." );
+  }
+}

http://git-wip-us.apache.org/repos/asf/drill/blob/92c9304f/exec/java-exec/src/test/java/org/apache/drill/vector/package-info.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/vector/package-info.java b/exec/java-exec/src/test/java/org/apache/drill/vector/package-info.java
new file mode 100644
index 0000000..c858814
--- /dev/null
+++ b/exec/java-exec/src/test/java/org/apache/drill/vector/package-info.java
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/**
+ * Tests for value vectors. Is in this module to allow use of
+ * the test tools which are available only in this module.
+ */
+package org.apache.drill.vector;

http://git-wip-us.apache.org/repos/asf/drill/blob/92c9304f/exec/memory/base/src/main/java/io/netty/buffer/DrillBuf.java
----------------------------------------------------------------------
diff --git a/exec/memory/base/src/main/java/io/netty/buffer/DrillBuf.java b/exec/memory/base/src/main/java/io/netty/buffer/DrillBuf.java
index 3793f25..5139086 100644
--- a/exec/memory/base/src/main/java/io/netty/buffer/DrillBuf.java
+++ b/exec/memory/base/src/main/java/io/netty/buffer/DrillBuf.java
@@ -748,6 +748,25 @@ public final class DrillBuf extends AbstractByteBuf implements AutoCloseable {
     return this;
   }
 
+  // Clone of UDLE's setBytes(), but with bounds checking done as a boolean,
+  // not assertion.
+
+  public boolean setBytesBounded(int index, byte[] src, int srcIndex, int length) {
+    // Must do here because Drill's UDLE is not ref counted.
+    // Done as an assert to avoid production overhead: if this is going
+    // to fail, it will do so spectacularly in tests, due to a programming error.
+    assert refCnt() > 0;
+    return udle.setBytesBounded(index, src, srcIndex, length);
+  }
+
+  // As above, but for direct memory.
+
+  public boolean setBytesBounded(int index, DrillBuf src, int srcIndex, int length) {
+    // See above.
+    assert refCnt() > 0;
+    return udle.setBytesBounded(index, src.udle, srcIndex, length);
+  }
+
   @Override
   public ByteBuf setBytes(int index, byte[] src, int srcIndex, int length) {
     udle.setBytes(index + offset, src, srcIndex, length);

http://git-wip-us.apache.org/repos/asf/drill/blob/92c9304f/exec/memory/base/src/main/java/io/netty/buffer/UnsafeDirectLittleEndian.java
----------------------------------------------------------------------
diff --git a/exec/memory/base/src/main/java/io/netty/buffer/UnsafeDirectLittleEndian.java b/exec/memory/base/src/main/java/io/netty/buffer/UnsafeDirectLittleEndian.java
index 6495d5d..c91944f 100644
--- a/exec/memory/base/src/main/java/io/netty/buffer/UnsafeDirectLittleEndian.java
+++ b/exec/memory/base/src/main/java/io/netty/buffer/UnsafeDirectLittleEndian.java
@@ -18,15 +18,17 @@
 
 package io.netty.buffer;
 
-import io.netty.util.internal.PlatformDependent;
-
 import java.nio.ByteOrder;
 import java.util.concurrent.atomic.AtomicLong;
 
+import io.netty.util.internal.PlatformDependent;
+
 /**
- * The underlying class we use for little-endian access to memory. Is used underneath DrillBufs to abstract away the
- * Netty classes and underlying Netty memory management.
+ * The underlying class we use for little-endian access to memory. Is used
+ * underneath DrillBufs to abstract away the Netty classes and underlying Netty
+ * memory management.
  */
+
 public final class UnsafeDirectLittleEndian extends WrappedByteBuf {
   private static final boolean NATIVE_ORDER = ByteOrder.nativeOrder() == ByteOrder.LITTLE_ENDIAN;
   private static final AtomicLong ID_GENERATOR = new AtomicLong(0);
@@ -49,7 +51,6 @@ public final class UnsafeDirectLittleEndian extends WrappedByteBuf {
 
   UnsafeDirectLittleEndian(PooledUnsafeDirectByteBuf buf, AtomicLong bufferCount, AtomicLong bufferSize) {
     this(buf, true, bufferCount, bufferSize);
-
   }
 
   private UnsafeDirectLittleEndian(AbstractByteBuf buf, boolean fake, AtomicLong bufferCount, AtomicLong bufferSize) {
@@ -67,21 +68,21 @@ public final class UnsafeDirectLittleEndian extends WrappedByteBuf {
     this.wrapped = buf;
     this.memoryAddress = buf.memoryAddress();
   }
-    private long addr(int index) {
-        return memoryAddress + index;
-    }
 
-    @Override
-    public long getLong(int index) {
-//        wrapped.checkIndex(index, 8);
-        long v = PlatformDependent.getLong(addr(index));
-        return v;
-    }
+  private long addr(int index) {
+    return memoryAddress + index;
+  }
 
-    @Override
-    public float getFloat(int index) {
-        return Float.intBitsToFloat(getInt(index));
-    }
+  @Override
+  public long getLong(int index) {
+//  wrapped.checkIndex(index, 8);
+    return PlatformDependent.getLong(addr(index));
+  }
+
+  @Override
+  public float getFloat(int index) {
+    return Float.intBitsToFloat(getInt(index));
+  }
 
   @Override
   public ByteBuf slice() {
@@ -174,6 +175,43 @@ public final class UnsafeDirectLittleEndian extends WrappedByteBuf {
     return this;
   }
 
+  // Clone of the super class checkIndex, but this version returns a boolean rather
+  // than throwing an exception.
+
+  protected boolean hasCapacity(int index, int fieldLength) {
+    assert fieldLength >= 0;
+    return (! (index < 0 || index > capacity() - fieldLength));
+  }
+
+  /**
+   * Write bytes into the buffer at the given index, if space is available.
+   * @param index location to write
+   * @param src bytes to write
+   * @param srcIndex start of data in the source array
+   * @param length length of the data to write
+   * @return true if the value was written, false if the value was not
+   * written because the value would overflow the buffer
+   */
+
+  public boolean setBytesBounded(int index, byte[] src, int srcIndex, int length) {
+    if (! hasCapacity(index, length)) {
+      return false;
+    }
+    PlatformDependent.copyMemory(src, srcIndex, addr(index), length);
+    return true;
+  }
+
+  // Version of the super class setBytes(), but with bounds checking done as a boolean,
+  // not assertion. This version requires a direct source buffer.
+
+  public boolean setBytesBounded(int index, UnsafeDirectLittleEndian src, int srcIndex, int length) {
+    if (! hasCapacity(index, length)) {
+      return false;
+    }
+    PlatformDependent.copyMemory(src.memoryAddress() + srcIndex, addr(index), length);
+    return true;
+  }
+
   @Override
   public ByteBuf writeShort(int value) {
     wrapped.ensureWritable(2);
@@ -266,5 +304,4 @@ public final class UnsafeDirectLittleEndian extends WrappedByteBuf {
     assert isAssertEnabled = true;
     ASSERT_ENABLED = isAssertEnabled;
   }
-
 }

http://git-wip-us.apache.org/repos/asf/drill/blob/92c9304f/exec/vector/src/main/codegen/templates/FixedValueVectors.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/codegen/templates/FixedValueVectors.java b/exec/vector/src/main/codegen/templates/FixedValueVectors.java
index 23188ce..1e83a4f 100644
--- a/exec/vector/src/main/codegen/templates/FixedValueVectors.java
+++ b/exec/vector/src/main/codegen/templates/FixedValueVectors.java
@@ -31,22 +31,60 @@ package org.apache.drill.exec.vector;
 import org.apache.drill.exec.util.DecimalUtility;
 
 /**
- * ${minor.class} implements a vector of fixed width values.  Elements in the vector are accessed
- * by position, starting from the logical start of the vector.  Values should be pushed onto the
- * vector sequentially, but may be randomly accessed.
- *   The width of each element is ${type.width} byte(s)
- *   The equivalent Java primitive is '${minor.javaType!type.javaType}'
+ * ${minor.class} implements a vector of fixed width values. Elements in the vector are accessed
+ * by position, starting from the logical start of the vector. Values should be pushed onto the
+ * vector sequentially, but may be accessed randomly.
+ * <ul>
+ * <li>The width of each element is {@link #VALUE_WIDTH} (= ${type.width}) byte<#if type.width != 1>s</#if>.</li>
+ * <li>The equivalent Java primitive is '${minor.javaType!type.javaType}'.</li>
+ * </ul>
  *
  * NB: this class is automatically generated from ${.template_name} and ValueVectorTypes.tdd using FreeMarker.
  */
-public final class ${minor.class}Vector extends BaseDataValueVector implements FixedWidthVector{
+public final class ${minor.class}Vector extends BaseDataValueVector implements FixedWidthVector {
   private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(${minor.class}Vector.class);
 
+  /**
+   * Width of each fixed-width value.
+   */
+
+  public static final int VALUE_WIDTH = ${type.width};
+
+  /**
+   * Maximum number of values that this fixed-width vector can hold
+   * and stay below the maximum vector size limit. This is the limit
+   * enforced when the vector is used to hold values in a repeated
+   * vector.
+   */
+
+  public static final int MAX_VALUE_COUNT = MAX_BUFFER_SIZE / VALUE_WIDTH;
+
+  /**
+   * Maximum number of values that this fixed-width vector can hold
+   * and stay below the maximum vector size limit and/or stay below
+   * the maximum row count. This is the limit enforced when the
+   * vector is used to hold scalar (required or nullable) values.
+   * <p>
+   * Note: <tt>MAX_ROW_COUNT</tt> is defined in the parent <tt>ValueVector</tt>
+   * class as the maximum number of rows in a record batch (64K). Use this
+   * in place of the <tt>Character.MAX_SIZE</tt> value previously used.
+   */
+
+  public static final int MAX_SCALAR_COUNT = Math.min(MAX_ROW_COUNT, MAX_VALUE_COUNT);
+
+  /**
+   * Actual maximum vector size, in bytes, given the number of fixed-width
+   * values that either fit in the maximum overall vector size, or that
+   * is no larger than the maximum vector item count.
+   */
+
+  public static final int NET_MAX_SCALAR_SIZE = VALUE_WIDTH * MAX_SCALAR_COUNT;
+
   private final FieldReader reader = new ${minor.class}ReaderImpl(${minor.class}Vector.this);
   private final Accessor accessor = new Accessor();
   private final Mutator mutator = new Mutator();
 
-  private int allocationSizeInBytes = INITIAL_VALUE_ALLOCATION * ${type.width};
+  private int allocationSizeInBytes = Math.min(INITIAL_VALUE_ALLOCATION * VALUE_WIDTH, MAX_BUFFER_SIZE);
   private int allocationMonitor = 0;
 
   public ${minor.class}Vector(MaterializedField field, BufferAllocator allocator) {
@@ -54,45 +92,41 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F
   }
 
   @Override
-  public FieldReader getReader(){
-    return reader;
-  }
+  public FieldReader getReader() { return reader; }
 
   @Override
   public int getBufferSizeFor(final int valueCount) {
     if (valueCount == 0) {
       return 0;
     }
-    return valueCount * ${type.width};
+    return valueCount * VALUE_WIDTH;
   }
 
   @Override
   public int getValueCapacity(){
-    return data.capacity() / ${type.width};
+    return data.capacity() / VALUE_WIDTH;
   }
 
   @Override
-  public Accessor getAccessor(){
-    return accessor;
-  }
+  public Accessor getAccessor() { return accessor; }
 
   @Override
-  public Mutator getMutator(){
-    return mutator;
-  }
+  public Mutator getMutator() { return mutator; }
 
   @Override
   public void setInitialCapacity(final int valueCount) {
-    final long size = 1L * valueCount * ${type.width};
+    final long size = (long) valueCount * VALUE_WIDTH;
+    // TODO: Replace this with MAX_BUFFER_SIZE once all
+    // code is aware of the maximum vector size.
     if (size > MAX_ALLOCATION_SIZE) {
       throw new OversizedAllocationException("Requested amount of memory is more than max allowed allocation size");
     }
-    allocationSizeInBytes = (int)size;
+    allocationSizeInBytes = (int) size;
   }
 
   @Override
   public void allocateNew() {
-    if(!allocateNewSafe()){
+    if (!allocateNewSafe()){
       throw new OutOfMemoryException("Failure while allocating buffer.");
     }
   }
@@ -123,11 +157,11 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F
    * Note that the maximum number of values a vector can allocate is Integer.MAX_VALUE / value width.
    *
    * @param valueCount
-   * @throws org.apache.drill.exec.memory.OutOfMemoryException if it can't allocate the new buffer
+   * @throws OutOfMemoryException if it can't allocate the new buffer
    */
   @Override
   public void allocateNew(final int valueCount) {
-    allocateBytes(valueCount * ${type.width});
+    allocateBytes(valueCount * VALUE_WIDTH);
   }
 
   @Override
@@ -139,6 +173,8 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F
   }
 
   private void allocateBytes(final long size) {
+    // TODO: Replace this with MAX_BUFFER_SIZE once all
+    // code is aware of the maximum vector size.
     if (size > MAX_ALLOCATION_SIZE) {
       throw new OversizedAllocationException("Requested amount of memory is more than max allowed allocation size");
     }
@@ -150,13 +186,15 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F
     allocationSizeInBytes = curSize;
   }
 
-/**
- * Allocate new buffer with double capacity, and copy data into the new buffer. Replace vector's buffer with new buffer, and release old one
- *
- * @throws org.apache.drill.exec.memory.OutOfMemoryException if it can't allocate the new buffer
- */
+  /**
+   * Allocate new buffer with double capacity, and copy data into the new buffer. Replace vector's buffer with new buffer, and release old one
+   *
+   * @throws org.apache.drill.exec.memory.OutOfMemoryException if it can't allocate the new buffer
+   */
   public void reAlloc() {
     final long newAllocationSize = allocationSizeInBytes * 2L;
+    // TODO: Replace this with MAX_BUFFER_SIZE once all
+    // code is aware of the maximum vector size.
     if (newAllocationSize > MAX_ALLOCATION_SIZE)  {
       throw new OversizedAllocationException("Unable to expand the buffer. Max allowed buffer size is reached.");
     }
@@ -185,7 +223,7 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F
     Preconditions.checkArgument(this.field.getPath().equals(metadata.getNamePart().getName()), "The field %s doesn't match the provided metadata %s.", this.field, metadata);
     final int actualLength = metadata.getBufferLength();
     final int valueCount = metadata.getValueCount();
-    final int expectedLength = valueCount * ${type.width};
+    final int expectedLength = valueCount * VALUE_WIDTH;
     assert actualLength == expectedLength : String.format("Expected to load %d bytes but actually loaded %d bytes", expectedLength, actualLength);
 
     clear();
@@ -220,8 +258,8 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F
   }
 
   public void splitAndTransferTo(int startIndex, int length, ${minor.class}Vector target) {
-    final int startPoint = startIndex * ${type.width};
-    final int sliceLength = length * ${type.width};
+    final int startPoint = startIndex * VALUE_WIDTH;
+    final int sliceLength = length * VALUE_WIDTH;
     target.clear();
     target.data = data.slice(startPoint, sliceLength).transferOwnership(target.allocator).buffer;
     target.data.writerIndex(sliceLength);
@@ -229,7 +267,7 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F
 
   @Override
   public int getPayloadByteCount() {
-    return getAccessor().getValueCount() * ${type.width};
+    return getAccessor().getValueCount() * VALUE_WIDTH;
   }
 
   private class TransferImpl implements TransferPair{
@@ -266,10 +304,10 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F
 
   public void copyFrom(int fromIndex, int thisIndex, ${minor.class}Vector from){
     <#if (type.width > 8)>
-    from.data.getBytes(fromIndex * ${type.width}, data, thisIndex * ${type.width}, ${type.width});
+    from.data.getBytes(fromIndex * VALUE_WIDTH, data, thisIndex * VALUE_WIDTH, VALUE_WIDTH);
     <#else> <#-- type.width <= 8 -->
-    data.set${(minor.javaType!type.javaType)?cap_first}(thisIndex * ${type.width},
-        from.data.get${(minor.javaType!type.javaType)?cap_first}(fromIndex * ${type.width})
+    data.set${(minor.javaType!type.javaType)?cap_first}(thisIndex * VALUE_WIDTH,
+        from.data.get${(minor.javaType!type.javaType)?cap_first}(fromIndex * VALUE_WIDTH)
     );
     </#if> <#-- type.width -->
   }
@@ -281,6 +319,11 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F
     copyFrom(fromIndex, thisIndex, from);
   }
 
+  @Override
+  public void copyEntry(int toIndex, ValueVector from, int fromIndex) {
+    ((${minor.class}Vector) from).data.getBytes(fromIndex * ${type.width}, data, toIndex * ${type.width}, ${type.width});
+  }
+
   public void decrementAllocationMonitor() {
     if (allocationMonitor > 0) {
       allocationMonitor = 0;
@@ -295,7 +338,7 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F
   public final class Accessor extends BaseDataValueVector.BaseAccessor {
     @Override
     public int getValueCount() {
-      return data.writerIndex() / ${type.width};
+      return data.writerIndex() / VALUE_WIDTH;
     }
 
     @Override
@@ -305,20 +348,19 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F
 
     <#if (type.width > 8)>
     public ${minor.javaType!type.javaType} get(int index) {
-      return data.slice(index * ${type.width}, ${type.width});
+      return data.slice(index * VALUE_WIDTH, VALUE_WIDTH);
     }
 
     <#if (minor.class == "Interval")>
     public void get(int index, ${minor.class}Holder holder){
-
-      final int offsetIndex = index * ${type.width};
+      final int offsetIndex = index * VALUE_WIDTH;
       holder.months = data.getInt(offsetIndex);
       holder.days = data.getInt(offsetIndex + ${minor.daysOffset});
       holder.milliseconds = data.getInt(offsetIndex + ${minor.millisecondsOffset});
     }
 
     public void get(int index, Nullable${minor.class}Holder holder){
-      final int offsetIndex = index * ${type.width};
+      final int offsetIndex = index * VALUE_WIDTH;
       holder.isSet = 1;
       holder.months = data.getInt(offsetIndex);
       holder.days = data.getInt(offsetIndex + ${minor.daysOffset});
@@ -327,7 +369,7 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F
 
     @Override
     public ${friendlyType} getObject(int index) {
-      final int offsetIndex = index * ${type.width};
+      final int offsetIndex = index * VALUE_WIDTH;
       final int months  = data.getInt(offsetIndex);
       final int days    = data.getInt(offsetIndex + ${minor.daysOffset});
       final int millis = data.getInt(offsetIndex + ${minor.millisecondsOffset});
@@ -337,7 +379,7 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F
 
     public StringBuilder getAsStringBuilder(int index) {
 
-      final int offsetIndex = index * ${type.width};
+      final int offsetIndex = index * VALUE_WIDTH;
 
       int months  = data.getInt(offsetIndex);
       final int days    = data.getInt(offsetIndex + ${minor.daysOffset});
@@ -372,14 +414,13 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F
 
     <#elseif (minor.class == "IntervalDay")>
     public void get(int index, ${minor.class}Holder holder){
-
-      final int offsetIndex = index * ${type.width};
+      final int offsetIndex = index * VALUE_WIDTH;
       holder.days = data.getInt(offsetIndex);
       holder.milliseconds = data.getInt(offsetIndex + ${minor.millisecondsOffset});
     }
 
     public void get(int index, Nullable${minor.class}Holder holder){
-      final int offsetIndex = index * ${type.width};
+      final int offsetIndex = index * VALUE_WIDTH;
       holder.isSet = 1;
       holder.days = data.getInt(offsetIndex);
       holder.milliseconds = data.getInt(offsetIndex + ${minor.millisecondsOffset});
@@ -387,7 +428,7 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F
 
     @Override
     public ${friendlyType} getObject(int index) {
-      final int offsetIndex = index * ${type.width};
+      final int offsetIndex = index * VALUE_WIDTH;
       final int millis = data.getInt(offsetIndex + ${minor.millisecondsOffset});
       final int  days   = data.getInt(offsetIndex);
       final Period p = new Period();
@@ -395,7 +436,7 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F
     }
 
     public StringBuilder getAsStringBuilder(int index) {
-      final int offsetIndex = index * ${type.width};
+      final int offsetIndex = index * VALUE_WIDTH;
 
       int millis = data.getInt(offsetIndex + ${minor.millisecondsOffset});
       final int  days   = data.getInt(offsetIndex);
@@ -419,10 +460,9 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F
               append(millis));
     }
 
-    <#elseif (minor.class == "Decimal28Sparse") || (minor.class == "Decimal38Sparse") || (minor.class == "Decimal28Dense") || (minor.class == "Decimal38Dense")>
-
+    <#elseif minor.class == "Decimal28Sparse" || minor.class == "Decimal38Sparse" || minor.class == "Decimal28Dense" || minor.class == "Decimal38Dense">
     public void get(int index, ${minor.class}Holder holder) {
-      holder.start = index * ${type.width};
+      holder.start = index * VALUE_WIDTH;
       holder.buffer = data;
       holder.scale = getField().getScale();
       holder.precision = getField().getPrecision();
@@ -430,7 +470,7 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F
 
     public void get(int index, Nullable${minor.class}Holder holder) {
       holder.isSet = 1;
-      holder.start = index * ${type.width};
+      holder.start = index * VALUE_WIDTH;
       holder.buffer = data;
       holder.scale = getField().getScale();
       holder.precision = getField().getPrecision();
@@ -440,65 +480,61 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F
     public ${friendlyType} getObject(int index) {
       <#if (minor.class == "Decimal28Sparse") || (minor.class == "Decimal38Sparse")>
       // Get the BigDecimal object
-      return DecimalUtility.getBigDecimalFromSparse(data, index * ${type.width}, ${minor.nDecimalDigits}, getField().getScale());
+      return DecimalUtility.getBigDecimalFromSparse(data, index * VALUE_WIDTH, ${minor.nDecimalDigits}, getField().getScale());
       <#else>
-      return DecimalUtility.getBigDecimalFromDense(data, index * ${type.width}, ${minor.nDecimalDigits}, getField().getScale(), ${minor.maxPrecisionDigits}, ${type.width});
+      return DecimalUtility.getBigDecimalFromDense(data, index * VALUE_WIDTH, ${minor.nDecimalDigits}, getField().getScale(), ${minor.maxPrecisionDigits}, VALUE_WIDTH);
       </#if>
     }
 
     <#else>
     public void get(int index, ${minor.class}Holder holder){
       holder.buffer = data;
-      holder.start = index * ${type.width};
+      holder.start = index * VALUE_WIDTH;
     }
 
     public void get(int index, Nullable${minor.class}Holder holder){
       holder.isSet = 1;
       holder.buffer = data;
-      holder.start = index * ${type.width};
+      holder.start = index * VALUE_WIDTH;
     }
 
     @Override
     public ${friendlyType} getObject(int index) {
-      return data.slice(index * ${type.width}, ${type.width})
+      return data.slice(index * VALUE_WIDTH, VALUE_WIDTH)
     }
 
     </#if>
     <#else> <#-- type.width <= 8 -->
-
     public ${minor.javaType!type.javaType} get(int index) {
-      return data.get${(minor.javaType!type.javaType)?cap_first}(index * ${type.width});
+      return data.get${(minor.javaType!type.javaType)?cap_first}(index * VALUE_WIDTH);
     }
 
     <#if type.width == 4>
     public long getTwoAsLong(int index) {
-      return data.getLong(index * ${type.width});
+      return data.getLong(index * VALUE_WIDTH);
     }
 
     </#if>
-
     <#if minor.class == "Date">
     @Override
     public ${friendlyType} getObject(int index) {
-        org.joda.time.DateTime date = new org.joda.time.DateTime(get(index), org.joda.time.DateTimeZone.UTC);
-        date = date.withZoneRetainFields(org.joda.time.DateTimeZone.getDefault());
-        return date;
+      org.joda.time.DateTime date = new org.joda.time.DateTime(get(index), org.joda.time.DateTimeZone.UTC);
+      date = date.withZoneRetainFields(org.joda.time.DateTimeZone.getDefault());
+      return date;
     }
 
     <#elseif minor.class == "TimeStamp">
     @Override
     public ${friendlyType} getObject(int index) {
-        org.joda.time.DateTime date = new org.joda.time.DateTime(get(index), org.joda.time.DateTimeZone.UTC);
-        date = date.withZoneRetainFields(org.joda.time.DateTimeZone.getDefault());
-        return date;
+      org.joda.time.DateTime date = new org.joda.time.DateTime(get(index), org.joda.time.DateTimeZone.UTC);
+      date = date.withZoneRetainFields(org.joda.time.DateTimeZone.getDefault());
+      return date;
     }
 
     <#elseif minor.class == "IntervalYear">
     @Override
     public ${friendlyType} getObject(int index) {
-
       final int value = get(index);
-
       final int years  = (value / org.apache.drill.exec.expr.fn.impl.DateUtility.yearsToMonths);
       final int months = (value % org.apache.drill.exec.expr.fn.impl.DateUtility.yearsToMonths);
       final Period p = new Period();
@@ -523,18 +559,16 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F
     <#elseif minor.class == "Time">
     @Override
     public DateTime getObject(int index) {
-
-        org.joda.time.DateTime time = new org.joda.time.DateTime(get(index), org.joda.time.DateTimeZone.UTC);
-        time = time.withZoneRetainFields(org.joda.time.DateTimeZone.getDefault());
-        return time;
+      org.joda.time.DateTime time = new org.joda.time.DateTime(get(index), org.joda.time.DateTimeZone.UTC);
+      time = time.withZoneRetainFields(org.joda.time.DateTimeZone.getDefault());
+      return time;
     }
 
     <#elseif minor.class == "Decimal9" || minor.class == "Decimal18">
     @Override
     public ${friendlyType} getObject(int index) {
-
-        final BigInteger value = BigInteger.valueOf(((${type.boxedType})get(index)).${type.javaType}Value());
-        return new BigDecimal(value, getField().getScale());
+      final BigInteger value = BigInteger.valueOf(((${type.boxedType})get(index)).${type.javaType}Value());
+      return new BigDecimal(value, getField().getScale());
     }
 
     <#else>
@@ -546,191 +580,298 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F
     public ${minor.javaType!type.javaType} getPrimitiveObject(int index) {
       return get(index);
     }
-    </#if>
 
+    </#if>
     public void get(int index, ${minor.class}Holder holder){
       <#if minor.class.startsWith("Decimal")>
       holder.scale = getField().getScale();
       holder.precision = getField().getPrecision();
       </#if>
 
-      holder.value = data.get${(minor.javaType!type.javaType)?cap_first}(index * ${type.width});
+      holder.value = data.get${(minor.javaType!type.javaType)?cap_first}(index * VALUE_WIDTH);
     }
 
     public void get(int index, Nullable${minor.class}Holder holder){
       holder.isSet = 1;
-      holder.value = data.get${(minor.javaType!type.javaType)?cap_first}(index * ${type.width});
+      holder.value = data.get${(minor.javaType!type.javaType)?cap_first}(index * VALUE_WIDTH);
     }
     </#if> <#-- type.width -->
- }
-
- /**
-  * ${minor.class}.Mutator implements a mutable vector of fixed width values.  Elements in the
-  * vector are accessed by position from the logical start of the vector.  Values should be pushed
-  * onto the vector sequentially, but may be randomly accessed.
-  *   The width of each element is ${type.width} byte(s)
-  *   The equivalent Java primitive is '${minor.javaType!type.javaType}'
-  *
-  * NB: this class is automatically generated from ValueVectorTypes.tdd using FreeMarker.
-  */
-  public final class Mutator extends BaseDataValueVector.BaseMutator {
-
-    private Mutator(){};
-   /**
-    * Set the element at the given index to the given value.  Note that widths smaller than
-    * 32 bits are handled by the DrillBuf interface.
-    *
-    * @param index   position of the bit to set
-    * @param value   value to set
-    */
+  }
+
+  /**
+   * ${minor.class}.Mutator implements a mutable vector of fixed width values.  Elements in the
+   * vector are accessed by position from the logical start of the vector.  Values should be pushed
+   * onto the vector sequentially, but may be randomly accessed.
+   * <ul>
+   * <li>The width of each element is {@link #VALUE_WIDTH} (= ${type.width}) byte(s).</li>
+   * <li>The equivalent Java primitive is '${minor.javaType!type.javaType}'</li>
+   * </ul>
+   *
+   * NB: this class is automatically generated from ValueVectorTypes.tdd using FreeMarker.
+   */
+   public final class Mutator extends BaseDataValueVector.BaseMutator {
+
+    private Mutator() {};
+
+    /**
+     * Set the element at the given index to the given value.  Note that widths smaller than
+     * 32 bits are handled by the DrillBuf interface.
+     *
+     * @param index   position of the bit to set
+     * @param value   value to set
+     */
+
   <#if (type.width > 8)>
     public void set(int index, <#if (type.width > 4)>${minor.javaType!type.javaType}<#else>int</#if> value) {
-      data.setBytes(index * ${type.width}, value, 0, ${type.width});
+      data.setBytes(index * VALUE_WIDTH, value, 0, VALUE_WIDTH);
     }
 
     public void setSafe(int index, <#if (type.width > 4)>${minor.javaType!type.javaType}<#else>int</#if> value) {
       while(index >= getValueCapacity()) {
         reAlloc();
       }
-      data.setBytes(index * ${type.width}, value, 0, ${type.width});
+      data.setBytes(index * VALUE_WIDTH, value, 0, VALUE_WIDTH);
     }
 
-  <#if (minor.class == "Interval")>
-    public void set(int index, int months, int days, int milliseconds){
-      final int offsetIndex = index * ${type.width};
-      data.setInt(offsetIndex, months);
-      data.setInt((offsetIndex + ${minor.daysOffset}), days);
-      data.setInt((offsetIndex + ${minor.millisecondsOffset}), milliseconds);
+    /**
+     * Set the value of a required or nullable vector. Enforces the value
+     * and size limits.
+     * @param index item to write
+     * @param value value to set
+     * @throws VectorOverflowException if the item was written, false if the index would
+     * overfill the vector
+     */
+
+    public void setScalar(int index, <#if (type.width > 4)>${minor.javaType!type.javaType}<#else>int</#if> value) throws VectorOverflowException {
+      if (index >= MAX_SCALAR_COUNT) {
+        throw new VectorOverflowException();
+      }
+      setSafe(index, value);
     }
 
-    protected void set(int index, ${minor.class}Holder holder){
-      set(index, holder.months, holder.days, holder.milliseconds);
+    /**
+     * Set the value of a repeated vector. Enforces only the size limit.
+     * @param index item to write
+     * @param value value to set
+     * @throws VectorOverflowException if the item was written, false if the index would
+     * overfill the vector
+     */
+
+    public void setArrayItem(int index, <#if (type.width > 4)>${minor.javaType!type.javaType}<#else>int</#if> value) throws VectorOverflowException {
+      if (index >= MAX_VALUE_COUNT) {
+        throw new VectorOverflowException();
+      }
+      setSafe(index, value);
     }
 
-    protected void set(int index, Nullable${minor.class}Holder holder){
-      set(index, holder.months, holder.days, holder.milliseconds);
+    <#if minor.class == "Interval">
+    public void set(int index, int months, int days, int milliseconds) {
+      final int offsetIndex = index * VALUE_WIDTH;
+      data.setInt(offsetIndex, months);
+      data.setInt((offsetIndex + ${minor.daysOffset}), days);
+      data.setInt((offsetIndex + ${minor.millisecondsOffset}), milliseconds);
     }
 
-    public void setSafe(int index, int months, int days, int milliseconds){
+    public void setSafe(int index, int months, int days, int milliseconds) {
       while(index >= getValueCapacity()) {
         reAlloc();
       }
       set(index, months, days, milliseconds);
     }
 
-    public void setSafe(int index, Nullable${minor.class}Holder holder){
+    public void setScalar(int index, int months, int days, int milliseconds) throws VectorOverflowException {
+      if (index >= MAX_SCALAR_COUNT) {
+        throw new VectorOverflowException();
+      }
+      setSafe(index, months, days, milliseconds);
+    }
+
+    public void setArrayItem(int index, int months, int days, int milliseconds) throws VectorOverflowException {
+      if (index >= MAX_VALUE_COUNT) {
+        throw new VectorOverflowException();
+      }
+      setSafe(index, months, days, milliseconds);
+    }
+
+    protected void set(int index, ${minor.class}Holder holder) {
+      set(index, holder.months, holder.days, holder.milliseconds);
+    }
+
+    public void setSafe(int index, ${minor.class}Holder holder) {
       setSafe(index, holder.months, holder.days, holder.milliseconds);
     }
 
-    public void setSafe(int index, ${minor.class}Holder holder){
+    public void setScalar(int index, ${minor.class}Holder holder) throws VectorOverflowException {
+      setScalar(index, holder.months, holder.days, holder.milliseconds);
+    }
+
+    public void setArrayItem(int index, ${minor.class}Holder holder) throws VectorOverflowException {
+      setArrayItem(index, holder.months, holder.days, holder.milliseconds);
+    }
+
+    protected void set(int index, Nullable${minor.class}Holder holder) {
+      set(index, holder.months, holder.days, holder.milliseconds);
+    }
+
+    public void setSafe(int index, Nullable${minor.class}Holder holder) {
       setSafe(index, holder.months, holder.days, holder.milliseconds);
     }
 
-  <#elseif (minor.class == "IntervalDay")>
-    public void set(int index, int days, int milliseconds){
-      final int offsetIndex = index * ${type.width};
-      data.setInt(offsetIndex, days);
-      data.setInt((offsetIndex + ${minor.millisecondsOffset}), milliseconds);
+    public void setScalar(int index, Nullable${minor.class}Holder holder) throws VectorOverflowException {
+      setScalar(index, holder.months, holder.days, holder.milliseconds);
     }
 
-    protected void set(int index, ${minor.class}Holder holder){
-      set(index, holder.days, holder.milliseconds);
+    public void setArrayItem(int index, Nullable${minor.class}Holder holder) throws VectorOverflowException {
+      setArrayItem(index, holder.months, holder.days, holder.milliseconds);
     }
 
-    protected void set(int index, Nullable${minor.class}Holder holder){
-      set(index, holder.days, holder.milliseconds);
+    <#elseif minor.class == "IntervalDay">
+    public void set(int index, int days, int milliseconds) {
+      final int offsetIndex = index * VALUE_WIDTH;
+      data.setInt(offsetIndex, days);
+      data.setInt((offsetIndex + ${minor.millisecondsOffset}), milliseconds);
     }
 
-    public void setSafe(int index, int days, int milliseconds){
+    public void setSafe(int index, int days, int milliseconds) {
       while(index >= getValueCapacity()) {
         reAlloc();
       }
       set(index, days, milliseconds);
     }
 
-    public void setSafe(int index, ${minor.class}Holder holder){
-      setSafe(index, holder.days, holder.milliseconds);
+    public void setScalar(int index, int days, int milliseconds) throws VectorOverflowException {
+      if (index >= MAX_SCALAR_COUNT) {
+        throw new VectorOverflowException();
+      }
+      setSafe(index, days, milliseconds);
     }
 
-    public void setSafe(int index, Nullable${minor.class}Holder holder){
+    public void setArrayItem(int index, int days, int milliseconds) throws VectorOverflowException {
+      if (index >= MAX_VALUE_COUNT) {
+        throw new VectorOverflowException();
+      }
+      setSafe(index, days, milliseconds);
+    }
+
+    protected void set(int index, ${minor.class}Holder holder) {
+      set(index, holder.days, holder.milliseconds);
+    }
+
+    public void setSafe(int index, ${minor.class}Holder holder) {
       setSafe(index, holder.days, holder.milliseconds);
     }
 
-  <#elseif (minor.class == "Decimal28Sparse" || minor.class == "Decimal38Sparse") || (minor.class == "Decimal28Dense") || (minor.class == "Decimal38Dense")>
-    public void set(int index, ${minor.class}Holder holder){
-      set(index, holder.start, holder.buffer);
+    public void setScalar(int index, ${minor.class}Holder holder) throws VectorOverflowException {
+      setScalar(index, holder.days, holder.milliseconds);
     }
 
-    void set(int index, Nullable${minor.class}Holder holder){
-      set(index, holder.start, holder.buffer);
+    public void setArrayItem(int index, ${minor.class}Holder holder) throws VectorOverflowException {
+      setArrayItem(index, holder.days, holder.milliseconds);
     }
 
-    public void setSafe(int index,  Nullable${minor.class}Holder holder){
-      setSafe(index, holder.start, holder.buffer);
+    protected void set(int index, Nullable${minor.class}Holder holder) {
+      set(index, holder.days, holder.milliseconds);
     }
 
-    public void setSafe(int index,  ${minor.class}Holder holder){
-      setSafe(index, holder.start, holder.buffer);
+    public void setSafe(int index, Nullable${minor.class}Holder holder){
+      setSafe(index, holder.days, holder.milliseconds);
+    }
+
+    public void setScalar(int index, Nullable${minor.class}Holder holder) throws VectorOverflowException {
+      setScalar(index, holder.days, holder.milliseconds);
     }
 
-    public void setSafe(int index, int start, DrillBuf buffer){
+    public void setArrayItem(int index, Nullable${minor.class}Holder holder) throws VectorOverflowException {
+      setArrayItem(index, holder.days, holder.milliseconds);
+    }
+
+    <#elseif minor.class == "Decimal28Sparse" || minor.class == "Decimal38Sparse" || minor.class == "Decimal28Dense" || minor.class == "Decimal38Dense">
+    public void setSafe(int index, int start, DrillBuf buffer) {
       while(index >= getValueCapacity()) {
         reAlloc();
       }
       set(index, start, buffer);
     }
 
-  <#if minor.class == "Decimal28Sparse" || minor.class == "Decimal38Sparse">
-    public void set(int index, BigDecimal value) {
-      DecimalUtility.getSparseFromBigDecimal(value, data, index * ${type.width},
-           field.getScale(), field.getPrecision(), ${minor.nDecimalDigits});
+    public void setScalar(int index, int start, DrillBuf buffer) throws VectorOverflowException {
+      if (index >= MAX_SCALAR_COUNT) {
+        throw new VectorOverflowException();
+      }
+      setSafe(index, start, buffer);
     }
 
-    public void setSafe(int index, BigDecimal value) {
-      while(index >= getValueCapacity()) {
-        reAlloc();
+    public void setArrayItem(int index, int start, DrillBuf buffer) throws VectorOverflowException {
+      if (index >= MAX_VALUE_COUNT) {
+        throw new VectorOverflowException();
       }
-      set(index, value);
+      setSafe(index, start, buffer);
     }
 
-  </#if>
-    public void set(int index, int start, DrillBuf buffer){
-      data.setBytes(index * ${type.width}, buffer, start, ${type.width});
+    public void set(int index, ${minor.class}Holder holder) {
+      set(index, holder.start, holder.buffer);
     }
 
-  <#else>
-    protected void set(int index, ${minor.class}Holder holder){
-      set(index, holder.start, holder.buffer);
+    public void setSafe(int index, ${minor.class}Holder holder) {
+      setSafe(index, holder.start, holder.buffer);
     }
 
-    public void set(int index, Nullable${minor.class}Holder holder){
-      set(index, holder.start, holder.buffer);
+    public void setScalar(int index, ${minor.class}Holder holder) throws VectorOverflowException {
+      setScalar(index, holder.start, holder.buffer);
     }
 
-    public void set(int index, int start, DrillBuf buffer){
-      data.setBytes(index * ${type.width}, buffer, start, ${type.width});
+    public void setArrayItem(int index, ${minor.class}Holder holder) throws VectorOverflowException {
+      setArrayItem(index, holder.start, holder.buffer);
     }
 
-    public void setSafe(int index, ${minor.class}Holder holder){
-      setSafe(index, holder.start, holder.buffer);
+    void set(int index, Nullable${minor.class}Holder holder) {
+      set(index, holder.start, holder.buffer);
     }
 
-    public void setSafe(int index, Nullable${minor.class}Holder holder){
+    public void setSafe(int index, Nullable${minor.class}Holder holder) {
       setSafe(index, holder.start, holder.buffer);
     }
 
-    public void setSafe(int index, int start, DrillBuf buffer){
+    public void setScalar(int index, Nullable${minor.class}Holder holder) throws VectorOverflowException {
+      setScalar(index, holder.start, holder.buffer);
+    }
+
+    public void setArrayItem(int index, Nullable${minor.class}Holder holder) throws VectorOverflowException {
+      setArrayItem(index, holder.start, holder.buffer);
+    }
+
+      <#if minor.class == "Decimal28Sparse" || minor.class == "Decimal38Sparse">
+    public void set(int index, BigDecimal value) {
+      DecimalUtility.getSparseFromBigDecimal(value, data, index * VALUE_WIDTH,
+           field.getScale(), field.getPrecision(), ${minor.nDecimalDigits});
+    }
+
+    public void setSafe(int index, BigDecimal value) {
       while(index >= getValueCapacity()) {
         reAlloc();
       }
-      set(index, holder);
+      set(index, value);
     }
 
-    public void set(int index, Nullable${minor.class}Holder holder){
-      data.setBytes(index * ${type.width}, holder.buffer, holder.start, ${type.width});
+    public void setScalar(int index, BigDecimal value) throws VectorOverflowException {
+      if (index >= MAX_SCALAR_COUNT) {
+        throw new VectorOverflowException();
+      }
+      setSafe(index, value);
     }
-   </#if>
 
+    public void setArrayItem(int index, BigDecimal value) throws VectorOverflowException {
+      if (index >= MAX_VALUE_COUNT) {
+        throw new VectorOverflowException();
+      }
+      setSafe(index, value);
+    }
+
+      </#if>
+    public void set(int index, int start, DrillBuf buffer){
+      data.setBytes(index * VALUE_WIDTH, buffer, start, VALUE_WIDTH);
+    }
+
+    </#if>
     @Override
     public void generateTestData(int count) {
       setValueCount(count);
@@ -738,46 +879,105 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F
       final int valueCount = getAccessor().getValueCount();
       for(int i = 0; i < valueCount; i++, even = !even) {
         final byte b = even ? Byte.MIN_VALUE : Byte.MAX_VALUE;
-        for(int w = 0; w < ${type.width}; w++){
+        for(int w = 0; w < VALUE_WIDTH; w++){
           data.setByte(i + w, b);
         }
       }
     }
 
-   <#else> <#-- type.width <= 8 -->
+  <#else> <#-- type.width <= 8 -->
     public void set(int index, <#if (type.width >= 4)>${minor.javaType!type.javaType}<#else>int</#if> value) {
-      data.set${(minor.javaType!type.javaType)?cap_first}(index * ${type.width}, value);
+      data.set${(minor.javaType!type.javaType)?cap_first}(index * VALUE_WIDTH, value);
     }
 
-   public void setSafe(int index, <#if (type.width >= 4)>${minor.javaType!type.javaType}<#else>int</#if> value) {
-     while(index >= getValueCapacity()) {
+    public void setSafe(int index, <#if (type.width >= 4)>${minor.javaType!type.javaType}<#else>int</#if> value) {
+      while(index >= getValueCapacity()) {
         reAlloc();
       }
       set(index, value);
     }
 
-    protected void set(int index, ${minor.class}Holder holder){
-      data.set${(minor.javaType!type.javaType)?cap_first}(index * ${type.width}, holder.value);
+    /**
+     * Set the value of a required or nullable vector. Enforces the value
+     * and size limits.
+     * @param index item to write
+     * @param value value to set
+     * @throws VectorOverflowException if the item was written, false if the index would
+     * overfill the vector
+     */
+
+    public void setScalar(int index, <#if (type.width >= 4)>${minor.javaType!type.javaType}<#else>int</#if> value) throws VectorOverflowException {
+      if (index >= MAX_SCALAR_COUNT) {
+        throw new VectorOverflowException();
+      }
+      setSafe(index, value);
+    }
+
+    /**
+     * Set the value of a repeated vector. Enforces only the size limit.
+     * @param index item to write
+     * @param value value to set
+     * @throws VectorOverflowException if the item was written, false if the index would
+     * overfill the vector
+     */
+
+    public void setArrayItem(int index, <#if (type.width >= 4)>${minor.javaType!type.javaType}<#else>int</#if> value) throws VectorOverflowException {
+      if (index >= MAX_VALUE_COUNT) {
+        throw new VectorOverflowException();
+      }
+      setSafe(index, value);
+    }
+
+    protected void set(int index, ${minor.class}Holder holder) {
+      data.set${(minor.javaType!type.javaType)?cap_first}(index * VALUE_WIDTH, holder.value);
     }
 
-    public void setSafe(int index, ${minor.class}Holder holder){
+    public void setSafe(int index, ${minor.class}Holder holder) {
       while(index >= getValueCapacity()) {
         reAlloc();
       }
       set(index, holder);
     }
 
-    protected void set(int index, Nullable${minor.class}Holder holder){
-      data.set${(minor.javaType!type.javaType)?cap_first}(index * ${type.width}, holder.value);
+    public void setScalar(int index, ${minor.class}Holder holder) throws VectorOverflowException {
+      if (index >= MAX_SCALAR_COUNT) {
+        throw new VectorOverflowException();
+      }
+      setSafe(index, holder);
     }
 
-    public void setSafe(int index, Nullable${minor.class}Holder holder){
+    public void setArrayItem(int index, ${minor.class}Holder holder) throws VectorOverflowException {
+      if (index >= MAX_VALUE_COUNT) {
+        throw new VectorOverflowException();
+      }
+      setSafe(index, holder);
+    }
+
+    protected void set(int index, Nullable${minor.class}Holder holder) {
+      data.set${(minor.javaType!type.javaType)?cap_first}(index * VALUE_WIDTH, holder.value);
+    }
+
+    public void setSafe(int index, Nullable${minor.class}Holder holder) {
       while(index >= getValueCapacity()) {
         reAlloc();
       }
       set(index, holder);
     }
 
+    public void setScalar(int index, Nullable${minor.class}Holder holder) throws VectorOverflowException {
+      if (index >= MAX_SCALAR_COUNT) {
+        throw new VectorOverflowException();
+      }
+      setSafe(index, holder);
+    }
+
+    public void setArrayItem(int index, Nullable${minor.class}Holder holder) throws VectorOverflowException {
+      if (index >= MAX_VALUE_COUNT) {
+        throw new VectorOverflowException();
+      }
+      setSafe(index, holder);
+    }
+
     @Override
     public void generateTestData(int size) {
       setValueCount(size);
@@ -806,10 +1006,34 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F
     }
 
   </#if> <#-- type.width -->
+    /**
+     * Backfill missing offsets from the given last written position to the
+     * given current write position. Used by the "new" size-safe column
+     * writers to allow skipping values. The <tt>set()</tt> and <tt>setSafe()</tt>
+     * <b>do not</b> fill empties. See DRILL-5529 and DRILL-5530.
+     * @param lastWrite the position of the last valid write: the offset
+     * to be copied forward
+     * @param index the current write position filling occurs up to,
+     * but not including, this position
+     * @throws VectorOverflowException if the item was written, false if the index would
+     * overfill the vector
+     */
+
+    public void fillEmptiesBounded(int lastWrite, int index)
+            throws VectorOverflowException {
+  <#if type.width <= 8>
+      for (int i = lastWrite + 1; i <= index; i++) {
+        setSafe(i, <#if (type.width >= 4)>(${minor.javaType!type.javaType})</#if> 0);
+      }
+  <#else>
+      throw new UnsupportedOperationException("Cannot zero-fill ${minor.class} vectors.");
+  </#if>
+    }
+
     @Override
     public void setValueCount(int valueCount) {
       final int currentValueCapacity = getValueCapacity();
-      final int idx = (${type.width} * valueCount);
+      final int idx = (VALUE_WIDTH * valueCount);
       while(valueCount > getValueCapacity()) {
         reAlloc();
       }
@@ -819,11 +1043,10 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F
         allocationMonitor = 0;
       }
       VectorTrimmer.trim(data, idx);
-      data.writerIndex(valueCount * ${type.width});
+      data.writerIndex(valueCount * VALUE_WIDTH);
     }
   }
 }
-
 </#if> <#-- type.major -->
 </#list>
 </#list>


Mime
View raw message