arrow-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jacq...@apache.org
Subject [06/17] arrow git commit: ARROW-1: Initial Arrow Code Commit
Date Wed, 17 Feb 2016 12:39:41 GMT
http://git-wip-us.apache.org/repos/asf/arrow/blob/fa5f0299/java/vector/src/main/codegen/templates/UnionVector.java
----------------------------------------------------------------------
diff --git a/java/vector/src/main/codegen/templates/UnionVector.java b/java/vector/src/main/codegen/templates/UnionVector.java
new file mode 100644
index 0000000..ba94ac2
--- /dev/null
+++ b/java/vector/src/main/codegen/templates/UnionVector.java
@@ -0,0 +1,467 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+<@pp.dropOutputFile />
+<@pp.changeOutputFile name="/org/apache/arrow/vector/complex/UnionVector.java" />
+
+
+<#include "/@includes/license.ftl" />
+
+package org.apache.arrow.vector.complex;
+
+<#include "/@includes/vv_imports.ftl" />
+import java.util.ArrayList;
+import java.util.Iterator;
+import org.apache.arrow.vector.complex.impl.ComplexCopier;
+import org.apache.arrow.vector.util.CallBack;
+import org.apache.arrow.vector.util.BasicTypeHelper;
+
+/*
+ * This class is generated using freemarker and the ${.template_name} template.
+ */
+@SuppressWarnings("unused")
+
+
+/**
+ * A vector which can hold values of different types. It does so by using a MapVector which contains a vector for each
+ * primitive type that is stored. MapVector is used in order to take advantage of its serialization/deserialization methods,
+ * as well as the addOrGet method.
+ *
+ * For performance reasons, UnionVector stores a cached reference to each subtype vector, to avoid having to do the map lookup
+ * each time the vector is accessed.
+ */
+public class UnionVector implements ValueVector {
+
+  private MaterializedField field;
+  private BufferAllocator allocator;
+  private Accessor accessor = new Accessor();
+  private Mutator mutator = new Mutator();
+  private int valueCount;
+
+  private MapVector internalMap;
+  private UInt1Vector typeVector;
+
+  private MapVector mapVector;
+  private ListVector listVector;
+
+  private FieldReader reader;
+  private NullableBitVector bit;
+
+  private int singleType = 0;
+  private ValueVector singleVector;
+  private MajorType majorType;
+
+  private final CallBack callBack;
+
+  public UnionVector(MaterializedField field, BufferAllocator allocator, CallBack callBack) {
+    this.field = field.clone();
+    this.allocator = allocator;
+    this.internalMap = new MapVector("internal", allocator, callBack);
+    this.typeVector = internalMap.addOrGet("types", new MajorType(MinorType.UINT1, DataMode.REQUIRED), UInt1Vector.class);
+    this.field.addChild(internalMap.getField().clone());
+    this.majorType = field.getType();
+    this.callBack = callBack;
+  }
+
+  public BufferAllocator getAllocator() {
+    return allocator;
+  }
+
+  public List<MinorType> getSubTypes() {
+    return majorType.getSubTypes();
+  }
+
+  public void addSubType(MinorType type) {
+    if (majorType.getSubTypes().contains(type)) {
+      return;
+    }
+    List<MinorType> subTypes = this.majorType.getSubTypes();
+    List<MinorType> newSubTypes = new ArrayList<>(subTypes);
+    newSubTypes.add(type);
+    majorType =  new MajorType(this.majorType.getMinorType(), this.majorType.getMode(), this.majorType.getPrecision(),
+            this.majorType.getScale(), this.majorType.getTimezone(), newSubTypes);
+    field = MaterializedField.create(field.getName(), majorType);
+    if (callBack != null) {
+      callBack.doWork();
+    }
+  }
+
+  private static final MajorType MAP_TYPE = new MajorType(MinorType.MAP, DataMode.OPTIONAL);
+
+  public MapVector getMap() {
+    if (mapVector == null) {
+      int vectorCount = internalMap.size();
+      mapVector = internalMap.addOrGet("map", MAP_TYPE, MapVector.class);
+      addSubType(MinorType.MAP);
+      if (internalMap.size() > vectorCount) {
+        mapVector.allocateNew();
+      }
+    }
+    return mapVector;
+  }
+
+  <#list vv.types as type><#list type.minor as minor><#assign name = minor.class?cap_first />
+  <#assign fields = minor.fields!type.fields />
+  <#assign uncappedName = name?uncap_first/>
+  <#if !minor.class?starts_with("Decimal")>
+
+  private Nullable${name}Vector ${uncappedName}Vector;
+  private static final MajorType ${name?upper_case}_TYPE = new MajorType(MinorType.${name?upper_case}, DataMode.OPTIONAL);
+
+  public Nullable${name}Vector get${name}Vector() {
+    if (${uncappedName}Vector == null) {
+      int vectorCount = internalMap.size();
+      ${uncappedName}Vector = internalMap.addOrGet("${uncappedName}", ${name?upper_case}_TYPE, Nullable${name}Vector.class);
+      addSubType(MinorType.${name?upper_case});
+      if (internalMap.size() > vectorCount) {
+        ${uncappedName}Vector.allocateNew();
+      }
+    }
+    return ${uncappedName}Vector;
+  }
+
+  </#if>
+
+  </#list></#list>
+
+  private static final MajorType LIST_TYPE = new MajorType(MinorType.LIST, DataMode.OPTIONAL);
+
+  public ListVector getList() {
+    if (listVector == null) {
+      int vectorCount = internalMap.size();
+      listVector = internalMap.addOrGet("list", LIST_TYPE, ListVector.class);
+      addSubType(MinorType.LIST);
+      if (internalMap.size() > vectorCount) {
+        listVector.allocateNew();
+      }
+    }
+    return listVector;
+  }
+
+  public int getTypeValue(int index) {
+    return typeVector.getAccessor().get(index);
+  }
+
+  public UInt1Vector getTypeVector() {
+    return typeVector;
+  }
+
+  @Override
+  public void allocateNew() throws OutOfMemoryException {
+    internalMap.allocateNew();
+    if (typeVector != null) {
+      typeVector.zeroVector();
+    }
+  }
+
+  @Override
+  public boolean allocateNewSafe() {
+    boolean safe = internalMap.allocateNewSafe();
+    if (safe) {
+      if (typeVector != null) {
+        typeVector.zeroVector();
+      }
+    }
+    return safe;
+  }
+
+  @Override
+  public void setInitialCapacity(int numRecords) {
+  }
+
+  @Override
+  public int getValueCapacity() {
+    return Math.min(typeVector.getValueCapacity(), internalMap.getValueCapacity());
+  }
+
+  @Override
+  public void close() {
+  }
+
+  @Override
+  public void clear() {
+    internalMap.clear();
+  }
+
+  @Override
+  public MaterializedField getField() {
+    return field;
+  }
+
+  @Override
+  public TransferPair getTransferPair(BufferAllocator allocator) {
+    return new TransferImpl(field, allocator);
+  }
+
+  @Override
+  public TransferPair getTransferPair(String ref, BufferAllocator allocator) {
+    return new TransferImpl(field.withPath(ref), allocator);
+  }
+
+  @Override
+  public TransferPair makeTransferPair(ValueVector target) {
+    return new TransferImpl((UnionVector) target);
+  }
+
+  public void transferTo(UnionVector target) {
+    internalMap.makeTransferPair(target.internalMap).transfer();
+    target.valueCount = valueCount;
+    target.majorType = majorType;
+  }
+
+  public void copyFrom(int inIndex, int outIndex, UnionVector from) {
+    from.getReader().setPosition(inIndex);
+    getWriter().setPosition(outIndex);
+    ComplexCopier.copy(from.reader, mutator.writer);
+  }
+
+  public void copyFromSafe(int inIndex, int outIndex, UnionVector from) {
+    copyFrom(inIndex, outIndex, from);
+  }
+
+  public ValueVector addVector(ValueVector v) {
+    String name = v.getField().getType().getMinorType().name().toLowerCase();
+    MajorType type = v.getField().getType();
+    Preconditions.checkState(internalMap.getChild(name) == null, String.format("%s vector already exists", name));
+    final ValueVector newVector = internalMap.addOrGet(name, type, (Class<ValueVector>) BasicTypeHelper.getValueVectorClass(type.getMinorType(), type.getMode()));
+    v.makeTransferPair(newVector).transfer();
+    internalMap.putChild(name, newVector);
+    addSubType(v.getField().getType().getMinorType());
+    return newVector;
+  }
+
+  private class TransferImpl implements TransferPair {
+
+    UnionVector to;
+
+    public TransferImpl(MaterializedField field, BufferAllocator allocator) {
+      to = new UnionVector(field, allocator, null);
+    }
+
+    public TransferImpl(UnionVector to) {
+      this.to = to;
+    }
+
+    @Override
+    public void transfer() {
+      transferTo(to);
+    }
+
+    @Override
+    public void splitAndTransfer(int startIndex, int length) {
+
+    }
+
+    @Override
+    public ValueVector getTo() {
+      return to;
+    }
+
+    @Override
+    public void copyValueSafe(int from, int to) {
+      this.to.copyFrom(from, to, UnionVector.this);
+    }
+  }
+
+  @Override
+  public Accessor getAccessor() {
+    return accessor;
+  }
+
+  @Override
+  public Mutator getMutator() {
+    return mutator;
+  }
+
+  @Override
+  public FieldReader getReader() {
+    if (reader == null) {
+      reader = new UnionReader(this);
+    }
+    return reader;
+  }
+
+  public FieldWriter getWriter() {
+    if (mutator.writer == null) {
+      mutator.writer = new UnionWriter(this);
+    }
+    return mutator.writer;
+  }
+
+//  @Override
+//  public UserBitShared.SerializedField getMetadata() {
+//    SerializedField.Builder b = getField() //
+//            .getAsBuilder() //
+//            .setBufferLength(getBufferSize()) //
+//            .setValueCount(valueCount);
+//
+//    b.addChild(internalMap.getMetadata());
+//    return b.build();
+//  }
+
+  @Override
+  public int getBufferSize() {
+    return internalMap.getBufferSize();
+  }
+
+  @Override
+  public int getBufferSizeFor(final int valueCount) {
+    if (valueCount == 0) {
+      return 0;
+    }
+
+    long bufferSize = 0;
+    for (final ValueVector v : (Iterable<ValueVector>) this) {
+      bufferSize += v.getBufferSizeFor(valueCount);
+    }
+
+    return (int) bufferSize;
+  }
+
+  @Override
+  public ArrowBuf[] getBuffers(boolean clear) {
+    return internalMap.getBuffers(clear);
+  }
+
+  @Override
+  public Iterator<ValueVector> iterator() {
+    List<ValueVector> vectors = Lists.newArrayList(internalMap.iterator());
+    vectors.add(typeVector);
+    return vectors.iterator();
+  }
+
+  public class Accessor extends BaseValueVector.BaseAccessor {
+
+
+    @Override
+    public Object getObject(int index) {
+      int type = typeVector.getAccessor().get(index);
+      switch (MinorType.values()[type]) {
+      case LATE:
+        return null;
+      <#list vv.types as type><#list type.minor as minor><#assign name = minor.class?cap_first />
+      <#assign fields = minor.fields!type.fields />
+      <#assign uncappedName = name?uncap_first/>
+      <#if !minor.class?starts_with("Decimal")>
+      case ${name?upper_case}:
+        return get${name}Vector().getAccessor().getObject(index);
+      </#if>
+
+      </#list></#list>
+      case MAP:
+        return getMap().getAccessor().getObject(index);
+      case LIST:
+        return getList().getAccessor().getObject(index);
+      default:
+        throw new UnsupportedOperationException("Cannot support type: " + MinorType.values()[type]);
+      }
+    }
+
+    public byte[] get(int index) {
+      return null;
+    }
+
+    public void get(int index, ComplexHolder holder) {
+    }
+
+    public void get(int index, UnionHolder holder) {
+      FieldReader reader = new UnionReader(UnionVector.this);
+      reader.setPosition(index);
+      holder.reader = reader;
+    }
+
+    @Override
+    public int getValueCount() {
+      return valueCount;
+    }
+
+    @Override
+    public boolean isNull(int index) {
+      return typeVector.getAccessor().get(index) == 0;
+    }
+
+    public int isSet(int index) {
+      return isNull(index) ? 0 : 1;
+    }
+  }
+
+  public class Mutator extends BaseValueVector.BaseMutator {
+
+    UnionWriter writer;
+
+    @Override
+    public void setValueCount(int valueCount) {
+      UnionVector.this.valueCount = valueCount;
+      internalMap.getMutator().setValueCount(valueCount);
+    }
+
+    public void setSafe(int index, UnionHolder holder) {
+      FieldReader reader = holder.reader;
+      if (writer == null) {
+        writer = new UnionWriter(UnionVector.this);
+      }
+      writer.setPosition(index);
+      MinorType type = reader.getType().getMinorType();
+      switch (type) {
+      <#list vv.types as type><#list type.minor as minor><#assign name = minor.class?cap_first />
+      <#assign fields = minor.fields!type.fields />
+      <#assign uncappedName = name?uncap_first/>
+      <#if !minor.class?starts_with("Decimal")>
+      case ${name?upper_case}:
+        Nullable${name}Holder ${uncappedName}Holder = new Nullable${name}Holder();
+        reader.read(${uncappedName}Holder);
+        setSafe(index, ${uncappedName}Holder);
+        break;
+      </#if>
+      </#list></#list>
+      case MAP: {
+        ComplexCopier.copy(reader, writer);
+        break;
+      }
+      case LIST: {
+        ComplexCopier.copy(reader, writer);
+        break;
+      }
+      default:
+        throw new UnsupportedOperationException();
+      }
+    }
+
+    <#list vv.types as type><#list type.minor as minor><#assign name = minor.class?cap_first />
+    <#assign fields = minor.fields!type.fields />
+    <#assign uncappedName = name?uncap_first/>
+    <#if !minor.class?starts_with("Decimal")>
+    public void setSafe(int index, Nullable${name}Holder holder) {
+      setType(index, MinorType.${name?upper_case});
+      get${name}Vector().getMutator().setSafe(index, holder);
+    }
+
+    </#if>
+    </#list></#list>
+
+    public void setType(int index, MinorType type) {
+      typeVector.getMutator().setSafe(index, type.ordinal());
+    }
+
+    @Override
+    public void reset() { }
+
+    @Override
+    public void generateTestData(int values) { }
+  }
+}

http://git-wip-us.apache.org/repos/asf/arrow/blob/fa5f0299/java/vector/src/main/codegen/templates/UnionWriter.java
----------------------------------------------------------------------
diff --git a/java/vector/src/main/codegen/templates/UnionWriter.java b/java/vector/src/main/codegen/templates/UnionWriter.java
new file mode 100644
index 0000000..c9c29e0
--- /dev/null
+++ b/java/vector/src/main/codegen/templates/UnionWriter.java
@@ -0,0 +1,228 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+<@pp.dropOutputFile />
+<@pp.changeOutputFile name="/org/apache/arrow/vector/complex/impl/UnionWriter.java" />
+
+
+<#include "/@includes/license.ftl" />
+
+package org.apache.arrow.vector.complex.impl;
+
+<#include "/@includes/vv_imports.ftl" />
+
+/*
+ * This class is generated using freemarker and the ${.template_name} template.
+ */
+@SuppressWarnings("unused")
+public class UnionWriter extends AbstractFieldWriter implements FieldWriter {
+
+  UnionVector data;
+  private MapWriter mapWriter;
+  private UnionListWriter listWriter;
+  private List<BaseWriter> writers = Lists.newArrayList();
+
+  public UnionWriter(BufferAllocator allocator) {
+    super(null);
+  }
+
+  public UnionWriter(UnionVector vector) {
+    super(null);
+    data = vector;
+  }
+
+  public UnionWriter(UnionVector vector, FieldWriter parent) {
+    super(null);
+    data = vector;
+  }
+
+  @Override
+  public void setPosition(int index) {
+    super.setPosition(index);
+    for (BaseWriter writer : writers) {
+      writer.setPosition(index);
+    }
+  }
+
+
+  @Override
+  public void start() {
+    data.getMutator().setType(idx(), MinorType.MAP);
+    getMapWriter().start();
+  }
+
+  @Override
+  public void end() {
+    getMapWriter().end();
+  }
+
+  @Override
+  public void startList() {
+    getListWriter().startList();
+    data.getMutator().setType(idx(), MinorType.LIST);
+  }
+
+  @Override
+  public void endList() {
+    getListWriter().endList();
+  }
+
+  private MapWriter getMapWriter() {
+    if (mapWriter == null) {
+      mapWriter = new SingleMapWriter(data.getMap(), null, true);
+      mapWriter.setPosition(idx());
+      writers.add(mapWriter);
+    }
+    return mapWriter;
+  }
+
+  public MapWriter asMap() {
+    data.getMutator().setType(idx(), MinorType.MAP);
+    return getMapWriter();
+  }
+
+  private ListWriter getListWriter() {
+    if (listWriter == null) {
+      listWriter = new UnionListWriter(data.getList());
+      listWriter.setPosition(idx());
+      writers.add(listWriter);
+    }
+    return listWriter;
+  }
+
+  public ListWriter asList() {
+    data.getMutator().setType(idx(), MinorType.LIST);
+    return getListWriter();
+  }
+
+  <#list vv.types as type><#list type.minor as minor><#assign name = minor.class?cap_first />
+  <#assign fields = minor.fields!type.fields />
+  <#assign uncappedName = name?uncap_first/>
+
+          <#if !minor.class?starts_with("Decimal")>
+
+  private ${name}Writer ${name?uncap_first}Writer;
+
+  private ${name}Writer get${name}Writer() {
+    if (${uncappedName}Writer == null) {
+      ${uncappedName}Writer = new Nullable${name}WriterImpl(data.get${name}Vector(), null);
+      ${uncappedName}Writer.setPosition(idx());
+      writers.add(${uncappedName}Writer);
+    }
+    return ${uncappedName}Writer;
+  }
+
+  public ${name}Writer as${name}() {
+    data.getMutator().setType(idx(), MinorType.${name?upper_case});
+    return get${name}Writer();
+  }
+
+  @Override
+  public void write(${name}Holder holder) {
+    data.getMutator().setType(idx(), MinorType.${name?upper_case});
+    get${name}Writer().setPosition(idx());
+    get${name}Writer().write${name}(<#list fields as field>holder.${field.name}<#if field_has_next>, </#if></#list>);
+  }
+
+  public void write${minor.class}(<#list fields as field>${field.type} ${field.name}<#if field_has_next>, </#if></#list>) {
+    data.getMutator().setType(idx(), MinorType.${name?upper_case});
+    get${name}Writer().setPosition(idx());
+    get${name}Writer().write${name}(<#list fields as field>${field.name}<#if field_has_next>, </#if></#list>);
+  }
+  </#if>
+
+  </#list></#list>
+
+  public void writeNull() {
+  }
+
+  @Override
+  public MapWriter map() {
+    data.getMutator().setType(idx(), MinorType.LIST);
+    getListWriter().setPosition(idx());
+    return getListWriter().map();
+  }
+
+  @Override
+  public ListWriter list() {
+    data.getMutator().setType(idx(), MinorType.LIST);
+    getListWriter().setPosition(idx());
+    return getListWriter().list();
+  }
+
+  @Override
+  public ListWriter list(String name) {
+    data.getMutator().setType(idx(), MinorType.MAP);
+    getMapWriter().setPosition(idx());
+    return getMapWriter().list(name);
+  }
+
+  @Override
+  public MapWriter map(String name) {
+    data.getMutator().setType(idx(), MinorType.MAP);
+    getMapWriter().setPosition(idx());
+    return getMapWriter().map(name);
+  }
+
+  <#list vv.types as type><#list type.minor as minor>
+  <#assign lowerName = minor.class?uncap_first />
+  <#if lowerName == "int" ><#assign lowerName = "integer" /></#if>
+  <#assign upperName = minor.class?upper_case />
+  <#assign capName = minor.class?cap_first />
+  <#if !minor.class?starts_with("Decimal")>
+  @Override
+  public ${capName}Writer ${lowerName}(String name) {
+    data.getMutator().setType(idx(), MinorType.MAP);
+    getMapWriter().setPosition(idx());
+    return getMapWriter().${lowerName}(name);
+  }
+
+  @Override
+  public ${capName}Writer ${lowerName}() {
+    data.getMutator().setType(idx(), MinorType.LIST);
+    getListWriter().setPosition(idx());
+    return getListWriter().${lowerName}();
+  }
+  </#if>
+  </#list></#list>
+
+  @Override
+  public void allocate() {
+    data.allocateNew();
+  }
+
+  @Override
+  public void clear() {
+    data.clear();
+  }
+
+  @Override
+  public void close() throws Exception {
+    data.close();
+  }
+
+  @Override
+  public MaterializedField getField() {
+    return data.getField();
+  }
+
+  @Override
+  public int getValueCapacity() {
+    return data.getValueCapacity();
+  }
+}

http://git-wip-us.apache.org/repos/asf/arrow/blob/fa5f0299/java/vector/src/main/codegen/templates/ValueHolders.java
----------------------------------------------------------------------
diff --git a/java/vector/src/main/codegen/templates/ValueHolders.java b/java/vector/src/main/codegen/templates/ValueHolders.java
new file mode 100644
index 0000000..2b14194
--- /dev/null
+++ b/java/vector/src/main/codegen/templates/ValueHolders.java
@@ -0,0 +1,116 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+<@pp.dropOutputFile />
+<#list vv.modes as mode>
+<#list vv.types as type>
+<#list type.minor as minor>
+
+<#assign className="${mode.prefix}${minor.class}Holder" />
+<@pp.changeOutputFile name="/org/apache/arrow/vector/holders/${className}.java" />
+
+<#include "/@includes/license.ftl" />
+
+package org.apache.arrow.vector.holders;
+
+<#include "/@includes/vv_imports.ftl" />
+
+public final class ${className} implements ValueHolder{
+  
+  public static final MajorType TYPE = new MajorType(MinorType.${minor.class?upper_case}, DataMode.${mode.name?upper_case});
+
+  public MajorType getType() {return TYPE;}
+  
+    <#if mode.name == "Repeated">
+    
+    /** The first index (inclusive) into the Vector. **/
+    public int start;
+    
+    /** The last index (exclusive) into the Vector. **/
+    public int end;
+    
+    /** The Vector holding the actual values. **/
+    public ${minor.class}Vector vector;
+    
+    <#else>
+    public static final int WIDTH = ${type.width};
+    
+    <#if mode.name == "Optional">public int isSet;</#if>
+    <#assign fields = minor.fields!type.fields />
+    <#list fields as field>
+    public ${field.type} ${field.name};
+    </#list>
+    
+    <#if minor.class.startsWith("Decimal")>
+    public static final int maxPrecision = ${minor.maxPrecisionDigits};
+    <#if minor.class.startsWith("Decimal28") || minor.class.startsWith("Decimal38")>
+    public static final int nDecimalDigits = ${minor.nDecimalDigits};
+    
+    public static int getInteger(int index, int start, ArrowBuf buffer) {
+      int value = buffer.getInt(start + (index * 4));
+
+      if (index == 0) {
+          /* the first byte contains sign bit, return value without it */
+          <#if minor.class.endsWith("Sparse")>
+          value = (value & 0x7FFFFFFF);
+          <#elseif minor.class.endsWith("Dense")>
+          value = (value & 0x0000007F);
+          </#if>
+      }
+      return value;
+    }
+
+    public static void setInteger(int index, int value, int start, ArrowBuf buffer) {
+        buffer.setInt(start + (index * 4), value);
+    }
+  
+    public static void setSign(boolean sign, int start, ArrowBuf buffer) {
+      // Set MSB to 1 if sign is negative
+      if (sign == true) {
+        int value = getInteger(0, start, buffer);
+        setInteger(0, (value | 0x80000000), start, buffer);
+      }
+    }
+  
+    public static boolean getSign(int start, ArrowBuf buffer) {
+      return ((buffer.getInt(start) & 0x80000000) != 0);
+    }
+    </#if></#if>
+    
+    @Deprecated
+    public int hashCode(){
+      throw new UnsupportedOperationException();
+    }
+
+    /*
+     * Reason for deprecation is that ValueHolders are potential scalar replacements
+     * and hence we don't want any methods to be invoked on them.
+     */
+    @Deprecated
+    public String toString(){
+      throw new UnsupportedOperationException();
+    }
+    </#if>
+    
+    
+    
+    
+}
+
+</#list>
+</#list>
+</#list>
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/arrow/blob/fa5f0299/java/vector/src/main/codegen/templates/VariableLengthVectors.java
----------------------------------------------------------------------
diff --git a/java/vector/src/main/codegen/templates/VariableLengthVectors.java b/java/vector/src/main/codegen/templates/VariableLengthVectors.java
new file mode 100644
index 0000000..13d53b8
--- /dev/null
+++ b/java/vector/src/main/codegen/templates/VariableLengthVectors.java
@@ -0,0 +1,644 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.lang.Override;
+
+import org.apache.drill.exec.exception.OutOfMemoryException;
+import org.apache.drill.exec.vector.BaseDataValueVector;
+import org.apache.drill.exec.vector.BaseValueVector;
+import org.apache.drill.exec.vector.VariableWidthVector;
+
+<@pp.dropOutputFile />
+<#list vv.types as type>
+<#list type.minor as minor>
+
+<#assign friendlyType = (minor.friendlyType!minor.boxedType!type.boxedType) />
+
+<#if type.major == "VarLen">
+<@pp.changeOutputFile name="/org/apache/arrow/vector/${minor.class}Vector.java" />
+
+<#include "/@includes/license.ftl" />
+
+package org.apache.arrow.vector;
+
+<#include "/@includes/vv_imports.ftl" />
+
+/**
+ * ${minor.class}Vector implements a vector of variable width values.  Elements in the vector
+ * are accessed by position from the logical start of the vector.  A fixed width offsetVector
+ * is used to convert an element's position to it's offset from the start of the (0-based)
+ * ArrowBuf.  Size is inferred by adjacent elements.
+ *   The width of each element is ${type.width} byte(s)
+ *   The equivalent Java primitive is '${minor.javaType!type.javaType}'
+ *
+ * NB: this class is automatically generated from ${.template_name} and ValueVectorTypes.tdd using FreeMarker.
+ */
+public final class ${minor.class}Vector extends BaseDataValueVector implements VariableWidthVector{
+  private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(${minor.class}Vector.class);
+
+  private static final int DEFAULT_RECORD_BYTE_COUNT = 8;
+  private static final int INITIAL_BYTE_COUNT = 4096 * DEFAULT_RECORD_BYTE_COUNT;
+  private static final int MIN_BYTE_COUNT = 4096;
+
+  public final static String OFFSETS_VECTOR_NAME = "$offsets$";
+  private final MaterializedField offsetsField = MaterializedField.create(OFFSETS_VECTOR_NAME, new MajorType(MinorType.UINT4, DataMode.REQUIRED));
+  private final UInt${type.width}Vector offsetVector = new UInt${type.width}Vector(offsetsField, allocator);
+  private final FieldReader reader = new ${minor.class}ReaderImpl(${minor.class}Vector.this);
+
+  private final Accessor accessor;
+  private final Mutator mutator;
+
+  private final UInt${type.width}Vector.Accessor oAccessor;
+
+  private int allocationSizeInBytes = INITIAL_BYTE_COUNT;
+  private int allocationMonitor = 0;
+
+  public ${minor.class}Vector(MaterializedField field, BufferAllocator allocator) {
+    super(field, allocator);
+    this.oAccessor = offsetVector.getAccessor();
+    this.accessor = new Accessor();
+    this.mutator = new Mutator();
+  }
+
+  @Override
+  public FieldReader getReader(){
+    return reader;
+  }
+
+  @Override
+  public int getBufferSize(){
+    if (getAccessor().getValueCount() == 0) {
+      return 0;
+    }
+    return offsetVector.getBufferSize() + data.writerIndex();
+  }
+
+  @Override
+  public int getBufferSizeFor(final int valueCount) {
+    if (valueCount == 0) {
+      return 0;
+    }
+
+    final int idx = offsetVector.getAccessor().get(valueCount);
+    return offsetVector.getBufferSizeFor(valueCount + 1) + idx;
+  }
+
+  @Override
+  public int getValueCapacity(){
+    return Math.max(offsetVector.getValueCapacity() - 1, 0);
+  }
+
+  @Override
+  public int getByteCapacity(){
+    return data.capacity();
+  }
+
+  @Override
+  public int getCurrentSizeInBytes() {
+    return offsetVector.getAccessor().get(getAccessor().getValueCount());
+  }
+
+  /**
+   * Return the number of bytes contained in the current var len byte vector.
+   * @return
+   */
+  public int getVarByteLength(){
+    final int valueCount = getAccessor().getValueCount();
+    if(valueCount == 0) {
+      return 0;
+    }
+    return offsetVector.getAccessor().get(valueCount);
+  }
+
+//  @Override
+//  public SerializedField getMetadata() {
+//    return getMetadataBuilder() //
+//             .addChild(offsetVector.getMetadata())
+//             .setValueCount(getAccessor().getValueCount()) //
+//             .setBufferLength(getBufferSize()) //
+//             .build();
+//  }
+//
+//  @Override
+//  public void load(SerializedField metadata, ArrowBuf buffer) {
+//     the bits vector is the first child (the order in which the children are added in getMetadataBuilder is significant)
+//    final SerializedField offsetField = metadata.getChild(0);
+//    offsetVector.load(offsetField, buffer);
+//
+//    final int capacity = buffer.capacity();
+//    final int offsetsLength = offsetField.getBufferLength();
+//    data = buffer.slice(offsetsLength, capacity - offsetsLength);
+//    data.retain();
+//  }
+
+  @Override
+  public void clear() {
+    super.clear();
+    offsetVector.clear();
+  }
+
+  @Override
+  public ArrowBuf[] getBuffers(boolean clear) {
+    final ArrowBuf[] buffers = ObjectArrays.concat(offsetVector.getBuffers(false), super.getBuffers(false), ArrowBuf.class);
+    if (clear) {
+      // does not make much sense but we have to retain buffers even when clear is set. refactor this interface.
+      for (final ArrowBuf buffer:buffers) {
+        buffer.retain(1);
+      }
+      clear();
+    }
+    return buffers;
+  }
+
+  public long getOffsetAddr(){
+    return offsetVector.getBuffer().memoryAddress();
+  }
+
+  public UInt${type.width}Vector getOffsetVector(){
+    return offsetVector;
+  }
+
+  @Override
+  public TransferPair getTransferPair(BufferAllocator allocator){
+    return new TransferImpl(getField(), allocator);
+  }
+
+  @Override
+  public TransferPair getTransferPair(String ref, BufferAllocator allocator){
+    return new TransferImpl(getField().withPath(ref), allocator);
+  }
+
+  @Override
+  public TransferPair makeTransferPair(ValueVector to) {
+    return new TransferImpl((${minor.class}Vector) to);
+  }
+
+  public void transferTo(${minor.class}Vector target){
+    target.clear();
+    this.offsetVector.transferTo(target.offsetVector);
+    target.data = data.transferOwnership(target.allocator).buffer;
+    target.data.writerIndex(data.writerIndex());
+    clear();
+  }
+
+  public void splitAndTransferTo(int startIndex, int length, ${minor.class}Vector target) {
+    UInt${type.width}Vector.Accessor offsetVectorAccessor = this.offsetVector.getAccessor();
+    final int startPoint = offsetVectorAccessor.get(startIndex);
+    final int sliceLength = offsetVectorAccessor.get(startIndex + length) - startPoint;
+    target.clear();
+    target.offsetVector.allocateNew(length + 1);
+    offsetVectorAccessor = this.offsetVector.getAccessor();
+    final UInt4Vector.Mutator targetOffsetVectorMutator = target.offsetVector.getMutator();
+    for (int i = 0; i < length + 1; i++) {
+      targetOffsetVectorMutator.set(i, offsetVectorAccessor.get(startIndex + i) - startPoint);
+    }
+    target.data = data.slice(startPoint, sliceLength).transferOwnership(target.allocator).buffer;
+    target.getMutator().setValueCount(length);
+}
+
+  protected void copyFrom(int fromIndex, int thisIndex, ${minor.class}Vector from){
+    final UInt4Vector.Accessor fromOffsetVectorAccessor = from.offsetVector.getAccessor();
+    final int start = fromOffsetVectorAccessor.get(fromIndex);
+    final int end = fromOffsetVectorAccessor.get(fromIndex + 1);
+    final int len = end - start;
+
+    final int outputStart = offsetVector.data.get${(minor.javaType!type.javaType)?cap_first}(thisIndex * ${type.width});
+    from.data.getBytes(start, data, outputStart, len);
+    offsetVector.data.set${(minor.javaType!type.javaType)?cap_first}( (thisIndex+1) * ${type.width}, outputStart + len);
+  }
+
+  public boolean copyFromSafe(int fromIndex, int thisIndex, ${minor.class}Vector from){
+    final UInt${type.width}Vector.Accessor fromOffsetVectorAccessor = from.offsetVector.getAccessor();
+    final int start = fromOffsetVectorAccessor.get(fromIndex);
+    final int end =   fromOffsetVectorAccessor.get(fromIndex + 1);
+    final int len = end - start;
+    final int outputStart = offsetVector.data.get${(minor.javaType!type.javaType)?cap_first}(thisIndex * ${type.width});
+
+    while(data.capacity() < outputStart + len) {
+        reAlloc();
+    }
+
+    offsetVector.getMutator().setSafe(thisIndex + 1, outputStart + len);
+    from.data.getBytes(start, data, outputStart, len);
+    return true;
+  }
+
+  private class TransferImpl implements TransferPair{
+    ${minor.class}Vector to;
+
+    public TransferImpl(MaterializedField field, BufferAllocator allocator){
+      to = new ${minor.class}Vector(field, allocator);
+    }
+
+    public TransferImpl(${minor.class}Vector to){
+      this.to = to;
+    }
+
+    @Override
+    public ${minor.class}Vector getTo(){
+      return to;
+    }
+
+    @Override
+    public void transfer(){
+      transferTo(to);
+    }
+
+    @Override
+    public void splitAndTransfer(int startIndex, int length) {
+      splitAndTransferTo(startIndex, length, to);
+    }
+
+    @Override
+    public void copyValueSafe(int fromIndex, int toIndex) {
+      to.copyFromSafe(fromIndex, toIndex, ${minor.class}Vector.this);
+    }
+  }
+
+  @Override
+  public void setInitialCapacity(final int valueCount) {
+    final long size = 1L * valueCount * ${type.width};
+    if (size > MAX_ALLOCATION_SIZE) {
+      throw new OversizedAllocationException("Requested amount of memory is more than max allowed allocation size");
+    }
+    allocationSizeInBytes = (int)size;
+    offsetVector.setInitialCapacity(valueCount + 1);
+  }
+
+  @Override
+  public void allocateNew() {
+    if(!allocateNewSafe()){
+      throw new OutOfMemoryException("Failure while allocating buffer.");
+    }
+  }
+
+  @Override
+  public boolean allocateNewSafe() {
+    long curAllocationSize = allocationSizeInBytes;
+    if (allocationMonitor > 10) {
+      curAllocationSize = Math.max(MIN_BYTE_COUNT, curAllocationSize / 2);
+      allocationMonitor = 0;
+    } else if (allocationMonitor < -2) {
+      curAllocationSize = curAllocationSize * 2L;
+      allocationMonitor = 0;
+    }
+
+    if (curAllocationSize > MAX_ALLOCATION_SIZE) {
+      return false;
+    }
+
+    clear();
+    /* Boolean to keep track if all the memory allocations were successful
+     * Used in the case of composite vectors when we need to allocate multiple
+     * buffers for multiple vectors. If one of the allocations failed we need to
+     * clear all the memory that we allocated
+     */
+    try {
+      final int requestedSize = (int)curAllocationSize;
+      data = allocator.buffer(requestedSize);
+      allocationSizeInBytes = requestedSize;
+      offsetVector.allocateNew();
+    } catch (OutOfMemoryException e) {
+      clear();
+      return false;
+    }
+    data.readerIndex(0);
+    offsetVector.zeroVector();
+    return true;
+  }
+
+  @Override
+  public void allocateNew(int totalBytes, int valueCount) {
+    clear();
+    assert totalBytes >= 0;
+    try {
+      data = allocator.buffer(totalBytes);
+      offsetVector.allocateNew(valueCount + 1);
+    } catch (RuntimeException e) {
+      clear();
+      throw e;
+    }
+    data.readerIndex(0);
+    allocationSizeInBytes = totalBytes;
+    offsetVector.zeroVector();
+  }
+
+  @Override
+  public void reset() {
+    allocationSizeInBytes = INITIAL_BYTE_COUNT;
+    allocationMonitor = 0;
+    data.readerIndex(0);
+    offsetVector.zeroVector();
+    super.reset();
+  }
+
+  public void reAlloc() {
+    final long newAllocationSize = allocationSizeInBytes*2L;
+    if (newAllocationSize > MAX_ALLOCATION_SIZE)  {
+      throw new OversizedAllocationException("Unable to expand the buffer. Max allowed buffer size is reached.");
+    }
+
+    final ArrowBuf newBuf = allocator.buffer((int)newAllocationSize);
+    newBuf.setBytes(0, data, 0, data.capacity());
+    data.release();
+    data = newBuf;
+    allocationSizeInBytes = (int)newAllocationSize;
+  }
+
+  public void decrementAllocationMonitor() {
+    if (allocationMonitor > 0) {
+      allocationMonitor = 0;
+    }
+    --allocationMonitor;
+  }
+
+  private void incrementAllocationMonitor() {
+    ++allocationMonitor;
+  }
+
+  @Override
+  public Accessor getAccessor(){
+    return accessor;
+  }
+
+  @Override
+  public Mutator getMutator() {
+    return mutator;
+  }
+
+  public final class Accessor extends BaseValueVector.BaseAccessor implements VariableWidthAccessor {
+    final UInt${type.width}Vector.Accessor oAccessor = offsetVector.getAccessor();
+    public long getStartEnd(int index){
+      return oAccessor.getTwoAsLong(index);
+    }
+
+    public byte[] get(int index) {
+      assert index >= 0;
+      final int startIdx = oAccessor.get(index);
+      final int length = oAccessor.get(index + 1) - startIdx;
+      assert length >= 0;
+      final byte[] dst = new byte[length];
+      data.getBytes(startIdx, dst, 0, length);
+      return dst;
+    }
+
+    @Override
+    public int getValueLength(int index) {
+      final UInt${type.width}Vector.Accessor offsetVectorAccessor = offsetVector.getAccessor();
+      return offsetVectorAccessor.get(index + 1) - offsetVectorAccessor.get(index);
+    }
+
+    public void get(int index, ${minor.class}Holder holder){
+      holder.start = oAccessor.get(index);
+      holder.end = oAccessor.get(index + 1);
+      holder.buffer = data;
+    }
+
+    public void get(int index, Nullable${minor.class}Holder holder){
+      holder.isSet = 1;
+      holder.start = oAccessor.get(index);
+      holder.end = oAccessor.get(index + 1);
+      holder.buffer = data;
+    }
+
+
+    <#switch minor.class>
+    <#case "VarChar">
+    @Override
+    public ${friendlyType} getObject(int index) {
+      Text text = new Text();
+      text.set(get(index));
+      return text;
+    }
+    <#break>
+    <#case "Var16Char">
+    @Override
+    public ${friendlyType} getObject(int index) {
+      return new String(get(index), Charsets.UTF_16);
+    }
+    <#break>
+    <#default>
+    @Override
+    public ${friendlyType} getObject(int index) {
+      return get(index);
+    }
+    </#switch>
+
+    @Override
+    public int getValueCount() {
+      return Math.max(offsetVector.getAccessor().getValueCount()-1, 0);
+    }
+
+    @Override
+    public boolean isNull(int index){
+      return false;
+    }
+
+    public UInt${type.width}Vector getOffsetVector(){
+      return offsetVector;
+    }
+  }
+
+  /**
+   * Mutable${minor.class} implements a vector of variable width values.  Elements in the vector
+   * are accessed by position from the logical start of the vector.  A fixed width offsetVector
+   * is used to convert an element's position to it's offset from the start of the (0-based)
+   * ArrowBuf.  Size is inferred by adjacent elements.
+   *   The width of each element is ${type.width} byte(s)
+   *   The equivalent Java primitive is '${minor.javaType!type.javaType}'
+   *
+   * NB: this class is automatically generated from ValueVectorTypes.tdd using FreeMarker.
+   */
+  public final class Mutator extends BaseValueVector.BaseMutator implements VariableWidthVector.VariableWidthMutator {
+
+    /**
+     * Set the variable length element at the specified index to the supplied byte array.
+     *
+     * @param index   position of the bit to set
+     * @param bytes   array of bytes to write
+     */
+    protected void set(int index, byte[] bytes) {
+      assert index >= 0;
+      final int currentOffset = offsetVector.getAccessor().get(index);
+      offsetVector.getMutator().set(index + 1, currentOffset + bytes.length);
+      data.setBytes(currentOffset, bytes, 0, bytes.length);
+    }
+
+    public void setSafe(int index, byte[] bytes) {
+      assert index >= 0;
+
+      final int currentOffset = offsetVector.getAccessor().get(index);
+      while (data.capacity() < currentOffset + bytes.length) {
+        reAlloc();
+      }
+      offsetVector.getMutator().setSafe(index + 1, currentOffset + bytes.length);
+      data.setBytes(currentOffset, bytes, 0, bytes.length);
+    }
+
+    /**
+     * Set the variable length element at the specified index to the supplied byte array.
+     *
+     * @param index   position of the bit to set
+     * @param bytes   array of bytes to write
+     * @param start   start index of bytes to write
+     * @param length  length of bytes to write
+     */
+    protected void set(int index, byte[] bytes, int start, int length) {
+      assert index >= 0;
+      final int currentOffset = offsetVector.getAccessor().get(index);
+      offsetVector.getMutator().set(index + 1, currentOffset + length);
+      data.setBytes(currentOffset, bytes, start, length);
+    }
+
+    public void setSafe(int index, ByteBuffer bytes, int start, int length) {
+      assert index >= 0;
+
+      int currentOffset = offsetVector.getAccessor().get(index);
+
+      while (data.capacity() < currentOffset + length) {
+        reAlloc();
+      }
+      offsetVector.getMutator().setSafe(index + 1, currentOffset + length);
+      data.setBytes(currentOffset, bytes, start, length);
+    }
+
+    public void setSafe(int index, byte[] bytes, int start, int length) {
+      assert index >= 0;
+
+      final int currentOffset = offsetVector.getAccessor().get(index);
+
+      while (data.capacity() < currentOffset + length) {
+        reAlloc();
+      }
+      offsetVector.getMutator().setSafe(index + 1, currentOffset + length);
+      data.setBytes(currentOffset, bytes, start, length);
+    }
+
+    @Override
+    public void setValueLengthSafe(int index, int length) {
+      final int offset = offsetVector.getAccessor().get(index);
+      while(data.capacity() < offset + length ) {
+        reAlloc();
+      }
+      offsetVector.getMutator().setSafe(index + 1, offsetVector.getAccessor().get(index) + length);
+    }
+
+
+    public void setSafe(int index, int start, int end, ArrowBuf buffer){
+      final int len = end - start;
+      final int outputStart = offsetVector.data.get${(minor.javaType!type.javaType)?cap_first}(index * ${type.width});
+
+      while(data.capacity() < outputStart + len) {
+        reAlloc();
+      }
+
+      offsetVector.getMutator().setSafe( index+1,  outputStart + len);
+      buffer.getBytes(start, data, outputStart, len);
+    }
+
+    public void setSafe(int index, Nullable${minor.class}Holder holder){
+      assert holder.isSet == 1;
+
+      final int start = holder.start;
+      final int end =   holder.end;
+      final int len = end - start;
+
+      int outputStart = offsetVector.data.get${(minor.javaType!type.javaType)?cap_first}(index * ${type.width});
+
+      while(data.capacity() < outputStart + len) {
+        reAlloc();
+      }
+
+      holder.buffer.getBytes(start, data, outputStart, len);
+      offsetVector.getMutator().setSafe( index+1,  outputStart + len);
+    }
+
+    public void setSafe(int index, ${minor.class}Holder holder){
+      final int start = holder.start;
+      final int end =   holder.end;
+      final int len = end - start;
+      final int outputStart = offsetVector.data.get${(minor.javaType!type.javaType)?cap_first}(index * ${type.width});
+
+      while(data.capacity() < outputStart + len) {
+        reAlloc();
+      }
+
+      holder.buffer.getBytes(start, data, outputStart, len);
+      offsetVector.getMutator().setSafe( index+1,  outputStart + len);
+    }
+
+    protected void set(int index, int start, int length, ArrowBuf buffer){
+      assert index >= 0;
+      final int currentOffset = offsetVector.getAccessor().get(index);
+      offsetVector.getMutator().set(index + 1, currentOffset + length);
+      final ArrowBuf bb = buffer.slice(start, length);
+      data.setBytes(currentOffset, bb);
+    }
+
+    protected void set(int index, Nullable${minor.class}Holder holder){
+      final int length = holder.end - holder.start;
+      final int currentOffset = offsetVector.getAccessor().get(index);
+      offsetVector.getMutator().set(index + 1, currentOffset + length);
+      data.setBytes(currentOffset, holder.buffer, holder.start, length);
+    }
+
+    protected void set(int index, ${minor.class}Holder holder){
+      final int length = holder.end - holder.start;
+      final int currentOffset = offsetVector.getAccessor().get(index);
+      offsetVector.getMutator().set(index + 1, currentOffset + length);
+      data.setBytes(currentOffset, holder.buffer, holder.start, length);
+    }
+
+    @Override
+    public void setValueCount(int valueCount) {
+      final int currentByteCapacity = getByteCapacity();
+      final int idx = offsetVector.getAccessor().get(valueCount);
+      data.writerIndex(idx);
+      if (valueCount > 0 && currentByteCapacity > idx * 2) {
+        incrementAllocationMonitor();
+      } else if (allocationMonitor > 0) {
+        allocationMonitor = 0;
+      }
+      VectorTrimmer.trim(data, idx);
+      offsetVector.getMutator().setValueCount(valueCount == 0 ? 0 : valueCount+1);
+    }
+
+    @Override
+    public void generateTestData(int size){
+      boolean even = true;
+      <#switch minor.class>
+      <#case "Var16Char">
+      final java.nio.charset.Charset charset = Charsets.UTF_16;
+      <#break>
+      <#case "VarChar">
+      <#default>
+      final java.nio.charset.Charset charset = Charsets.UTF_8;
+      </#switch>
+      final byte[] evenValue = new String("aaaaa").getBytes(charset);
+      final byte[] oddValue = new String("bbbbbbbbbb").getBytes(charset);
+      for(int i =0; i < size; i++, even = !even){
+        set(i, even ? evenValue : oddValue);
+        }
+      setValueCount(size);
+    }
+  }
+}
+
+</#if> <#-- type.major -->
+</#list>
+</#list>

http://git-wip-us.apache.org/repos/asf/arrow/blob/fa5f0299/java/vector/src/main/java/org/apache/arrow/vector/AddOrGetResult.java
----------------------------------------------------------------------
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/AddOrGetResult.java b/java/vector/src/main/java/org/apache/arrow/vector/AddOrGetResult.java
new file mode 100644
index 0000000..388eb9c
--- /dev/null
+++ b/java/vector/src/main/java/org/apache/arrow/vector/AddOrGetResult.java
@@ -0,0 +1,38 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.arrow.vector;
+
+import com.google.common.base.Preconditions;
+
+public class AddOrGetResult<V extends ValueVector> {
+  private final V vector;
+  private final boolean created;
+
+  public AddOrGetResult(V vector, boolean created) {
+    this.vector = Preconditions.checkNotNull(vector);
+    this.created = created;
+  }
+
+  public V getVector() {
+    return vector;
+  }
+
+  public boolean isCreated() {
+    return created;
+  }
+}

http://git-wip-us.apache.org/repos/asf/arrow/blob/fa5f0299/java/vector/src/main/java/org/apache/arrow/vector/AllocationHelper.java
----------------------------------------------------------------------
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/AllocationHelper.java b/java/vector/src/main/java/org/apache/arrow/vector/AllocationHelper.java
new file mode 100644
index 0000000..54c3cd7
--- /dev/null
+++ b/java/vector/src/main/java/org/apache/arrow/vector/AllocationHelper.java
@@ -0,0 +1,61 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.arrow.vector;
+
+import org.apache.arrow.vector.complex.RepeatedFixedWidthVectorLike;
+import org.apache.arrow.vector.complex.RepeatedVariableWidthVectorLike;
+
+public class AllocationHelper {
+//  private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(AllocationHelper.class);
+
+  public static void allocate(ValueVector v, int valueCount, int bytesPerValue) {
+    allocate(v, valueCount, bytesPerValue, 5);
+  }
+
+  public static void allocatePrecomputedChildCount(ValueVector v, int valueCount, int bytesPerValue, int childValCount){
+    if(v instanceof FixedWidthVector) {
+      ((FixedWidthVector) v).allocateNew(valueCount);
+    } else if (v instanceof VariableWidthVector) {
+      ((VariableWidthVector) v).allocateNew(valueCount * bytesPerValue, valueCount);
+    } else if(v instanceof RepeatedFixedWidthVectorLike) {
+      ((RepeatedFixedWidthVectorLike) v).allocateNew(valueCount, childValCount);
+    } else if(v instanceof RepeatedVariableWidthVectorLike) {
+      ((RepeatedVariableWidthVectorLike) v).allocateNew(childValCount * bytesPerValue, valueCount, childValCount);
+    } else {
+      v.allocateNew();
+    }
+  }
+
+  public static void allocate(ValueVector v, int valueCount, int bytesPerValue, int repeatedPerTop){
+    allocatePrecomputedChildCount(v, valueCount, bytesPerValue, repeatedPerTop * valueCount);
+  }
+
+  /**
+   * Allocates the exact amount if v is fixed width, otherwise falls back to dynamic allocation
+   * @param v value vector we are trying to allocate
+   * @param valueCount  size we are trying to allocate
+   * @throws org.apache.drill.exec.memory.OutOfMemoryException if it can't allocate the memory
+   */
+  public static void allocateNew(ValueVector v, int valueCount) {
+    if (v instanceof  FixedWidthVector) {
+      ((FixedWidthVector) v).allocateNew(valueCount);
+    } else {
+      v.allocateNew();
+    }
+  }
+}

http://git-wip-us.apache.org/repos/asf/arrow/blob/fa5f0299/java/vector/src/main/java/org/apache/arrow/vector/BaseDataValueVector.java
----------------------------------------------------------------------
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/BaseDataValueVector.java b/java/vector/src/main/java/org/apache/arrow/vector/BaseDataValueVector.java
new file mode 100644
index 0000000..b129ea9
--- /dev/null
+++ b/java/vector/src/main/java/org/apache/arrow/vector/BaseDataValueVector.java
@@ -0,0 +1,91 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.arrow.vector;
+
+import io.netty.buffer.ArrowBuf;
+
+import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.vector.types.MaterializedField;
+
+
+public abstract class BaseDataValueVector extends BaseValueVector {
+
+  protected final static byte[] emptyByteArray = new byte[]{}; // Nullable vectors use this
+
+  protected ArrowBuf data;
+
+  public BaseDataValueVector(MaterializedField field, BufferAllocator allocator) {
+    super(field, allocator);
+    data = allocator.getEmpty();
+  }
+
+  @Override
+  public void clear() {
+    if (data != null) {
+      data.release();
+    }
+    data = allocator.getEmpty();
+    super.clear();
+  }
+
+  @Override
+  public void close() {
+    clear();
+    if (data != null) {
+      data.release();
+      data = null;
+    }
+    super.close();
+  }
+
+  @Override
+  public ArrowBuf[] getBuffers(boolean clear) {
+    ArrowBuf[] out;
+    if (getBufferSize() == 0) {
+      out = new ArrowBuf[0];
+    } else {
+      out = new ArrowBuf[]{data};
+      data.readerIndex(0);
+      if (clear) {
+        data.retain(1);
+      }
+    }
+    if (clear) {
+      clear();
+    }
+    return out;
+  }
+
+  @Override
+  public int getBufferSize() {
+    if (getAccessor().getValueCount() == 0) {
+      return 0;
+    }
+    return data.writerIndex();
+  }
+
+  public ArrowBuf getBuffer() {
+    return data;
+  }
+
+  /**
+   * This method has a similar effect of allocateNew() without actually clearing and reallocating
+   * the value vector. The purpose is to move the value vector to a "mutate" state
+   */
+  public void reset() {}
+}

http://git-wip-us.apache.org/repos/asf/arrow/blob/fa5f0299/java/vector/src/main/java/org/apache/arrow/vector/BaseValueVector.java
----------------------------------------------------------------------
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/BaseValueVector.java b/java/vector/src/main/java/org/apache/arrow/vector/BaseValueVector.java
new file mode 100644
index 0000000..8bca3c0
--- /dev/null
+++ b/java/vector/src/main/java/org/apache/arrow/vector/BaseValueVector.java
@@ -0,0 +1,125 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.arrow.vector;
+
+import io.netty.buffer.ArrowBuf;
+
+import java.util.Iterator;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Iterators;
+
+import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.vector.types.MaterializedField;
+import org.apache.arrow.vector.util.TransferPair;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public abstract class BaseValueVector implements ValueVector {
+  private static final Logger logger = LoggerFactory.getLogger(BaseValueVector.class);
+
+  public static final int MAX_ALLOCATION_SIZE = Integer.MAX_VALUE;
+  public static final int INITIAL_VALUE_ALLOCATION = 4096;
+
+  protected final BufferAllocator allocator;
+  protected final MaterializedField field;
+
+  protected BaseValueVector(MaterializedField field, BufferAllocator allocator) {
+    this.field = Preconditions.checkNotNull(field, "field cannot be null");
+    this.allocator = Preconditions.checkNotNull(allocator, "allocator cannot be null");
+  }
+
+  @Override
+  public String toString() {
+    return super.toString() + "[field = " + field + ", ...]";
+  }
+
+  @Override
+  public void clear() {
+    getMutator().reset();
+  }
+
+  @Override
+  public void close() {
+    clear();
+  }
+
+  @Override
+  public MaterializedField getField() {
+    return field;
+  }
+
+  public MaterializedField getField(String ref){
+    return getField().withPath(ref);
+  }
+
+  @Override
+  public TransferPair getTransferPair(BufferAllocator allocator) {
+    return getTransferPair(getField().getPath(), allocator);
+  }
+
+//  public static SerializedField getMetadata(BaseValueVector vector) {
+//    return getMetadataBuilder(vector).build();
+//  }
+//
+//  protected static SerializedField.Builder getMetadataBuilder(BaseValueVector vector) {
+//    return SerializedFieldHelper.getAsBuilder(vector.getField())
+//        .setValueCount(vector.getAccessor().getValueCount())
+//        .setBufferLength(vector.getBufferSize());
+//  }
+
+  public abstract static class BaseAccessor implements ValueVector.Accessor {
+    protected BaseAccessor() { }
+
+    @Override
+    public boolean isNull(int index) {
+      return false;
+    }
+  }
+
+  public abstract static class BaseMutator implements ValueVector.Mutator {
+    protected BaseMutator() { }
+
+    @Override
+    public void generateTestData(int values) {}
+
+    //TODO: consider making mutator stateless(if possible) on another issue.
+    public void reset() {}
+  }
+
+  @Override
+  public Iterator<ValueVector> iterator() {
+    return Iterators.emptyIterator();
+  }
+
+  public static boolean checkBufRefs(final ValueVector vv) {
+    for(final ArrowBuf buffer : vv.getBuffers(false)) {
+      if (buffer.refCnt() <= 0) {
+        throw new IllegalStateException("zero refcount");
+      }
+    }
+
+    return true;
+  }
+
+  @Override
+  public BufferAllocator getAllocator() {
+    return allocator;
+  }
+}
+

http://git-wip-us.apache.org/repos/asf/arrow/blob/fa5f0299/java/vector/src/main/java/org/apache/arrow/vector/BitVector.java
----------------------------------------------------------------------
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/BitVector.java b/java/vector/src/main/java/org/apache/arrow/vector/BitVector.java
new file mode 100644
index 0000000..952e902
--- /dev/null
+++ b/java/vector/src/main/java/org/apache/arrow/vector/BitVector.java
@@ -0,0 +1,450 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.arrow.vector;
+
+import io.netty.buffer.ArrowBuf;
+
+import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.memory.OutOfMemoryException;
+import org.apache.arrow.vector.complex.impl.BitReaderImpl;
+import org.apache.arrow.vector.complex.reader.FieldReader;
+import org.apache.arrow.vector.holders.BitHolder;
+import org.apache.arrow.vector.holders.NullableBitHolder;
+import org.apache.arrow.vector.types.MaterializedField;
+import org.apache.arrow.vector.util.OversizedAllocationException;
+import org.apache.arrow.vector.util.TransferPair;
+
+/**
+ * Bit implements a vector of bit-width values. Elements in the vector are accessed by position from the logical start
+ * of the vector. The width of each element is 1 bit. The equivalent Java primitive is an int containing the value '0'
+ * or '1'.
+ */
+public final class BitVector extends BaseDataValueVector implements FixedWidthVector {
+  static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(BitVector.class);
+
+  private final FieldReader reader = new BitReaderImpl(BitVector.this);
+  private final Accessor accessor = new Accessor();
+  private final Mutator mutator = new Mutator();
+
+  private int valueCount;
+  private int allocationSizeInBytes = INITIAL_VALUE_ALLOCATION;
+  private int allocationMonitor = 0;
+
+  public BitVector(MaterializedField field, BufferAllocator allocator) {
+    super(field, allocator);
+  }
+
+  @Override
+  public FieldReader getReader() {
+    return reader;
+  }
+
+  @Override
+  public int getBufferSize() {
+    return getSizeFromCount(valueCount);
+  }
+
+  @Override
+  public int getBufferSizeFor(final int valueCount) {
+    return getSizeFromCount(valueCount);
+  }
+
+  private int getSizeFromCount(int valueCount) {
+    return (int) Math.ceil(valueCount / 8.0);
+  }
+
+  @Override
+  public int getValueCapacity() {
+    return (int)Math.min((long)Integer.MAX_VALUE, data.capacity() * 8L);
+  }
+
+  private int getByteIndex(int index) {
+    return (int) Math.floor(index / 8.0);
+  }
+
+  @Override
+  public void setInitialCapacity(final int valueCount) {
+    allocationSizeInBytes = getSizeFromCount(valueCount);
+  }
+
+  @Override
+  public void allocateNew() {
+    if (!allocateNewSafe()) {
+      throw new OutOfMemoryException();
+    }
+  }
+
+  @Override
+  public boolean allocateNewSafe() {
+    long curAllocationSize = allocationSizeInBytes;
+    if (allocationMonitor > 10) {
+      curAllocationSize = Math.max(8, allocationSizeInBytes / 2);
+      allocationMonitor = 0;
+    } else if (allocationMonitor < -2) {
+      curAllocationSize = allocationSizeInBytes * 2L;
+      allocationMonitor = 0;
+    }
+
+    try {
+      allocateBytes(curAllocationSize);
+    } catch (OutOfMemoryException ex) {
+      return false;
+    }
+    return true;
+  }
+
+  @Override
+  public void reset() {
+    valueCount = 0;
+    allocationSizeInBytes = INITIAL_VALUE_ALLOCATION;
+    allocationMonitor = 0;
+    zeroVector();
+    super.reset();
+  }
+
+  /**
+   * Allocate a new memory space for this vector. Must be called prior to using the ValueVector.
+   *
+   * @param valueCount
+   *          The number of values which can be contained within this vector.
+   */
+  @Override
+  public void allocateNew(int valueCount) {
+    final int size = getSizeFromCount(valueCount);
+    allocateBytes(size);
+  }
+
+  private void allocateBytes(final long size) {
+    if (size > MAX_ALLOCATION_SIZE) {
+      throw new OversizedAllocationException("Requested amount of memory is more than max allowed allocation size");
+    }
+
+    final int curSize = (int) size;
+    clear();
+    data = allocator.buffer(curSize);
+    zeroVector();
+    allocationSizeInBytes = curSize;
+  }
+
+  /**
+   * Allocate new buffer with double capacity, and copy data into the new buffer. Replace vector's buffer with new buffer, and release old one
+   */
+  public void reAlloc() {
+    final long newAllocationSize = allocationSizeInBytes * 2L;
+    if (newAllocationSize > MAX_ALLOCATION_SIZE) {
+      throw new OversizedAllocationException("Requested amount of memory is more than max allowed allocation size");
+    }
+
+    final int curSize = (int)newAllocationSize;
+    final ArrowBuf newBuf = allocator.buffer(curSize);
+    newBuf.setZero(0, newBuf.capacity());
+    newBuf.setBytes(0, data, 0, data.capacity());
+    data.release();
+    data = newBuf;
+    allocationSizeInBytes = curSize;
+  }
+
+  /**
+   * {@inheritDoc}
+   */
+  @Override
+  public void zeroVector() {
+    data.setZero(0, data.capacity());
+  }
+
+  public void copyFrom(int inIndex, int outIndex, BitVector from) {
+    this.mutator.set(outIndex, from.accessor.get(inIndex));
+  }
+
+  public boolean copyFromSafe(int inIndex, int outIndex, BitVector from) {
+    if (outIndex >= this.getValueCapacity()) {
+      decrementAllocationMonitor();
+      return false;
+    }
+    copyFrom(inIndex, outIndex, from);
+    return true;
+  }
+
+//  @Override
+//  public void load(SerializedField metadata, DrillBuf buffer) {
+//    Preconditions.checkArgument(this.field.getPath().equals(metadata.getNamePart().getName()), "The field %s doesn't match the provided metadata %s.", this.field, metadata);
+//    final int valueCount = metadata.getValueCount();
+//    final int expectedLength = getSizeFromCount(valueCount);
+//    final int actualLength = metadata.getBufferLength();
+//    assert expectedLength == actualLength: "expected and actual buffer sizes do not match";
+//
+//    clear();
+//    data = buffer.slice(0, actualLength);
+//    data.retain();
+//    this.valueCount = valueCount;
+//  }
+
+  @Override
+  public Mutator getMutator() {
+    return new Mutator();
+  }
+
+  @Override
+  public Accessor getAccessor() {
+    return new Accessor();
+  }
+
+  @Override
+  public TransferPair getTransferPair(BufferAllocator allocator) {
+    return new TransferImpl(getField(), allocator);
+  }
+
+  @Override
+  public TransferPair getTransferPair(String ref, BufferAllocator allocator) {
+    return new TransferImpl(getField().withPath(ref), allocator);
+  }
+
+  @Override
+  public TransferPair makeTransferPair(ValueVector to) {
+    return new TransferImpl((BitVector) to);
+  }
+
+
+  public void transferTo(BitVector target) {
+    target.clear();
+    if (target.data != null) {
+      target.data.release();
+    }
+    target.data = data;
+    target.data.retain(1);
+    target.valueCount = valueCount;
+    clear();
+  }
+
+  public void splitAndTransferTo(int startIndex, int length, BitVector target) {
+    assert startIndex + length <= valueCount;
+    int firstByte = getByteIndex(startIndex);
+    int byteSize = getSizeFromCount(length);
+    int offset = startIndex % 8;
+    if (offset == 0) {
+      target.clear();
+      // slice
+      if (target.data != null) {
+        target.data.release();
+      }
+      target.data = (ArrowBuf) data.slice(firstByte, byteSize);
+      target.data.retain(1);
+    } else {
+      // Copy data
+      // When the first bit starts from the middle of a byte (offset != 0), copy data from src BitVector.
+      // Each byte in the target is composed by a part in i-th byte, another part in (i+1)-th byte.
+      // The last byte copied to target is a bit tricky :
+      //   1) if length requires partly byte (length % 8 !=0), copy the remaining bits only.
+      //   2) otherwise, copy the last byte in the same way as to the prior bytes.
+      target.clear();
+      target.allocateNew(length);
+      // TODO maybe do this one word at a time, rather than byte?
+      for(int i = 0; i < byteSize - 1; i++) {
+        target.data.setByte(i, (((this.data.getByte(firstByte + i) & 0xFF) >>> offset) + (this.data.getByte(firstByte + i + 1) <<  (8 - offset))));
+      }
+      if (length % 8 != 0) {
+        target.data.setByte(byteSize - 1, ((this.data.getByte(firstByte + byteSize - 1) & 0xFF) >>> offset));
+      } else {
+        target.data.setByte(byteSize - 1,
+            (((this.data.getByte(firstByte + byteSize - 1) & 0xFF) >>> offset) + (this.data.getByte(firstByte + byteSize) <<  (8 - offset))));
+      }
+    }
+    target.getMutator().setValueCount(length);
+  }
+
+  private class TransferImpl implements TransferPair {
+    BitVector to;
+
+    public TransferImpl(MaterializedField field, BufferAllocator allocator) {
+      this.to = new BitVector(field, allocator);
+    }
+
+    public TransferImpl(BitVector to) {
+      this.to = to;
+    }
+
+    @Override
+    public BitVector getTo() {
+      return to;
+    }
+
+    @Override
+    public void transfer() {
+      transferTo(to);
+    }
+
+    @Override
+    public void splitAndTransfer(int startIndex, int length) {
+      splitAndTransferTo(startIndex, length, to);
+    }
+
+    @Override
+    public void copyValueSafe(int fromIndex, int toIndex) {
+      to.copyFromSafe(fromIndex, toIndex, BitVector.this);
+    }
+  }
+
+  private void decrementAllocationMonitor() {
+    if (allocationMonitor > 0) {
+      allocationMonitor = 0;
+    }
+    --allocationMonitor;
+  }
+
+  private void incrementAllocationMonitor() {
+    ++allocationMonitor;
+  }
+
+  public class Accessor extends BaseAccessor {
+
+    /**
+     * Get the byte holding the desired bit, then mask all other bits. Iff the result is 0, the bit was not set.
+     *
+     * @param index
+     *          position of the bit in the vector
+     * @return 1 if set, otherwise 0
+     */
+    public final int get(int index) {
+      int byteIndex = index >> 3;
+      byte b = data.getByte(byteIndex);
+      int bitIndex = index & 7;
+      return Long.bitCount(b &  (1L << bitIndex));
+    }
+
+    @Override
+    public boolean isNull(int index) {
+      return false;
+    }
+
+    @Override
+    public final Boolean getObject(int index) {
+      return new Boolean(get(index) != 0);
+    }
+
+    @Override
+    public final int getValueCount() {
+      return valueCount;
+    }
+
+    public final void get(int index, BitHolder holder) {
+      holder.value = get(index);
+    }
+
+    public final void get(int index, NullableBitHolder holder) {
+      holder.isSet = 1;
+      holder.value = get(index);
+    }
+  }
+
+  /**
+   * MutableBit implements a vector of bit-width values. Elements in the vector are accessed by position from the
+   * logical start of the vector. Values should be pushed onto the vector sequentially, but may be randomly accessed.
+   *
+   * NB: this class is automatically generated from ValueVectorTypes.tdd using FreeMarker.
+   */
+  public class Mutator extends BaseMutator {
+
+    private Mutator() {
+    }
+
+    /**
+     * Set the bit at the given index to the specified value.
+     *
+     * @param index
+     *          position of the bit to set
+     * @param value
+     *          value to set (either 1 or 0)
+     */
+    public final void set(int index, int value) {
+      int byteIndex = index >> 3;
+      int bitIndex = index & 7;
+      byte currentByte = data.getByte(byteIndex);
+      byte bitMask = (byte) (1L << bitIndex);
+      if (value != 0) {
+        currentByte |= bitMask;
+      } else {
+        currentByte -= (bitMask & currentByte);
+      }
+
+      data.setByte(byteIndex, currentByte);
+    }
+
+    public final void set(int index, BitHolder holder) {
+      set(index, holder.value);
+    }
+
+    final void set(int index, NullableBitHolder holder) {
+      set(index, holder.value);
+    }
+
+    public void setSafe(int index, int value) {
+      while(index >= getValueCapacity()) {
+        reAlloc();
+      }
+      set(index, value);
+    }
+
+    public void setSafe(int index, BitHolder holder) {
+      while(index >= getValueCapacity()) {
+        reAlloc();
+      }
+      set(index, holder.value);
+    }
+
+    public void setSafe(int index, NullableBitHolder holder) {
+      while(index >= getValueCapacity()) {
+        reAlloc();
+      }
+      set(index, holder.value);
+    }
+
+    @Override
+    public final void setValueCount(int valueCount) {
+      int currentValueCapacity = getValueCapacity();
+      BitVector.this.valueCount = valueCount;
+      int idx = getSizeFromCount(valueCount);
+      while(valueCount > getValueCapacity()) {
+        reAlloc();
+      }
+      if (valueCount > 0 && currentValueCapacity > valueCount * 2) {
+        incrementAllocationMonitor();
+      } else if (allocationMonitor > 0) {
+        allocationMonitor = 0;
+      }
+      VectorTrimmer.trim(data, idx);
+    }
+
+    @Override
+    public final void generateTestData(int values) {
+      boolean even = true;
+      for(int i = 0; i < values; i++, even = !even) {
+        if (even) {
+          set(i, 1);
+        }
+      }
+      setValueCount(values);
+    }
+
+  }
+
+  @Override
+  public void clear() {
+    this.valueCount = 0;
+    super.clear();
+  }
+}

http://git-wip-us.apache.org/repos/asf/arrow/blob/fa5f0299/java/vector/src/main/java/org/apache/arrow/vector/FixedWidthVector.java
----------------------------------------------------------------------
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/FixedWidthVector.java b/java/vector/src/main/java/org/apache/arrow/vector/FixedWidthVector.java
new file mode 100644
index 0000000..5905700
--- /dev/null
+++ b/java/vector/src/main/java/org/apache/arrow/vector/FixedWidthVector.java
@@ -0,0 +1,35 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.arrow.vector;
+
+
+public interface FixedWidthVector extends ValueVector{
+
+  /**
+   * Allocate a new memory space for this vector.  Must be called prior to using the ValueVector.
+   *
+   * @param valueCount   Number of values in the vector.
+   */
+  void allocateNew(int valueCount);
+
+/**
+ * Zero out the underlying buffer backing this vector.
+ */
+  void zeroVector();
+
+}

http://git-wip-us.apache.org/repos/asf/arrow/blob/fa5f0299/java/vector/src/main/java/org/apache/arrow/vector/NullableVector.java
----------------------------------------------------------------------
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableVector.java
new file mode 100644
index 0000000..00c33fc
--- /dev/null
+++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableVector.java
@@ -0,0 +1,23 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.arrow.vector;
+
+public interface NullableVector extends ValueVector{
+
+  ValueVector getValuesVector();
+}

http://git-wip-us.apache.org/repos/asf/arrow/blob/fa5f0299/java/vector/src/main/java/org/apache/arrow/vector/NullableVectorDefinitionSetter.java
----------------------------------------------------------------------
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableVectorDefinitionSetter.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableVectorDefinitionSetter.java
new file mode 100644
index 0000000..b819c5d
--- /dev/null
+++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableVectorDefinitionSetter.java
@@ -0,0 +1,23 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.arrow.vector;
+
+public interface NullableVectorDefinitionSetter {
+
+  public void setIndexDefined(int index);
+}

http://git-wip-us.apache.org/repos/asf/arrow/blob/fa5f0299/java/vector/src/main/java/org/apache/arrow/vector/ObjectVector.java
----------------------------------------------------------------------
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/ObjectVector.java b/java/vector/src/main/java/org/apache/arrow/vector/ObjectVector.java
new file mode 100644
index 0000000..b806b18
--- /dev/null
+++ b/java/vector/src/main/java/org/apache/arrow/vector/ObjectVector.java
@@ -0,0 +1,220 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.arrow.vector;
+
+import io.netty.buffer.ArrowBuf;
+
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+
+import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.memory.OutOfMemoryException;
+import org.apache.arrow.vector.complex.reader.FieldReader;
+import org.apache.arrow.vector.holders.ObjectHolder;
+import org.apache.arrow.vector.types.MaterializedField;
+import org.apache.arrow.vector.util.TransferPair;
+
+public class ObjectVector extends BaseValueVector {
+  private final Accessor accessor = new Accessor();
+  private final Mutator mutator = new Mutator();
+  private int maxCount = 0;
+  private int count = 0;
+  private int allocationSize = 4096;
+
+  private List<Object[]> objectArrayList = new ArrayList<>();
+
+  public ObjectVector(MaterializedField field, BufferAllocator allocator) {
+    super(field, allocator);
+  }
+
+  public void addNewArray() {
+    objectArrayList.add(new Object[allocationSize]);
+    maxCount += allocationSize;
+  }
+
+  @Override
+  public FieldReader getReader() {
+    throw new UnsupportedOperationException("ObjectVector does not support this");
+  }
+
+  public final class Mutator implements ValueVector.Mutator {
+
+    public void set(int index, Object obj) {
+      int listOffset = index / allocationSize;
+      if (listOffset >= objectArrayList.size()) {
+        addNewArray();
+      }
+      objectArrayList.get(listOffset)[index % allocationSize] = obj;
+    }
+
+    public boolean setSafe(int index, long value) {
+      set(index, value);
+      return true;
+    }
+
+    protected void set(int index, ObjectHolder holder) {
+      set(index, holder.obj);
+    }
+
+    public boolean setSafe(int index, ObjectHolder holder){
+      set(index, holder);
+      return true;
+    }
+
+    @Override
+    public void setValueCount(int valueCount) {
+      count = valueCount;
+    }
+
+    @Override
+    public void reset() {
+      count = 0;
+      maxCount = 0;
+      objectArrayList = new ArrayList<>();
+      addNewArray();
+    }
+
+    @Override
+    public void generateTestData(int values) {
+    }
+  }
+
+  @Override
+  public void setInitialCapacity(int numRecords) {
+    // NoOp
+  }
+
+  @Override
+  public void allocateNew() throws OutOfMemoryException {
+    addNewArray();
+  }
+
+  public void allocateNew(int valueCount) throws OutOfMemoryException {
+    while (maxCount < valueCount) {
+      addNewArray();
+    }
+  }
+
+  @Override
+  public boolean allocateNewSafe() {
+    allocateNew();
+    return true;
+  }
+
+  @Override
+  public int getBufferSize() {
+    throw new UnsupportedOperationException("ObjectVector does not support this");
+  }
+
+  @Override
+  public int getBufferSizeFor(final int valueCount) {
+    throw new UnsupportedOperationException("ObjectVector does not support this");
+  }
+
+  @Override
+  public void close() {
+    clear();
+  }
+
+  @Override
+  public void clear() {
+    objectArrayList.clear();
+    maxCount = 0;
+    count = 0;
+  }
+
+  @Override
+  public MaterializedField getField() {
+    return field;
+  }
+
+  @Override
+  public TransferPair getTransferPair(BufferAllocator allocator) {
+    throw new UnsupportedOperationException("ObjectVector does not support this");
+  }
+
+  @Override
+  public TransferPair makeTransferPair(ValueVector to) {
+    throw new UnsupportedOperationException("ObjectVector does not support this");
+  }
+
+  @Override
+  public TransferPair getTransferPair(String ref, BufferAllocator allocator) {
+    throw new UnsupportedOperationException("ObjectVector does not support this");
+  }
+
+  @Override
+  public int getValueCapacity() {
+    return maxCount;
+  }
+
+  @Override
+  public Accessor getAccessor() {
+    return accessor;
+  }
+
+  @Override
+  public ArrowBuf[] getBuffers(boolean clear) {
+    throw new UnsupportedOperationException("ObjectVector does not support this");
+  }
+
+//  @Override
+//  public void load(UserBitShared.SerializedField metadata, DrillBuf buffer) {
+//    throw new UnsupportedOperationException("ObjectVector does not support this");
+//  }
+//
+//  @Override
+//  public UserBitShared.SerializedField getMetadata() {
+//    throw new UnsupportedOperationException("ObjectVector does not support this");
+//  }
+
+  @Override
+  public Mutator getMutator() {
+    return mutator;
+  }
+
+  @Override
+  public Iterator<ValueVector> iterator() {
+    throw new UnsupportedOperationException("ObjectVector does not support this");
+  }
+
+  public final class Accessor extends BaseAccessor {
+    @Override
+    public Object getObject(int index) {
+      int listOffset = index / allocationSize;
+      if (listOffset >= objectArrayList.size()) {
+        addNewArray();
+      }
+      return objectArrayList.get(listOffset)[index % allocationSize];
+    }
+
+    @Override
+    public int getValueCount() {
+      return count;
+    }
+
+    public Object get(int index) {
+      return getObject(index);
+    }
+
+    public void get(int index, ObjectHolder holder){
+      holder.obj = getObject(index);
+    }
+  }
+}

http://git-wip-us.apache.org/repos/asf/arrow/blob/fa5f0299/java/vector/src/main/java/org/apache/arrow/vector/SchemaChangeCallBack.java
----------------------------------------------------------------------
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/SchemaChangeCallBack.java b/java/vector/src/main/java/org/apache/arrow/vector/SchemaChangeCallBack.java
new file mode 100644
index 0000000..fc0a066
--- /dev/null
+++ b/java/vector/src/main/java/org/apache/arrow/vector/SchemaChangeCallBack.java
@@ -0,0 +1,52 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.arrow.vector;
+
+import org.apache.arrow.vector.util.CallBack;
+
+
+public class SchemaChangeCallBack implements CallBack {
+  private boolean schemaChanged = false;
+
+  /**
+   * Constructs a schema-change callback with the schema-changed state set to
+   * {@code false}.
+   */
+  public SchemaChangeCallBack() {
+  }
+
+  /**
+   * Sets the schema-changed state to {@code true}.
+   */
+  @Override
+  public void doWork() {
+    schemaChanged = true;
+  }
+
+  /**
+   * Returns the value of schema-changed state, <strong>resetting</strong> the
+   * schema-changed state to {@code false}.
+   */
+  public boolean getSchemaChangedAndReset() {
+    final boolean current = schemaChanged;
+    schemaChanged = false;
+    return current;
+  }
+}
+


Mime
View raw message