arrow-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jacq...@apache.org
Subject [05/17] arrow git commit: ARROW-1: Initial Arrow Code Commit
Date Wed, 17 Feb 2016 12:39:40 GMT
http://git-wip-us.apache.org/repos/asf/arrow/blob/fa5f0299/java/vector/src/main/java/org/apache/arrow/vector/ValueHolderHelper.java
----------------------------------------------------------------------
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/ValueHolderHelper.java b/java/vector/src/main/java/org/apache/arrow/vector/ValueHolderHelper.java
new file mode 100644
index 0000000..61ce285
--- /dev/null
+++ b/java/vector/src/main/java/org/apache/arrow/vector/ValueHolderHelper.java
@@ -0,0 +1,203 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.arrow.vector;
+
+import io.netty.buffer.ArrowBuf;
+
+import java.math.BigDecimal;
+
+import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.vector.holders.BigIntHolder;
+import org.apache.arrow.vector.holders.BitHolder;
+import org.apache.arrow.vector.holders.DateHolder;
+import org.apache.arrow.vector.holders.Decimal18Holder;
+import org.apache.arrow.vector.holders.Decimal28SparseHolder;
+import org.apache.arrow.vector.holders.Decimal38SparseHolder;
+import org.apache.arrow.vector.holders.Decimal9Holder;
+import org.apache.arrow.vector.holders.Float4Holder;
+import org.apache.arrow.vector.holders.Float8Holder;
+import org.apache.arrow.vector.holders.IntHolder;
+import org.apache.arrow.vector.holders.IntervalDayHolder;
+import org.apache.arrow.vector.holders.IntervalYearHolder;
+import org.apache.arrow.vector.holders.NullableBitHolder;
+import org.apache.arrow.vector.holders.TimeHolder;
+import org.apache.arrow.vector.holders.TimeStampHolder;
+import org.apache.arrow.vector.holders.VarCharHolder;
+import org.apache.arrow.vector.util.DecimalUtility;
+
+import com.google.common.base.Charsets;
+
+
+public class ValueHolderHelper {
+  static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ValueHolderHelper.class);
+
+  public static IntHolder getIntHolder(int value) {
+    IntHolder holder = new IntHolder();
+    holder.value = value;
+
+    return holder;
+  }
+
+  public static BigIntHolder getBigIntHolder(long value) {
+    BigIntHolder holder = new BigIntHolder();
+    holder.value = value;
+
+    return holder;
+  }
+
+  public static Float4Holder getFloat4Holder(float value) {
+    Float4Holder holder = new Float4Holder();
+    holder.value = value;
+
+    return holder;
+  }
+
+  public static Float8Holder getFloat8Holder(double value) {
+    Float8Holder holder = new Float8Holder();
+    holder.value = value;
+
+    return holder;
+  }
+
+  public static DateHolder getDateHolder(long value) {
+    DateHolder holder = new DateHolder();
+    holder.value = value;
+    return holder;
+  }
+
+  public static TimeHolder getTimeHolder(int value) {
+    TimeHolder holder = new TimeHolder();
+    holder.value = value;
+    return holder;
+  }
+
+  public static TimeStampHolder getTimeStampHolder(long value) {
+    TimeStampHolder holder = new TimeStampHolder();
+    holder.value = value;
+    return holder;
+  }
+
+  public static BitHolder getBitHolder(int value) {
+    BitHolder holder = new BitHolder();
+    holder.value = value;
+
+    return holder;
+  }
+
+  public static NullableBitHolder getNullableBitHolder(boolean isNull, int value) {
+    NullableBitHolder holder = new NullableBitHolder();
+    holder.isSet = isNull? 0 : 1;
+    if (! isNull) {
+      holder.value = value;
+    }
+
+    return holder;
+  }
+
+  public static VarCharHolder getVarCharHolder(ArrowBuf buf, String s){
+    VarCharHolder vch = new VarCharHolder();
+
+    byte[] b = s.getBytes(Charsets.UTF_8);
+    vch.start = 0;
+    vch.end = b.length;
+    vch.buffer = buf.reallocIfNeeded(b.length);
+    vch.buffer.setBytes(0, b);
+    return vch;
+  }
+
+  public static VarCharHolder getVarCharHolder(BufferAllocator a, String s){
+    VarCharHolder vch = new VarCharHolder();
+
+    byte[] b = s.getBytes(Charsets.UTF_8);
+    vch.start = 0;
+    vch.end = b.length;
+    vch.buffer = a.buffer(b.length); //
+    vch.buffer.setBytes(0, b);
+    return vch;
+  }
+
+
+  public static IntervalYearHolder getIntervalYearHolder(int intervalYear) {
+    IntervalYearHolder holder = new IntervalYearHolder();
+
+    holder.value = intervalYear;
+    return holder;
+  }
+
+  public static IntervalDayHolder getIntervalDayHolder(int days, int millis) {
+      IntervalDayHolder dch = new IntervalDayHolder();
+
+      dch.days = days;
+      dch.milliseconds = millis;
+      return dch;
+  }
+
+  public static Decimal9Holder getDecimal9Holder(int decimal, int scale, int precision) {
+    Decimal9Holder dch = new Decimal9Holder();
+
+    dch.scale = scale;
+    dch.precision = precision;
+    dch.value = decimal;
+
+    return dch;
+  }
+
+  public static Decimal18Holder getDecimal18Holder(long decimal, int scale, int precision) {
+    Decimal18Holder dch = new Decimal18Holder();
+
+    dch.scale = scale;
+    dch.precision = precision;
+    dch.value = decimal;
+
+    return dch;
+  }
+
+  public static Decimal28SparseHolder getDecimal28Holder(ArrowBuf buf, String decimal) {
+
+    Decimal28SparseHolder dch = new Decimal28SparseHolder();
+
+    BigDecimal bigDecimal = new BigDecimal(decimal);
+
+    dch.scale = bigDecimal.scale();
+    dch.precision = bigDecimal.precision();
+    Decimal28SparseHolder.setSign(bigDecimal.signum() == -1, dch.start, dch.buffer);
+    dch.start = 0;
+    dch.buffer = buf.reallocIfNeeded(5 * DecimalUtility.INTEGER_SIZE);
+    DecimalUtility
+        .getSparseFromBigDecimal(bigDecimal, dch.buffer, dch.start, dch.scale, dch.precision, dch.nDecimalDigits);
+
+    return dch;
+  }
+
+  public static Decimal38SparseHolder getDecimal38Holder(ArrowBuf buf, String decimal) {
+
+      Decimal38SparseHolder dch = new Decimal38SparseHolder();
+
+      BigDecimal bigDecimal = new BigDecimal(decimal);
+
+      dch.scale = bigDecimal.scale();
+      dch.precision = bigDecimal.precision();
+      Decimal38SparseHolder.setSign(bigDecimal.signum() == -1, dch.start, dch.buffer);
+      dch.start = 0;
+    dch.buffer = buf.reallocIfNeeded(dch.maxPrecision * DecimalUtility.INTEGER_SIZE);
+    DecimalUtility
+        .getSparseFromBigDecimal(bigDecimal, dch.buffer, dch.start, dch.scale, dch.precision, dch.nDecimalDigits);
+
+      return dch;
+  }
+}

http://git-wip-us.apache.org/repos/asf/arrow/blob/fa5f0299/java/vector/src/main/java/org/apache/arrow/vector/ValueVector.java
----------------------------------------------------------------------
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/ValueVector.java b/java/vector/src/main/java/org/apache/arrow/vector/ValueVector.java
new file mode 100644
index 0000000..c05f0e7
--- /dev/null
+++ b/java/vector/src/main/java/org/apache/arrow/vector/ValueVector.java
@@ -0,0 +1,222 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.arrow.vector;
+
+import java.io.Closeable;
+
+import io.netty.buffer.ArrowBuf;
+
+import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.memory.OutOfMemoryException;
+import org.apache.arrow.vector.complex.reader.FieldReader;
+import org.apache.arrow.vector.types.MaterializedField;
+import org.apache.arrow.vector.util.TransferPair;
+
+/**
+ * An abstraction that is used to store a sequence of values in an individual column.
+ *
+ * A {@link ValueVector value vector} stores underlying data in-memory in a columnar fashion that is compact and
+ * efficient. The column whose data is stored, is referred by {@link #getField()}.
+ *
+ * A vector when instantiated, relies on a {@link org.apache.drill.exec.record.DeadBuf dead buffer}. It is important
+ * that vector is allocated before attempting to read or write.
+ *
+ * There are a few "rules" around vectors:
+ *
+ * <ul>
+ *   <li>values need to be written in order (e.g. index 0, 1, 2, 5)</li>
+ *   <li>null vectors start with all values as null before writing anything</li>
+ *   <li>for variable width types, the offset vector should be all zeros before writing</li>
+ *   <li>you must call setValueCount before a vector can be read</li>
+ *   <li>you should never write to a vector once it has been read.</li>
+ * </ul>
+ *
+ * Please note that the current implementation doesn't enfore those rules, hence we may find few places that
+ * deviate from these rules (e.g. offset vectors in Variable Length and Repeated vector)
+ *
+ * This interface "should" strive to guarantee this order of operation:
+ * <blockquote>
+ * allocate > mutate > setvaluecount > access > clear (or allocate to start the process over).
+ * </blockquote>
+ */
+public interface ValueVector extends Closeable, Iterable<ValueVector> {
+  /**
+   * Allocate new buffers. ValueVector implements logic to determine how much to allocate.
+   * @throws OutOfMemoryException Thrown if no memory can be allocated.
+   */
+  void allocateNew() throws OutOfMemoryException;
+
+  /**
+   * Allocates new buffers. ValueVector implements logic to determine how much to allocate.
+   * @return Returns true if allocation was succesful.
+   */
+  boolean allocateNewSafe();
+
+  BufferAllocator getAllocator();
+
+  /**
+   * Set the initial record capacity
+   * @param numRecords
+   */
+  void setInitialCapacity(int numRecords);
+
+  /**
+   * Returns the maximum number of values that can be stored in this vector instance.
+   */
+  int getValueCapacity();
+
+  /**
+   * Alternative to clear(). Allows use as an AutoCloseable in try-with-resources.
+   */
+  @Override
+  void close();
+
+  /**
+   * Release the underlying DrillBuf and reset the ValueVector to empty.
+   */
+  void clear();
+
+  /**
+   * Get information about how this field is materialized.
+   */
+  MaterializedField getField();
+
+  /**
+   * Returns a {@link org.apache.arrow.vector.util.TransferPair transfer pair}, creating a new target vector of
+   * the same type.
+   */
+  TransferPair getTransferPair(BufferAllocator allocator);
+
+  TransferPair getTransferPair(String ref, BufferAllocator allocator);
+
+  /**
+   * Returns a new {@link org.apache.arrow.vector.util.TransferPair transfer pair} that is used to transfer underlying
+   * buffers into the target vector.
+   */
+  TransferPair makeTransferPair(ValueVector target);
+
+  /**
+   * Returns an {@link org.apache.arrow.vector.ValueVector.Accessor accessor} that is used to read from this vector
+   * instance.
+   */
+  Accessor getAccessor();
+
+  /**
+   * Returns an {@link org.apache.arrow.vector.ValueVector.Mutator mutator} that is used to write to this vector
+   * instance.
+   */
+  Mutator getMutator();
+
+  /**
+   * Returns a {@link org.apache.arrow.vector.complex.reader.FieldReader field reader} that supports reading values
+   * from this vector.
+   */
+  FieldReader getReader();
+
+  /**
+   * Get the metadata for this field. Used in serialization
+   *
+   * @return FieldMetadata for this field.
+   */
+//  SerializedField getMetadata();
+
+  /**
+   * Returns the number of bytes that is used by this vector instance.
+   */
+  int getBufferSize();
+
+  /**
+   * Returns the number of bytes that is used by this vector if it holds the given number
+   * of values. The result will be the same as if Mutator.setValueCount() were called, followed
+   * by calling getBufferSize(), but without any of the closing side-effects that setValueCount()
+   * implies wrt finishing off the population of a vector. Some operations might wish to use
+   * this to determine how much memory has been used by a vector so far, even though it is
+   * not finished being populated.
+   *
+   * @param valueCount the number of values to assume this vector contains
+   * @return the buffer size if this vector is holding valueCount values
+   */
+  int getBufferSizeFor(int valueCount);
+
+  /**
+   * Return the underlying buffers associated with this vector. Note that this doesn't impact the reference counts for
+   * this buffer so it only should be used for in-context access. Also note that this buffer changes regularly thus
+   * external classes shouldn't hold a reference to it (unless they change it).
+   * @param clear Whether to clear vector before returning; the buffers will still be refcounted;
+   *   but the returned array will be the only reference to them
+   *
+   * @return The underlying {@link io.netty.buffer.ArrowBuf buffers} that is used by this vector instance.
+   */
+  ArrowBuf[] getBuffers(boolean clear);
+
+  /**
+   * Load the data provided in the buffer. Typically used when deserializing from the wire.
+   *
+   * @param metadata
+   *          Metadata used to decode the incoming buffer.
+   * @param buffer
+   *          The buffer that contains the ValueVector.
+   */
+//  void load(SerializedField metadata, DrillBuf buffer);
+
+  /**
+   * An abstraction that is used to read from this vector instance.
+   */
+  interface Accessor {
+    /**
+     * Get the Java Object representation of the element at the specified position. Useful for testing.
+     *
+     * @param index
+     *          Index of the value to get
+     */
+    Object getObject(int index);
+
+    /**
+     * Returns the number of values that is stored in this vector.
+     */
+    int getValueCount();
+
+    /**
+     * Returns true if the value at the given index is null, false otherwise.
+     */
+    boolean isNull(int index);
+  }
+
+  /**
+   * An abstractiong that is used to write into this vector instance.
+   */
+  interface Mutator {
+    /**
+     * Sets the number of values that is stored in this vector to the given value count.
+     *
+     * @param valueCount  value count to set.
+     */
+    void setValueCount(int valueCount);
+
+    /**
+     * Resets the mutator to pristine state.
+     */
+    void reset();
+
+    /**
+     * @deprecated  this has nothing to do with value vector abstraction and should be removed.
+     */
+    @Deprecated
+    void generateTestData(int values);
+  }
+}

http://git-wip-us.apache.org/repos/asf/arrow/blob/fa5f0299/java/vector/src/main/java/org/apache/arrow/vector/VariableWidthVector.java
----------------------------------------------------------------------
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/VariableWidthVector.java b/java/vector/src/main/java/org/apache/arrow/vector/VariableWidthVector.java
new file mode 100644
index 0000000..e227bb4
--- /dev/null
+++ b/java/vector/src/main/java/org/apache/arrow/vector/VariableWidthVector.java
@@ -0,0 +1,51 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.arrow.vector;
+
+import io.netty.buffer.ArrowBuf;
+
+public interface VariableWidthVector extends ValueVector{
+
+  /**
+   * Allocate a new memory space for this vector.  Must be called prior to using the ValueVector.
+   *
+   * @param totalBytes   Desired size of the underlying data buffer.
+   * @param valueCount   Number of values in the vector.
+   */
+  void allocateNew(int totalBytes, int valueCount);
+
+  /**
+   * Provide the maximum amount of variable width bytes that can be stored int his vector.
+   * @return
+   */
+  int getByteCapacity();
+
+  VariableWidthMutator getMutator();
+
+  VariableWidthAccessor getAccessor();
+
+  interface VariableWidthAccessor extends Accessor {
+    int getValueLength(int index);
+  }
+
+  int getCurrentSizeInBytes();
+
+  interface VariableWidthMutator extends Mutator {
+    void setValueLengthSafe(int index, int length);
+  }
+}

http://git-wip-us.apache.org/repos/asf/arrow/blob/fa5f0299/java/vector/src/main/java/org/apache/arrow/vector/VectorDescriptor.java
----------------------------------------------------------------------
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/VectorDescriptor.java b/java/vector/src/main/java/org/apache/arrow/vector/VectorDescriptor.java
new file mode 100644
index 0000000..fdad99a
--- /dev/null
+++ b/java/vector/src/main/java/org/apache/arrow/vector/VectorDescriptor.java
@@ -0,0 +1,83 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.arrow.vector;
+
+import java.util.Collection;
+
+import com.google.common.base.Preconditions;
+
+import org.apache.arrow.vector.types.MaterializedField;
+import org.apache.arrow.vector.types.Types.MajorType;
+
+public class VectorDescriptor {
+  private static final String DEFAULT_NAME = "NONE";
+
+  private final MaterializedField field;
+
+  public VectorDescriptor(final MajorType type) {
+    this(DEFAULT_NAME, type);
+  }
+
+  public VectorDescriptor(final String name, final MajorType type) {
+    this(MaterializedField.create(name, type));
+  }
+
+  public VectorDescriptor(final MaterializedField field) {
+    this.field = Preconditions.checkNotNull(field, "field cannot be null");
+  }
+
+  public MaterializedField getField() {
+    return field;
+  }
+
+  public MajorType getType() {
+    return field.getType();
+  }
+
+  public String getName() {
+    return field.getLastName();
+  }
+
+  public Collection<MaterializedField> getChildren() {
+    return field.getChildren();
+  }
+
+  public boolean hasName() {
+    return getName() != DEFAULT_NAME;
+  }
+
+  public VectorDescriptor withName(final String name) {
+    return new VectorDescriptor(field.withPath(name));
+  }
+
+  public VectorDescriptor withType(final MajorType type) {
+    return new VectorDescriptor(field.withType(type));
+  }
+
+  public static VectorDescriptor create(final String name, final MajorType type) {
+    return new VectorDescriptor(name, type);
+  }
+
+  public static VectorDescriptor create(final MajorType type) {
+    return new VectorDescriptor(type);
+  }
+
+  public static VectorDescriptor create(final MaterializedField field) {
+    return new VectorDescriptor(field);
+  }
+}

http://git-wip-us.apache.org/repos/asf/arrow/blob/fa5f0299/java/vector/src/main/java/org/apache/arrow/vector/VectorTrimmer.java
----------------------------------------------------------------------
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/VectorTrimmer.java b/java/vector/src/main/java/org/apache/arrow/vector/VectorTrimmer.java
new file mode 100644
index 0000000..055857e
--- /dev/null
+++ b/java/vector/src/main/java/org/apache/arrow/vector/VectorTrimmer.java
@@ -0,0 +1,33 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.arrow.vector;
+
+import io.netty.buffer.ByteBuf;
+import io.netty.buffer.ArrowBuf;
+
+public class VectorTrimmer {
+  static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(VectorTrimmer.class);
+
+  public static void trim(ByteBuf data, int idx) {
+    data.writerIndex(idx);
+    if (data instanceof ArrowBuf) {
+      // data.capacity(idx);
+      data.writerIndex(idx);
+    }
+  }
+}

http://git-wip-us.apache.org/repos/asf/arrow/blob/fa5f0299/java/vector/src/main/java/org/apache/arrow/vector/ZeroVector.java
----------------------------------------------------------------------
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/ZeroVector.java b/java/vector/src/main/java/org/apache/arrow/vector/ZeroVector.java
new file mode 100644
index 0000000..78de870
--- /dev/null
+++ b/java/vector/src/main/java/org/apache/arrow/vector/ZeroVector.java
@@ -0,0 +1,181 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.arrow.vector;
+
+import io.netty.buffer.ArrowBuf;
+
+import java.util.Iterator;
+
+import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.memory.OutOfMemoryException;
+import org.apache.arrow.vector.complex.impl.NullReader;
+import org.apache.arrow.vector.complex.reader.FieldReader;
+import org.apache.arrow.vector.types.MaterializedField;
+import org.apache.arrow.vector.types.Types;
+import org.apache.arrow.vector.types.Types.MinorType;
+import org.apache.arrow.vector.util.TransferPair;
+
+import com.google.common.collect.Iterators;
+
+public class ZeroVector implements ValueVector {
+  public final static ZeroVector INSTANCE = new ZeroVector();
+
+  private final MaterializedField field = MaterializedField.create("[DEFAULT]", Types.required(MinorType.LATE));
+
+  private final TransferPair defaultPair = new TransferPair() {
+    @Override
+    public void transfer() { }
+
+    @Override
+    public void splitAndTransfer(int startIndex, int length) { }
+
+    @Override
+    public ValueVector getTo() {
+      return ZeroVector.this;
+    }
+
+    @Override
+    public void copyValueSafe(int from, int to) { }
+  };
+
+  private final Accessor defaultAccessor = new Accessor() {
+    @Override
+    public Object getObject(int index) {
+      return null;
+    }
+
+    @Override
+    public int getValueCount() {
+      return 0;
+    }
+
+    @Override
+    public boolean isNull(int index) {
+      return true;
+    }
+  };
+
+  private final Mutator defaultMutator = new Mutator() {
+    @Override
+    public void setValueCount(int valueCount) { }
+
+    @Override
+    public void reset() { }
+
+    @Override
+    public void generateTestData(int values) { }
+  };
+
+  public ZeroVector() { }
+
+  @Override
+  public void close() { }
+
+  @Override
+  public void clear() { }
+
+  @Override
+  public MaterializedField getField() {
+    return field;
+  }
+
+  @Override
+  public TransferPair getTransferPair(BufferAllocator allocator) {
+    return defaultPair;
+  }
+
+//  @Override
+//  public UserBitShared.SerializedField getMetadata() {
+//    return getField()
+//        .getAsBuilder()
+//        .setBufferLength(getBufferSize())
+//        .setValueCount(getAccessor().getValueCount())
+//        .build();
+//  }
+
+  @Override
+  public Iterator iterator() {
+    return Iterators.emptyIterator();
+  }
+
+  @Override
+  public int getBufferSize() {
+    return 0;
+  }
+
+  @Override
+  public int getBufferSizeFor(final int valueCount) {
+    return 0;
+  }
+
+  @Override
+  public ArrowBuf[] getBuffers(boolean clear) {
+    return new ArrowBuf[0];
+  }
+
+  @Override
+  public void allocateNew() throws OutOfMemoryException {
+    allocateNewSafe();
+  }
+
+  @Override
+  public boolean allocateNewSafe() {
+    return true;
+  }
+
+  @Override
+  public BufferAllocator getAllocator() {
+    throw new UnsupportedOperationException("Tried to get allocator from ZeroVector");
+  }
+
+  @Override
+  public void setInitialCapacity(int numRecords) { }
+
+  @Override
+  public int getValueCapacity() {
+    return 0;
+  }
+
+  @Override
+  public TransferPair getTransferPair(String ref, BufferAllocator allocator) {
+    return defaultPair;
+  }
+
+  @Override
+  public TransferPair makeTransferPair(ValueVector target) {
+    return defaultPair;
+  }
+
+  @Override
+  public Accessor getAccessor() {
+    return defaultAccessor;
+  }
+
+  @Override
+  public Mutator getMutator() {
+    return defaultMutator;
+  }
+
+  @Override
+  public FieldReader getReader() {
+    return NullReader.INSTANCE;
+  }
+
+//  @Override
+//  public void load(UserBitShared.SerializedField metadata, DrillBuf buffer) { }
+}

http://git-wip-us.apache.org/repos/asf/arrow/blob/fa5f0299/java/vector/src/main/java/org/apache/arrow/vector/complex/AbstractContainerVector.java
----------------------------------------------------------------------
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/AbstractContainerVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/AbstractContainerVector.java
new file mode 100644
index 0000000..c671c9e
--- /dev/null
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/AbstractContainerVector.java
@@ -0,0 +1,143 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.arrow.vector.complex;
+
+import java.util.Collection;
+
+import javax.annotation.Nullable;
+
+import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.memory.OutOfMemoryException;
+import org.apache.arrow.vector.ValueVector;
+import org.apache.arrow.vector.types.MaterializedField;
+import org.apache.arrow.vector.types.Types.DataMode;
+import org.apache.arrow.vector.types.Types.MajorType;
+import org.apache.arrow.vector.types.Types.MinorType;
+import org.apache.arrow.vector.util.CallBack;
+
+import com.google.common.base.Function;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Sets;
+
+/**
+ * Base class for composite vectors.
+ *
+ * This class implements common functionality of composite vectors.
+ */
+public abstract class AbstractContainerVector implements ValueVector {
+  static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(AbstractContainerVector.class);
+
+  protected MaterializedField field;
+  protected final BufferAllocator allocator;
+  protected final CallBack callBack;
+
+  protected AbstractContainerVector(MaterializedField field, BufferAllocator allocator, CallBack callBack) {
+    this.field = Preconditions.checkNotNull(field);
+    this.allocator = allocator;
+    this.callBack = callBack;
+  }
+
+  @Override
+  public void allocateNew() throws OutOfMemoryException {
+    if (!allocateNewSafe()) {
+      throw new OutOfMemoryException();
+    }
+  }
+
+  public BufferAllocator getAllocator() {
+    return allocator;
+  }
+
+  /**
+   * Returns the field definition of this instance.
+   */
+  @Override
+  public MaterializedField getField() {
+    return field;
+  }
+
+  /**
+   * Returns a {@link org.apache.arrow.vector.ValueVector} corresponding to the given field name if exists or null.
+   */
+  public ValueVector getChild(String name) {
+    return getChild(name, ValueVector.class);
+  }
+
+  /**
+   * Returns a sequence of field names in the order that they show up in the schema.
+   */
+  protected Collection<String> getChildFieldNames() {
+    return Sets.newLinkedHashSet(Iterables.transform(field.getChildren(), new Function<MaterializedField, String>() {
+      @Nullable
+      @Override
+      public String apply(MaterializedField field) {
+        return Preconditions.checkNotNull(field).getLastName();
+      }
+    }));
+  }
+
+  /**
+   * Clears out all underlying child vectors.
+   */
+ @Override
+  public void close() {
+    for (ValueVector vector:(Iterable<ValueVector>)this) {
+      vector.close();
+    }
+  }
+
+  protected <T extends ValueVector> T typeify(ValueVector v, Class<T> clazz) {
+    if (clazz.isAssignableFrom(v.getClass())) {
+      return (T) v;
+    }
+    throw new IllegalStateException(String.format("Vector requested [%s] was different than type stored [%s].  Drill doesn't yet support hetergenous types.", clazz.getSimpleName(), v.getClass().getSimpleName()));
+  }
+
+  MajorType getLastPathType() {
+    if((this.getField().getType().getMinorType() == MinorType.LIST  &&
+        this.getField().getType().getMode() == DataMode.REPEATED)) {  // Use Repeated scalar type instead of Required List.
+      VectorWithOrdinal vord = getChildVectorWithOrdinal(null);
+      ValueVector v = vord.vector;
+      if (! (v instanceof  AbstractContainerVector)) {
+        return v.getField().getType();
+      }
+    } else if (this.getField().getType().getMinorType() == MinorType.MAP  &&
+        this.getField().getType().getMode() == DataMode.REPEATED) {  // Use Required Map
+      return new MajorType(MinorType.MAP, DataMode.REQUIRED);
+    }
+
+    return this.getField().getType();
+  }
+
+  protected boolean supportsDirectRead() {
+    return false;
+  }
+
+  // return the number of child vectors
+  public abstract int size();
+
+  // add a new vector with the input MajorType or return the existing vector if we already added one with the same type
+  public abstract <T extends ValueVector> T addOrGet(String name, MajorType type, Class<T> clazz);
+
+  // return the child vector with the input name
+  public abstract <T extends ValueVector> T getChild(String name, Class<T> clazz);
+
+  // return the child vector's ordinal in the composite container
+  public abstract VectorWithOrdinal getChildVectorWithOrdinal(String name);
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/arrow/blob/fa5f0299/java/vector/src/main/java/org/apache/arrow/vector/complex/AbstractMapVector.java
----------------------------------------------------------------------
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/AbstractMapVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/AbstractMapVector.java
new file mode 100644
index 0000000..d4189b2
--- /dev/null
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/AbstractMapVector.java
@@ -0,0 +1,278 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.arrow.vector.complex;
+
+import io.netty.buffer.ArrowBuf;
+
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.List;
+
+import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.vector.ValueVector;
+import org.apache.arrow.vector.types.MaterializedField;
+import org.apache.arrow.vector.types.Types.MajorType;
+import org.apache.arrow.vector.util.BasicTypeHelper;
+import org.apache.arrow.vector.util.CallBack;
+import org.apache.arrow.vector.util.MapWithOrdinal;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Lists;
+
+/*
+ * Base class for MapVectors. Currently used by RepeatedMapVector and MapVector
+ */
+public abstract class AbstractMapVector extends AbstractContainerVector {
+  private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(AbstractContainerVector.class);
+
+  // Maintains a map with key as field name and value is the vector itself
+  private final MapWithOrdinal<String, ValueVector> vectors =  new MapWithOrdinal<>();
+
+  protected AbstractMapVector(MaterializedField field, BufferAllocator allocator, CallBack callBack) {
+    super(field.clone(), allocator, callBack);
+    MaterializedField clonedField = field.clone();
+    // create the hierarchy of the child vectors based on the materialized field
+    for (MaterializedField child : clonedField.getChildren()) {
+      if (!child.equals(BaseRepeatedValueVector.OFFSETS_FIELD)) {
+        final String fieldName = child.getLastName();
+        final ValueVector v = BasicTypeHelper.getNewVector(child, allocator, callBack);
+        putVector(fieldName, v);
+      }
+    }
+  }
+
+  @Override
+  public void close() {
+    for(final ValueVector valueVector : vectors.values()) {
+      valueVector.close();
+    }
+    vectors.clear();
+
+    super.close();
+  }
+
+  @Override
+  public boolean allocateNewSafe() {
+    /* boolean to keep track if all the memory allocation were successful
+     * Used in the case of composite vectors when we need to allocate multiple
+     * buffers for multiple vectors. If one of the allocations failed we need to
+     * clear all the memory that we allocated
+     */
+    boolean success = false;
+    try {
+      for (final ValueVector v : vectors.values()) {
+        if (!v.allocateNewSafe()) {
+          return false;
+        }
+      }
+      success = true;
+    } finally {
+      if (!success) {
+        clear();
+      }
+    }
+    return true;
+  }
+
+  /**
+   * Adds a new field with the given parameters or replaces the existing one and consequently returns the resultant
+   * {@link org.apache.arrow.vector.ValueVector}.
+   *
+   * Execution takes place in the following order:
+   * <ul>
+   *   <li>
+   *     if field is new, create and insert a new vector of desired type.
+   *   </li>
+   *   <li>
+   *     if field exists and existing vector is of desired vector type, return the vector.
+   *   </li>
+   *   <li>
+   *     if field exists and null filled, clear the existing vector; create and insert a new vector of desired type.
+   *   </li>
+   *   <li>
+   *     otherwise, throw an {@link java.lang.IllegalStateException}
+   *   </li>
+   * </ul>
+   *
+   * @param name name of the field
+   * @param type type of the field
+   * @param clazz class of expected vector type
+   * @param <T> class type of expected vector type
+   * @throws java.lang.IllegalStateException raised if there is a hard schema change
+   *
+   * @return resultant {@link org.apache.arrow.vector.ValueVector}
+   */
+  @Override
+  public <T extends ValueVector> T addOrGet(String name, MajorType type, Class<T> clazz) {
+    final ValueVector existing = getChild(name);
+    boolean create = false;
+    if (existing == null) {
+      create = true;
+    } else if (clazz.isAssignableFrom(existing.getClass())) {
+      return (T) existing;
+    } else if (nullFilled(existing)) {
+      existing.clear();
+      create = true;
+    }
+    if (create) {
+      final T vector = (T) BasicTypeHelper.getNewVector(name, allocator, type, callBack);
+      putChild(name, vector);
+      if (callBack!=null) {
+        callBack.doWork();
+      }
+      return vector;
+    }
+    final String message = "Drill does not support schema change yet. Existing[%s] and desired[%s] vector types mismatch";
+    throw new IllegalStateException(String.format(message, existing.getClass().getSimpleName(), clazz.getSimpleName()));
+  }
+
+  private boolean nullFilled(ValueVector vector) {
+    for (int r = 0; r < vector.getAccessor().getValueCount(); r++) {
+      if (!vector.getAccessor().isNull(r)) {
+        return false;
+      }
+    }
+    return true;
+  }
+
+  /**
+   * Returns a {@link org.apache.arrow.vector.ValueVector} corresponding to the given ordinal identifier.
+   */
+  public ValueVector getChildByOrdinal(int id) {
+    return vectors.getByOrdinal(id);
+  }
+
+  /**
+   * Returns a {@link org.apache.arrow.vector.ValueVector} instance of subtype of <T> corresponding to the given
+   * field name if exists or null.
+   */
+  @Override
+  public <T extends ValueVector> T getChild(String name, Class<T> clazz) {
+    final ValueVector v = vectors.get(name.toLowerCase());
+    if (v == null) {
+      return null;
+    }
+    return typeify(v, clazz);
+  }
+
+  /**
+   * Inserts the vector with the given name if it does not exist else replaces it with the new value.
+   *
+   * Note that this method does not enforce any vector type check nor throws a schema change exception.
+   */
+  protected void putChild(String name, ValueVector vector) {
+    putVector(name, vector);
+    field.addChild(vector.getField());
+  }
+
+  /**
+   * Inserts the input vector into the map if it does not exist, replaces if it exists already
+   * @param name  field name
+   * @param vector  vector to be inserted
+   */
+  protected void putVector(String name, ValueVector vector) {
+    final ValueVector old = vectors.put(
+        Preconditions.checkNotNull(name, "field name cannot be null").toLowerCase(),
+        Preconditions.checkNotNull(vector, "vector cannot be null")
+    );
+    if (old != null && old != vector) {
+      logger.debug("Field [{}] mutated from [{}] to [{}]", name, old.getClass().getSimpleName(),
+                   vector.getClass().getSimpleName());
+    }
+  }
+
+  /**
+   * Returns a sequence of underlying child vectors.
+   */
+  protected Collection<ValueVector> getChildren() {
+    return vectors.values();
+  }
+
+  /**
+   * Returns the number of underlying child vectors.
+   */
+  @Override
+  public int size() {
+    return vectors.size();
+  }
+
+  @Override
+  public Iterator<ValueVector> iterator() {
+    return vectors.values().iterator();
+  }
+
+  /**
+   * Returns a list of scalar child vectors recursing the entire vector hierarchy.
+   */
+  public List<ValueVector> getPrimitiveVectors() {
+    final List<ValueVector> primitiveVectors = Lists.newArrayList();
+    for (final ValueVector v : vectors.values()) {
+      if (v instanceof AbstractMapVector) {
+        AbstractMapVector mapVector = (AbstractMapVector) v;
+        primitiveVectors.addAll(mapVector.getPrimitiveVectors());
+      } else {
+        primitiveVectors.add(v);
+      }
+    }
+    return primitiveVectors;
+  }
+
+  /**
+   * Returns a vector with its corresponding ordinal mapping if field exists or null.
+   */
+  @Override
+  public VectorWithOrdinal getChildVectorWithOrdinal(String name) {
+    final int ordinal = vectors.getOrdinal(name.toLowerCase());
+    if (ordinal < 0) {
+      return null;
+    }
+    final ValueVector vector = vectors.getByOrdinal(ordinal);
+    return new VectorWithOrdinal(vector, ordinal);
+  }
+
+  @Override
+  public ArrowBuf[] getBuffers(boolean clear) {
+    final List<ArrowBuf> buffers = Lists.newArrayList();
+
+    for (final ValueVector vector : vectors.values()) {
+      for (final ArrowBuf buf : vector.getBuffers(false)) {
+        buffers.add(buf);
+        if (clear) {
+          buf.retain(1);
+        }
+      }
+      if (clear) {
+        vector.clear();
+      }
+    }
+
+    return buffers.toArray(new ArrowBuf[buffers.size()]);
+  }
+
+  @Override
+  public int getBufferSize() {
+    int actualBufSize = 0 ;
+
+    for (final ValueVector v : vectors.values()) {
+      for (final ArrowBuf buf : v.getBuffers(false)) {
+        actualBufSize += buf.writerIndex();
+      }
+    }
+    return actualBufSize;
+  }
+}

http://git-wip-us.apache.org/repos/asf/arrow/blob/fa5f0299/java/vector/src/main/java/org/apache/arrow/vector/complex/BaseRepeatedValueVector.java
----------------------------------------------------------------------
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/BaseRepeatedValueVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/BaseRepeatedValueVector.java
new file mode 100644
index 0000000..6518897
--- /dev/null
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/BaseRepeatedValueVector.java
@@ -0,0 +1,260 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.arrow.vector.complex;
+
+import io.netty.buffer.ArrowBuf;
+
+import java.util.Collections;
+import java.util.Iterator;
+
+import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.vector.AddOrGetResult;
+import org.apache.arrow.vector.BaseValueVector;
+import org.apache.arrow.vector.UInt4Vector;
+import org.apache.arrow.vector.ValueVector;
+import org.apache.arrow.vector.VectorDescriptor;
+import org.apache.arrow.vector.ZeroVector;
+import org.apache.arrow.vector.types.MaterializedField;
+import org.apache.arrow.vector.types.Types.DataMode;
+import org.apache.arrow.vector.types.Types.MajorType;
+import org.apache.arrow.vector.types.Types.MinorType;
+import org.apache.arrow.vector.util.BasicTypeHelper;
+import org.apache.arrow.vector.util.SchemaChangeRuntimeException;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ObjectArrays;
+
+public abstract class BaseRepeatedValueVector extends BaseValueVector implements RepeatedValueVector {
+
+  public final static ValueVector DEFAULT_DATA_VECTOR = ZeroVector.INSTANCE;
+  public final static String OFFSETS_VECTOR_NAME = "$offsets$";
+  public final static String DATA_VECTOR_NAME = "$data$";
+
+  public final static MaterializedField OFFSETS_FIELD =
+    MaterializedField.create(OFFSETS_VECTOR_NAME, new MajorType(MinorType.UINT4, DataMode.REQUIRED));
+
+  protected final UInt4Vector offsets;
+  protected ValueVector vector;
+
+  protected BaseRepeatedValueVector(MaterializedField field, BufferAllocator allocator) {
+    this(field, allocator, DEFAULT_DATA_VECTOR);
+  }
+
+  protected BaseRepeatedValueVector(MaterializedField field, BufferAllocator allocator, ValueVector vector) {
+    super(field, allocator);
+    this.offsets = new UInt4Vector(OFFSETS_FIELD, allocator);
+    this.vector = Preconditions.checkNotNull(vector, "data vector cannot be null");
+  }
+
+  @Override
+  public boolean allocateNewSafe() {
+    /* boolean to keep track if all the memory allocation were successful
+     * Used in the case of composite vectors when we need to allocate multiple
+     * buffers for multiple vectors. If one of the allocations failed we need to
+     * clear all the memory that we allocated
+     */
+    boolean success = false;
+    try {
+      if (!offsets.allocateNewSafe()) {
+        return false;
+      }
+      success = vector.allocateNewSafe();
+    } finally {
+      if (!success) {
+        clear();
+      }
+    }
+    offsets.zeroVector();
+    return success;
+  }
+
+
+  @Override
+  public UInt4Vector getOffsetVector() {
+    return offsets;
+  }
+
+  @Override
+  public ValueVector getDataVector() {
+    return vector;
+  }
+
+  @Override
+  public void setInitialCapacity(int numRecords) {
+    offsets.setInitialCapacity(numRecords + 1);
+    vector.setInitialCapacity(numRecords * RepeatedValueVector.DEFAULT_REPEAT_PER_RECORD);
+  }
+
+  @Override
+  public int getValueCapacity() {
+    final int offsetValueCapacity = Math.max(offsets.getValueCapacity() - 1, 0);
+    if (vector == DEFAULT_DATA_VECTOR) {
+      return offsetValueCapacity;
+    }
+    return Math.min(vector.getValueCapacity(), offsetValueCapacity);
+  }
+
+//  @Override
+//  protected UserBitShared.SerializedField.Builder getMetadataBuilder() {
+//    return super.getMetadataBuilder()
+//        .addChild(offsets.getMetadata())
+//        .addChild(vector.getMetadata());
+//  }
+
+  @Override
+  public int getBufferSize() {
+    if (getAccessor().getValueCount() == 0) {
+      return 0;
+    }
+    return offsets.getBufferSize() + vector.getBufferSize();
+  }
+
+  @Override
+  public int getBufferSizeFor(int valueCount) {
+    if (valueCount == 0) {
+      return 0;
+    }
+
+    return offsets.getBufferSizeFor(valueCount + 1) + vector.getBufferSizeFor(valueCount);
+  }
+
+  @Override
+  public Iterator<ValueVector> iterator() {
+    return Collections.singleton(getDataVector()).iterator();
+  }
+
+  @Override
+  public void clear() {
+    offsets.clear();
+    vector.clear();
+    super.clear();
+  }
+
+  @Override
+  public ArrowBuf[] getBuffers(boolean clear) {
+    final ArrowBuf[] buffers = ObjectArrays.concat(offsets.getBuffers(false), vector.getBuffers(false), ArrowBuf.class);
+    if (clear) {
+      for (ArrowBuf buffer:buffers) {
+        buffer.retain();
+      }
+      clear();
+    }
+    return buffers;
+  }
+
+//  @Override
+//  public void load(UserBitShared.SerializedField metadata, DrillBuf buffer) {
+//    final UserBitShared.SerializedField offsetMetadata = metadata.getChild(0);
+//    offsets.load(offsetMetadata, buffer);
+//
+//    final UserBitShared.SerializedField vectorMetadata = metadata.getChild(1);
+//    if (getDataVector() == DEFAULT_DATA_VECTOR) {
+//      addOrGetVector(VectorDescriptor.create(vectorMetadata.getMajorType()));
+//    }
+//
+//    final int offsetLength = offsetMetadata.getBufferLength();
+//    final int vectorLength = vectorMetadata.getBufferLength();
+//    vector.load(vectorMetadata, buffer.slice(offsetLength, vectorLength));
+//  }
+
+  /**
+   * Returns 1 if inner vector is explicitly set via #addOrGetVector else 0
+   *
+   * @see {@link ContainerVectorLike#size}
+   */
+  @Override
+  public int size() {
+    return vector == DEFAULT_DATA_VECTOR ? 0:1;
+  }
+
+  @Override
+  public <T extends ValueVector> AddOrGetResult<T> addOrGetVector(VectorDescriptor descriptor) {
+    boolean created = false;
+    if (vector == DEFAULT_DATA_VECTOR && descriptor.getType().getMinorType() != MinorType.LATE) {
+      final MaterializedField field = descriptor.withName(DATA_VECTOR_NAME).getField();
+      vector = BasicTypeHelper.getNewVector(field, allocator);
+      // returned vector must have the same field
+      assert field.equals(vector.getField());
+      getField().addChild(field);
+      created = true;
+    }
+
+    final MajorType actual = vector.getField().getType();
+    if (!actual.equals(descriptor.getType())) {
+      final String msg = String.format("Inner vector type mismatch. Requested type: [%s], actual type: [%s]",
+          descriptor.getType(), actual);
+      throw new SchemaChangeRuntimeException(msg);
+    }
+
+    return new AddOrGetResult<>((T)vector, created);
+  }
+
+  protected void replaceDataVector(ValueVector v) {
+    vector.clear();
+    vector = v;
+  }
+
+  public abstract class BaseRepeatedAccessor extends BaseValueVector.BaseAccessor implements RepeatedAccessor {
+
+    @Override
+    public int getValueCount() {
+      return Math.max(offsets.getAccessor().getValueCount() - 1, 0);
+    }
+
+    @Override
+    public int getInnerValueCount() {
+      return vector.getAccessor().getValueCount();
+    }
+
+    @Override
+    public int getInnerValueCountAt(int index) {
+      return offsets.getAccessor().get(index+1) - offsets.getAccessor().get(index);
+    }
+
+    @Override
+    public boolean isNull(int index) {
+      return false;
+    }
+
+    @Override
+    public boolean isEmpty(int index) {
+      return false;
+    }
+  }
+
+  public abstract class BaseRepeatedMutator extends BaseValueVector.BaseMutator implements RepeatedMutator {
+
+    @Override
+    public void startNewValue(int index) {
+      while (offsets.getValueCapacity() <= index) {
+        offsets.reAlloc();
+      }
+      offsets.getMutator().setSafe(index+1, offsets.getAccessor().get(index));
+      setValueCount(index+1);
+    }
+
+    @Override
+    public void setValueCount(int valueCount) {
+      // TODO: populate offset end points
+      offsets.getMutator().setValueCount(valueCount == 0 ? 0 : valueCount+1);
+      final int childValueCount = valueCount == 0 ? 0 : offsets.getAccessor().get(valueCount);
+      vector.getMutator().setValueCount(childValueCount);
+    }
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/arrow/blob/fa5f0299/java/vector/src/main/java/org/apache/arrow/vector/complex/ContainerVectorLike.java
----------------------------------------------------------------------
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/ContainerVectorLike.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/ContainerVectorLike.java
new file mode 100644
index 0000000..e50b0d0
--- /dev/null
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/ContainerVectorLike.java
@@ -0,0 +1,43 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.arrow.vector.complex;
+
+import org.apache.arrow.vector.AddOrGetResult;
+import org.apache.arrow.vector.ValueVector;
+import org.apache.arrow.vector.VectorDescriptor;
+
+/**
+ * A mix-in used for introducing container vector-like behaviour.
+ */
+public interface ContainerVectorLike {
+
+  /**
+   * Creates and adds a child vector if none with the same name exists, else returns the vector instance.
+   *
+   * @param  descriptor vector descriptor
+   * @return  result of operation wrapping vector corresponding to the given descriptor and whether it's newly created
+   * @throws org.apache.drill.common.exceptions.DrillRuntimeException
+   *    if schema change is not permissible between the given and existing data vector types.
+   */
+  <T extends ValueVector> AddOrGetResult<T> addOrGetVector(VectorDescriptor descriptor);
+
+  /**
+   * Returns the number of child vectors in this container vector-like instance.
+   */
+  int size();
+}

http://git-wip-us.apache.org/repos/asf/arrow/blob/fa5f0299/java/vector/src/main/java/org/apache/arrow/vector/complex/EmptyValuePopulator.java
----------------------------------------------------------------------
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/EmptyValuePopulator.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/EmptyValuePopulator.java
new file mode 100644
index 0000000..df69975
--- /dev/null
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/EmptyValuePopulator.java
@@ -0,0 +1,54 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.arrow.vector.complex;
+
+import org.apache.arrow.vector.UInt4Vector;
+
+import com.google.common.base.Preconditions;
+
+/**
+ * A helper class that is used to track and populate empty values in repeated value vectors.
+ */
+public class EmptyValuePopulator {
+  private final UInt4Vector offsets;
+
+  public EmptyValuePopulator(UInt4Vector offsets) {
+    this.offsets = Preconditions.checkNotNull(offsets, "offsets cannot be null");
+  }
+
+  /**
+   * Marks all values since the last set as empty. The last set value is obtained from underlying offsets vector.
+   *
+   * @param lastIndex  the last index (inclusive) in the offsets vector until which empty population takes place
+   * @throws java.lang.IndexOutOfBoundsException  if lastIndex is negative or greater than offsets capacity.
+   */
+  public void populate(int lastIndex) {
+    if (lastIndex < 0) {
+      throw new IndexOutOfBoundsException("index cannot be negative");
+    }
+    final UInt4Vector.Accessor accessor = offsets.getAccessor();
+    final UInt4Vector.Mutator mutator = offsets.getMutator();
+    final int lastSet = Math.max(accessor.getValueCount() - 1, 0);
+    final int previousEnd = accessor.get(lastSet);//0 ? 0 : accessor.get(lastSet);
+    for (int i = lastSet; i < lastIndex; i++) {
+      mutator.setSafe(i + 1, previousEnd);
+    }
+    mutator.setValueCount(lastIndex+1);
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/arrow/blob/fa5f0299/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java
----------------------------------------------------------------------
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java
new file mode 100644
index 0000000..8387c9e
--- /dev/null
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java
@@ -0,0 +1,321 @@
+/*******************************************************************************
+
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ ******************************************************************************/
+package org.apache.arrow.vector.complex;
+
+import io.netty.buffer.ArrowBuf;
+
+import java.util.List;
+
+import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.memory.OutOfMemoryException;
+import org.apache.arrow.vector.AddOrGetResult;
+import org.apache.arrow.vector.UInt1Vector;
+import org.apache.arrow.vector.UInt4Vector;
+import org.apache.arrow.vector.ValueVector;
+import org.apache.arrow.vector.VectorDescriptor;
+import org.apache.arrow.vector.ZeroVector;
+import org.apache.arrow.vector.complex.impl.ComplexCopier;
+import org.apache.arrow.vector.complex.impl.UnionListReader;
+import org.apache.arrow.vector.complex.impl.UnionListWriter;
+import org.apache.arrow.vector.complex.reader.FieldReader;
+import org.apache.arrow.vector.complex.writer.FieldWriter;
+import org.apache.arrow.vector.types.MaterializedField;
+import org.apache.arrow.vector.types.Types.DataMode;
+import org.apache.arrow.vector.types.Types.MajorType;
+import org.apache.arrow.vector.types.Types.MinorType;
+import org.apache.arrow.vector.util.CallBack;
+import org.apache.arrow.vector.util.JsonStringArrayList;
+import org.apache.arrow.vector.util.TransferPair;
+
+import com.google.common.collect.ObjectArrays;
+
+public class ListVector extends BaseRepeatedValueVector {
+
+  private UInt4Vector offsets;
+  private final UInt1Vector bits;
+  private Mutator mutator = new Mutator();
+  private Accessor accessor = new Accessor();
+  private UnionListWriter writer;
+  private UnionListReader reader;
+  private CallBack callBack;
+
+  public ListVector(MaterializedField field, BufferAllocator allocator, CallBack callBack) {
+    super(field, allocator);
+    this.bits = new UInt1Vector(MaterializedField.create("$bits$", new MajorType(MinorType.UINT1, DataMode.REQUIRED)), allocator);
+    offsets = getOffsetVector();
+    this.field.addChild(getDataVector().getField());
+    this.writer = new UnionListWriter(this);
+    this.reader = new UnionListReader(this);
+    this.callBack = callBack;
+  }
+
+  public UnionListWriter getWriter() {
+    return writer;
+  }
+
+  @Override
+  public void allocateNew() throws OutOfMemoryException {
+    super.allocateNewSafe();
+  }
+
+  public void transferTo(ListVector target) {
+    offsets.makeTransferPair(target.offsets).transfer();
+    bits.makeTransferPair(target.bits).transfer();
+    if (target.getDataVector() instanceof ZeroVector) {
+      target.addOrGetVector(new VectorDescriptor(vector.getField().getType()));
+    }
+    getDataVector().makeTransferPair(target.getDataVector()).transfer();
+  }
+
+  public void copyFromSafe(int inIndex, int outIndex, ListVector from) {
+    copyFrom(inIndex, outIndex, from);
+  }
+
+  public void copyFrom(int inIndex, int outIndex, ListVector from) {
+    FieldReader in = from.getReader();
+    in.setPosition(inIndex);
+    FieldWriter out = getWriter();
+    out.setPosition(outIndex);
+    ComplexCopier.copy(in, out);
+  }
+
+  @Override
+  public ValueVector getDataVector() {
+    return vector;
+  }
+
+  @Override
+  public TransferPair getTransferPair(String ref, BufferAllocator allocator) {
+    return new TransferImpl(field.withPath(ref), allocator);
+  }
+
+  @Override
+  public TransferPair makeTransferPair(ValueVector target) {
+    return new TransferImpl((ListVector) target);
+  }
+
+  private class TransferImpl implements TransferPair {
+
+    ListVector to;
+
+    public TransferImpl(MaterializedField field, BufferAllocator allocator) {
+      to = new ListVector(field, allocator, null);
+      to.addOrGetVector(new VectorDescriptor(vector.getField().getType()));
+    }
+
+    public TransferImpl(ListVector to) {
+      this.to = to;
+      to.addOrGetVector(new VectorDescriptor(vector.getField().getType()));
+    }
+
+    @Override
+    public void transfer() {
+      transferTo(to);
+    }
+
+    @Override
+    public void splitAndTransfer(int startIndex, int length) {
+      to.allocateNew();
+      for (int i = 0; i < length; i++) {
+        copyValueSafe(startIndex + i, i);
+      }
+    }
+
+    @Override
+    public ValueVector getTo() {
+      return to;
+    }
+
+    @Override
+    public void copyValueSafe(int from, int to) {
+      this.to.copyFrom(from, to, ListVector.this);
+    }
+  }
+
+  @Override
+  public Accessor getAccessor() {
+    return accessor;
+  }
+
+  @Override
+  public Mutator getMutator() {
+    return mutator;
+  }
+
+  @Override
+  public FieldReader getReader() {
+    return reader;
+  }
+
+  @Override
+  public boolean allocateNewSafe() {
+    /* boolean to keep track if all the memory allocation were successful
+     * Used in the case of composite vectors when we need to allocate multiple
+     * buffers for multiple vectors. If one of the allocations failed we need to
+     * clear all the memory that we allocated
+     */
+    boolean success = false;
+    try {
+      if (!offsets.allocateNewSafe()) {
+        return false;
+      }
+      success = vector.allocateNewSafe();
+      success = success && bits.allocateNewSafe();
+    } finally {
+      if (!success) {
+        clear();
+      }
+    }
+    if (success) {
+      offsets.zeroVector();
+      bits.zeroVector();
+    }
+    return success;
+  }
+
+//  @Override
+//  protected UserBitShared.SerializedField.Builder getMetadataBuilder() {
+//    return getField().getAsBuilder()
+//            .setValueCount(getAccessor().getValueCount())
+//            .setBufferLength(getBufferSize())
+//            .addChild(offsets.getMetadata())
+//            .addChild(bits.getMetadata())
+//            .addChild(vector.getMetadata());
+//  }
+  public <T extends ValueVector> AddOrGetResult<T> addOrGetVector(VectorDescriptor descriptor) {
+    AddOrGetResult<T> result = super.addOrGetVector(descriptor);
+    reader = new UnionListReader(this);
+    return result;
+  }
+
+  @Override
+  public int getBufferSize() {
+    if (getAccessor().getValueCount() == 0) {
+      return 0;
+    }
+    return offsets.getBufferSize() + bits.getBufferSize() + vector.getBufferSize();
+  }
+
+  @Override
+  public void clear() {
+    offsets.clear();
+    vector.clear();
+    bits.clear();
+    lastSet = 0;
+    super.clear();
+  }
+
+  @Override
+  public ArrowBuf[] getBuffers(boolean clear) {
+    final ArrowBuf[] buffers = ObjectArrays.concat(offsets.getBuffers(false), ObjectArrays.concat(bits.getBuffers(false),
+            vector.getBuffers(false), ArrowBuf.class), ArrowBuf.class);
+    if (clear) {
+      for (ArrowBuf buffer:buffers) {
+        buffer.retain();
+      }
+      clear();
+    }
+    return buffers;
+  }
+
+//  @Override
+//  public void load(UserBitShared.SerializedField metadata, DrillBuf buffer) {
+//    final UserBitShared.SerializedField offsetMetadata = metadata.getChild(0);
+//    offsets.load(offsetMetadata, buffer);
+//
+//    final int offsetLength = offsetMetadata.getBufferLength();
+//    final UserBitShared.SerializedField bitMetadata = metadata.getChild(1);
+//    final int bitLength = bitMetadata.getBufferLength();
+//    bits.load(bitMetadata, buffer.slice(offsetLength, bitLength));
+//
+//    final UserBitShared.SerializedField vectorMetadata = metadata.getChild(2);
+//    if (getDataVector() == DEFAULT_DATA_VECTOR) {
+//      addOrGetVector(VectorDescriptor.create(vectorMetadata.getMajorType()));
+//    }
+//
+//    final int vectorLength = vectorMetadata.getBufferLength();
+//    vector.load(vectorMetadata, buffer.slice(offsetLength + bitLength, vectorLength));
+//  }
+
+  public UnionVector promoteToUnion() {
+    MaterializedField newField = MaterializedField.create(getField().getPath(), new MajorType(MinorType.UNION, DataMode.OPTIONAL));
+    UnionVector vector = new UnionVector(newField, allocator, null);
+    replaceDataVector(vector);
+    reader = new UnionListReader(this);
+    return vector;
+  }
+
+  private int lastSet;
+
+  public class Accessor extends BaseRepeatedAccessor {
+
+    @Override
+    public Object getObject(int index) {
+      if (isNull(index)) {
+        return null;
+      }
+      final List<Object> vals = new JsonStringArrayList<>();
+      final UInt4Vector.Accessor offsetsAccessor = offsets.getAccessor();
+      final int start = offsetsAccessor.get(index);
+      final int end = offsetsAccessor.get(index + 1);
+      final ValueVector.Accessor valuesAccessor = getDataVector().getAccessor();
+      for(int i = start; i < end; i++) {
+        vals.add(valuesAccessor.getObject(i));
+      }
+      return vals;
+    }
+
+    @Override
+    public boolean isNull(int index) {
+      return bits.getAccessor().get(index) == 0;
+    }
+  }
+
+  public class Mutator extends BaseRepeatedMutator {
+    public void setNotNull(int index) {
+      bits.getMutator().setSafe(index, 1);
+      lastSet = index + 1;
+    }
+
+    @Override
+    public void startNewValue(int index) {
+      for (int i = lastSet; i <= index; i++) {
+        offsets.getMutator().setSafe(i + 1, offsets.getAccessor().get(i));
+      }
+      setNotNull(index);
+      lastSet = index + 1;
+    }
+
+    @Override
+    public void setValueCount(int valueCount) {
+      // TODO: populate offset end points
+      if (valueCount == 0) {
+        offsets.getMutator().setValueCount(0);
+      } else {
+        for (int i = lastSet; i < valueCount; i++) {
+          offsets.getMutator().setSafe(i + 1, offsets.getAccessor().get(i));
+        }
+        offsets.getMutator().setValueCount(valueCount + 1);
+      }
+      final int childValueCount = valueCount == 0 ? 0 : offsets.getAccessor().get(valueCount);
+      vector.getMutator().setValueCount(childValueCount);
+      bits.getMutator().setValueCount(valueCount);
+    }
+  }
+}

http://git-wip-us.apache.org/repos/asf/arrow/blob/fa5f0299/java/vector/src/main/java/org/apache/arrow/vector/complex/MapVector.java
----------------------------------------------------------------------
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/MapVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/MapVector.java
new file mode 100644
index 0000000..1bbce73
--- /dev/null
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/MapVector.java
@@ -0,0 +1,374 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.arrow.vector.complex;
+
+import io.netty.buffer.ArrowBuf;
+
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.Map;
+
+import javax.annotation.Nullable;
+
+import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.vector.BaseValueVector;
+import org.apache.arrow.vector.ValueVector;
+import org.apache.arrow.vector.complex.RepeatedMapVector.MapSingleCopier;
+import org.apache.arrow.vector.complex.impl.SingleMapReaderImpl;
+import org.apache.arrow.vector.complex.reader.FieldReader;
+import org.apache.arrow.vector.holders.ComplexHolder;
+import org.apache.arrow.vector.types.MaterializedField;
+import org.apache.arrow.vector.types.Types.DataMode;
+import org.apache.arrow.vector.types.Types.MajorType;
+import org.apache.arrow.vector.types.Types.MinorType;
+import org.apache.arrow.vector.util.CallBack;
+import org.apache.arrow.vector.util.JsonStringHashMap;
+import org.apache.arrow.vector.util.TransferPair;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Ordering;
+import com.google.common.primitives.Ints;
+
+public class MapVector extends AbstractMapVector {
+  //private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(MapVector.class);
+
+  public final static MajorType TYPE = new MajorType(MinorType.MAP, DataMode.OPTIONAL);
+
+  private final SingleMapReaderImpl reader = new SingleMapReaderImpl(MapVector.this);
+  private final Accessor accessor = new Accessor();
+  private final Mutator mutator = new Mutator();
+  private int valueCount;
+
+  public MapVector(String path, BufferAllocator allocator, CallBack callBack){
+    this(MaterializedField.create(path, TYPE), allocator, callBack);
+  }
+
+  public MapVector(MaterializedField field, BufferAllocator allocator, CallBack callBack){
+    super(field, allocator, callBack);
+  }
+
+  @Override
+  public FieldReader getReader() {
+    //return new SingleMapReaderImpl(MapVector.this);
+    return reader;
+  }
+
+  transient private MapTransferPair ephPair;
+  transient private MapSingleCopier ephPair2;
+
+  public void copyFromSafe(int fromIndex, int thisIndex, MapVector from) {
+    if(ephPair == null || ephPair.from != from) {
+      ephPair = (MapTransferPair) from.makeTransferPair(this);
+    }
+    ephPair.copyValueSafe(fromIndex, thisIndex);
+  }
+
+  public void copyFromSafe(int fromSubIndex, int thisIndex, RepeatedMapVector from) {
+    if(ephPair2 == null || ephPair2.from != from) {
+      ephPair2 = from.makeSingularCopier(this);
+    }
+    ephPair2.copySafe(fromSubIndex, thisIndex);
+  }
+
+  @Override
+  protected boolean supportsDirectRead() {
+    return true;
+  }
+
+  public Iterator<String> fieldNameIterator() {
+    return getChildFieldNames().iterator();
+  }
+
+  @Override
+  public void setInitialCapacity(int numRecords) {
+    for (final ValueVector v : (Iterable<ValueVector>) this) {
+      v.setInitialCapacity(numRecords);
+    }
+  }
+
+  @Override
+  public int getBufferSize() {
+    if (valueCount == 0 || size() == 0) {
+      return 0;
+    }
+    long buffer = 0;
+    for (final ValueVector v : (Iterable<ValueVector>)this) {
+      buffer += v.getBufferSize();
+    }
+
+    return (int) buffer;
+  }
+
+  @Override
+  public int getBufferSizeFor(final int valueCount) {
+    if (valueCount == 0) {
+      return 0;
+    }
+
+    long bufferSize = 0;
+    for (final ValueVector v : (Iterable<ValueVector>) this) {
+      bufferSize += v.getBufferSizeFor(valueCount);
+    }
+
+    return (int) bufferSize;
+  }
+
+  @Override
+  public ArrowBuf[] getBuffers(boolean clear) {
+    int expectedSize = getBufferSize();
+    int actualSize   = super.getBufferSize();
+
+    Preconditions.checkArgument(expectedSize == actualSize);
+    return super.getBuffers(clear);
+  }
+
+  @Override
+  public TransferPair getTransferPair(BufferAllocator allocator) {
+    return new MapTransferPair(this, getField().getPath(), allocator);
+  }
+
+  @Override
+  public TransferPair makeTransferPair(ValueVector to) {
+    return new MapTransferPair(this, (MapVector) to);
+  }
+
+  @Override
+  public TransferPair getTransferPair(String ref, BufferAllocator allocator) {
+    return new MapTransferPair(this, ref, allocator);
+  }
+
+  protected static class MapTransferPair implements TransferPair{
+    private final TransferPair[] pairs;
+    private final MapVector from;
+    private final MapVector to;
+
+    public MapTransferPair(MapVector from, String path, BufferAllocator allocator) {
+      this(from, new MapVector(MaterializedField.create(path, TYPE), allocator, from.callBack), false);
+    }
+
+    public MapTransferPair(MapVector from, MapVector to) {
+      this(from, to, true);
+    }
+
+    protected MapTransferPair(MapVector from, MapVector to, boolean allocate) {
+      this.from = from;
+      this.to = to;
+      this.pairs = new TransferPair[from.size()];
+      this.to.ephPair = null;
+      this.to.ephPair2 = null;
+
+      int i = 0;
+      ValueVector vector;
+      for (String child:from.getChildFieldNames()) {
+        int preSize = to.size();
+        vector = from.getChild(child);
+        if (vector == null) {
+          continue;
+        }
+        //DRILL-1872: we add the child fields for the vector, looking up the field by name. For a map vector,
+        // the child fields may be nested fields of the top level child. For example if the structure
+        // of a child field is oa.oab.oabc then we add oa, then add oab to oa then oabc to oab.
+        // But the children member of a Materialized field is a HashSet. If the fields are added in the
+        // children HashSet, and the hashCode of the Materialized field includes the hash code of the
+        // children, the hashCode value of oa changes *after* the field has been added to the HashSet.
+        // (This is similar to what happens in ScanBatch where the children cannot be added till they are
+        // read). To take care of this, we ensure that the hashCode of the MaterializedField does not
+        // include the hashCode of the children but is based only on MaterializedField$key.
+        final ValueVector newVector = to.addOrGet(child, vector.getField().getType(), vector.getClass());
+        if (allocate && to.size() != preSize) {
+          newVector.allocateNew();
+        }
+        pairs[i++] = vector.makeTransferPair(newVector);
+      }
+    }
+
+    @Override
+    public void transfer() {
+      for (final TransferPair p : pairs) {
+        p.transfer();
+      }
+      to.valueCount = from.valueCount;
+      from.clear();
+    }
+
+    @Override
+    public ValueVector getTo() {
+      return to;
+    }
+
+    @Override
+    public void copyValueSafe(int from, int to) {
+      for (TransferPair p : pairs) {
+        p.copyValueSafe(from, to);
+      }
+    }
+
+    @Override
+    public void splitAndTransfer(int startIndex, int length) {
+      for (TransferPair p : pairs) {
+        p.splitAndTransfer(startIndex, length);
+      }
+      to.getMutator().setValueCount(length);
+    }
+  }
+
+  @Override
+  public int getValueCapacity() {
+    if (size() == 0) {
+      return 0;
+    }
+
+    final Ordering<ValueVector> natural = new Ordering<ValueVector>() {
+      @Override
+      public int compare(@Nullable ValueVector left, @Nullable ValueVector right) {
+        return Ints.compare(
+            Preconditions.checkNotNull(left).getValueCapacity(),
+            Preconditions.checkNotNull(right).getValueCapacity()
+        );
+      }
+    };
+
+    return natural.min(getChildren()).getValueCapacity();
+  }
+
+  @Override
+  public Accessor getAccessor() {
+    return accessor;
+  }
+
+//  @Override
+//  public void load(SerializedField metadata, DrillBuf buf) {
+//    final List<SerializedField> fields = metadata.getChildList();
+//    valueCount = metadata.getValueCount();
+//
+//    int bufOffset = 0;
+//    for (final SerializedField child : fields) {
+//      final MaterializedField fieldDef = SerializedFieldHelper.create(child);
+//
+//      ValueVector vector = getChild(fieldDef.getLastName());
+//      if (vector == null) {
+//         if we arrive here, we didn't have a matching vector.
+//        vector = BasicTypeHelper.getNewVector(fieldDef, allocator);
+//        putChild(fieldDef.getLastName(), vector);
+//      }
+//      if (child.getValueCount() == 0) {
+//        vector.clear();
+//      } else {
+//        vector.load(child, buf.slice(bufOffset, child.getBufferLength()));
+//      }
+//      bufOffset += child.getBufferLength();
+//    }
+//
+//    assert bufOffset == buf.capacity();
+//  }
+//
+//  @Override
+//  public SerializedField getMetadata() {
+//    SerializedField.Builder b = getField() //
+//        .getAsBuilder() //
+//        .setBufferLength(getBufferSize()) //
+//        .setValueCount(valueCount);
+//
+//
+//    for(ValueVector v : getChildren()) {
+//      b.addChild(v.getMetadata());
+//    }
+//    return b.build();
+//  }
+
+  @Override
+  public Mutator getMutator() {
+    return mutator;
+  }
+
+  public class Accessor extends BaseValueVector.BaseAccessor {
+
+    @Override
+    public Object getObject(int index) {
+      Map<String, Object> vv = new JsonStringHashMap<>();
+      for (String child:getChildFieldNames()) {
+        ValueVector v = getChild(child);
+        // TODO(DRILL-4001):  Resolve this hack:
+        // The index/value count check in the following if statement is a hack
+        // to work around the current fact that RecordBatchLoader.load and
+        // MapVector.load leave child vectors with a length of zero (as opposed
+        // to matching the lengths of siblings and the parent map vector)
+        // because they don't remove (or set the lengths of) vectors from
+        // previous batches that aren't in the current batch.
+        if (v != null && index < v.getAccessor().getValueCount()) {
+          Object value = v.getAccessor().getObject(index);
+          if (value != null) {
+            vv.put(child, value);
+          }
+        }
+      }
+      return vv;
+    }
+
+    public void get(int index, ComplexHolder holder) {
+      reader.setPosition(index);
+      holder.reader = reader;
+    }
+
+    @Override
+    public int getValueCount() {
+      return valueCount;
+    }
+  }
+
+  public ValueVector getVectorById(int id) {
+    return getChildByOrdinal(id);
+  }
+
+  public class Mutator extends BaseValueVector.BaseMutator {
+
+    @Override
+    public void setValueCount(int valueCount) {
+      for (final ValueVector v : getChildren()) {
+        v.getMutator().setValueCount(valueCount);
+      }
+      MapVector.this.valueCount = valueCount;
+    }
+
+    @Override
+    public void reset() { }
+
+    @Override
+    public void generateTestData(int values) { }
+  }
+
+  @Override
+  public void clear() {
+    for (final ValueVector v : getChildren()) {
+      v.clear();
+    }
+    valueCount = 0;
+  }
+
+  @Override
+  public void close() {
+    final Collection<ValueVector> vectors = getChildren();
+    for (final ValueVector v : vectors) {
+      v.close();
+    }
+    vectors.clear();
+    valueCount = 0;
+
+    super.close();
+ }
+}

http://git-wip-us.apache.org/repos/asf/arrow/blob/fa5f0299/java/vector/src/main/java/org/apache/arrow/vector/complex/Positionable.java
----------------------------------------------------------------------
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/Positionable.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/Positionable.java
new file mode 100644
index 0000000..9345118
--- /dev/null
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/Positionable.java
@@ -0,0 +1,22 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.arrow.vector.complex;
+
+public interface Positionable {
+  public void setPosition(int index);
+}

http://git-wip-us.apache.org/repos/asf/arrow/blob/fa5f0299/java/vector/src/main/java/org/apache/arrow/vector/complex/RepeatedFixedWidthVectorLike.java
----------------------------------------------------------------------
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/RepeatedFixedWidthVectorLike.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/RepeatedFixedWidthVectorLike.java
new file mode 100644
index 0000000..23850bc
--- /dev/null
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/RepeatedFixedWidthVectorLike.java
@@ -0,0 +1,40 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.arrow.vector.complex;
+
+/**
+ * A {@link org.apache.arrow.vector.ValueVector} mix-in that can be used in conjunction with
+ * {@link RepeatedValueVector} subtypes.
+ */
+public interface RepeatedFixedWidthVectorLike {
+  /**
+   * Allocate a new memory space for this vector.  Must be called prior to using the ValueVector.
+   *
+   * @param valueCount   Number of separate repeating groupings.
+   * @param innerValueCount   Number of supported values in the vector.
+   */
+  void allocateNew(int valueCount, int innerValueCount);
+
+  /**
+   * Load the records in the provided buffer based on the given number of values.
+   * @param valueCount   Number of separate repeating groupings.
+   * @param innerValueCount Number atomic values the buffer contains.
+   * @param buf Incoming buffer.
+   * @return The number of bytes of the buffer that were consumed.
+   */
+}


Mime
View raw message