arrow-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From s..@apache.org
Subject [3/5] arrow git commit: ARROW-259: Use Flatbuffer Field type instead of MaterializedField
Date Thu, 18 Aug 2016 23:30:36 GMT
http://git-wip-us.apache.org/repos/asf/arrow/blob/e7e399db/java/vector/src/main/java/org/apache/arrow/vector/complex/AbstractMapVector.java
----------------------------------------------------------------------
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/AbstractMapVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/AbstractMapVector.java
index de6ae82..5964f80 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/AbstractMapVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/AbstractMapVector.java
@@ -17,17 +17,17 @@
  */
 package org.apache.arrow.vector.complex;
 
+import com.google.common.collect.ImmutableList;
 import io.netty.buffer.ArrowBuf;
 
-import java.util.Collection;
+import java.util.ArrayList;
 import java.util.Iterator;
 import java.util.List;
 
+import org.apache.arrow.flatbuf.Field;
 import org.apache.arrow.memory.BufferAllocator;
 import org.apache.arrow.vector.ValueVector;
-import org.apache.arrow.vector.types.MaterializedField;
-import org.apache.arrow.vector.types.Types.MajorType;
-import org.apache.arrow.vector.util.BasicTypeHelper;
+import org.apache.arrow.vector.types.Types.MinorType;
 import org.apache.arrow.vector.util.CallBack;
 import org.apache.arrow.vector.util.MapWithOrdinal;
 
@@ -43,17 +43,8 @@ public abstract class AbstractMapVector extends AbstractContainerVector {
   // Maintains a map with key as field name and value is the vector itself
   private final MapWithOrdinal<String, ValueVector> vectors =  new MapWithOrdinal<>();
 
-  protected AbstractMapVector(MaterializedField field, BufferAllocator allocator, CallBack callBack) {
-    super(field.clone(), allocator, callBack);
-    MaterializedField clonedField = field.clone();
-    // create the hierarchy of the child vectors based on the materialized field
-    for (MaterializedField child : clonedField.getChildren()) {
-      if (!child.equals(BaseRepeatedValueVector.OFFSETS_FIELD)) {
-        final String fieldName = child.getLastName();
-        final ValueVector v = BasicTypeHelper.getNewVector(child, allocator, callBack);
-        putVector(fieldName, v);
-      }
-    }
+  protected AbstractMapVector(String name, BufferAllocator allocator, CallBack callBack) {
+    super(name, allocator, callBack);
   }
 
   @Override
@@ -109,8 +100,8 @@ public abstract class AbstractMapVector extends AbstractContainerVector {
    *   </li>
    * </ul>
    *
-   * @param name name of the field
-   * @param type type of the field
+   * @param name the name of the field
+   * @param minorType the minorType for the vector
    * @param clazz class of expected vector type
    * @param <T> class type of expected vector type
    * @throws java.lang.IllegalStateException raised if there is a hard schema change
@@ -118,7 +109,7 @@ public abstract class AbstractMapVector extends AbstractContainerVector {
    * @return resultant {@link org.apache.arrow.vector.ValueVector}
    */
   @Override
-  public <T extends ValueVector> T addOrGet(String name, MajorType type, Class<T> clazz) {
+  public <T extends ValueVector> T addOrGet(String name, MinorType minorType, Class<T> clazz, int... precisionScale) {
     final ValueVector existing = getChild(name);
     boolean create = false;
     if (existing == null) {
@@ -130,7 +121,7 @@ public abstract class AbstractMapVector extends AbstractContainerVector {
       create = true;
     }
     if (create) {
-      final T vector = (T) BasicTypeHelper.getNewVector(name, allocator, type, callBack);
+      final T vector = (T) minorType.getNewVector(name, allocator, callBack, precisionScale);
       putChild(name, vector);
       if (callBack!=null) {
         callBack.doWork();
@@ -177,7 +168,6 @@ public abstract class AbstractMapVector extends AbstractContainerVector {
    */
   protected void putChild(String name, ValueVector vector) {
     putVector(name, vector);
-    field.addChild(vector.getField());
   }
 
   /**
@@ -199,8 +189,21 @@ public abstract class AbstractMapVector extends AbstractContainerVector {
   /**
    * Returns a sequence of underlying child vectors.
    */
-  protected Collection<ValueVector> getChildren() {
-    return vectors.values();
+  protected List<ValueVector> getChildren() {
+    int size = vectors.size();
+    List<ValueVector> children = new ArrayList<>();
+    for (int i = 0; i < size; i++) {
+      children.add(vectors.getByOrdinal(i));
+    }
+    return children;
+  }
+
+  protected List<String> getChildFieldNames() {
+    ImmutableList.Builder<String> builder = ImmutableList.builder();
+    for (ValueVector child : getChildren()) {
+      builder.add(child.getField().getName());
+    }
+    return builder.build();
   }
 
   /**

http://git-wip-us.apache.org/repos/asf/arrow/blob/e7e399db/java/vector/src/main/java/org/apache/arrow/vector/complex/BaseRepeatedValueVector.java
----------------------------------------------------------------------
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/BaseRepeatedValueVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/BaseRepeatedValueVector.java
index 6518897..4226274 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/BaseRepeatedValueVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/BaseRepeatedValueVector.java
@@ -22,22 +22,18 @@ import io.netty.buffer.ArrowBuf;
 import java.util.Collections;
 import java.util.Iterator;
 
+import org.apache.arrow.flatbuf.Type;
 import org.apache.arrow.memory.BufferAllocator;
 import org.apache.arrow.vector.AddOrGetResult;
 import org.apache.arrow.vector.BaseValueVector;
 import org.apache.arrow.vector.UInt4Vector;
 import org.apache.arrow.vector.ValueVector;
-import org.apache.arrow.vector.VectorDescriptor;
 import org.apache.arrow.vector.ZeroVector;
-import org.apache.arrow.vector.types.MaterializedField;
-import org.apache.arrow.vector.types.Types.DataMode;
-import org.apache.arrow.vector.types.Types.MajorType;
-import org.apache.arrow.vector.types.Types.MinorType;
-import org.apache.arrow.vector.util.BasicTypeHelper;
-import org.apache.arrow.vector.util.SchemaChangeRuntimeException;
 
 import com.google.common.base.Preconditions;
 import com.google.common.collect.ObjectArrays;
+import org.apache.arrow.vector.types.Types.MinorType;
+import org.apache.arrow.vector.util.SchemaChangeRuntimeException;
 
 public abstract class BaseRepeatedValueVector extends BaseValueVector implements RepeatedValueVector {
 
@@ -45,19 +41,16 @@ public abstract class BaseRepeatedValueVector extends BaseValueVector implements
   public final static String OFFSETS_VECTOR_NAME = "$offsets$";
   public final static String DATA_VECTOR_NAME = "$data$";
 
-  public final static MaterializedField OFFSETS_FIELD =
-    MaterializedField.create(OFFSETS_VECTOR_NAME, new MajorType(MinorType.UINT4, DataMode.REQUIRED));
-
   protected final UInt4Vector offsets;
   protected ValueVector vector;
 
-  protected BaseRepeatedValueVector(MaterializedField field, BufferAllocator allocator) {
-    this(field, allocator, DEFAULT_DATA_VECTOR);
+  protected BaseRepeatedValueVector(String name, BufferAllocator allocator) {
+    this(name, allocator, DEFAULT_DATA_VECTOR);
   }
 
-  protected BaseRepeatedValueVector(MaterializedField field, BufferAllocator allocator, ValueVector vector) {
-    super(field, allocator);
-    this.offsets = new UInt4Vector(OFFSETS_FIELD, allocator);
+  protected BaseRepeatedValueVector(String name, BufferAllocator allocator, ValueVector vector) {
+    super(name, allocator);
+    this.offsets = new UInt4Vector(OFFSETS_VECTOR_NAME, allocator);
     this.vector = Preconditions.checkNotNull(vector, "data vector cannot be null");
   }
 
@@ -109,13 +102,6 @@ public abstract class BaseRepeatedValueVector extends BaseValueVector implements
     return Math.min(vector.getValueCapacity(), offsetValueCapacity);
   }
 
-//  @Override
-//  protected UserBitShared.SerializedField.Builder getMetadataBuilder() {
-//    return super.getMetadataBuilder()
-//        .addChild(offsets.getMetadata())
-//        .addChild(vector.getMetadata());
-//  }
-
   @Override
   public int getBufferSize() {
     if (getAccessor().getValueCount() == 0) {
@@ -157,47 +143,24 @@ public abstract class BaseRepeatedValueVector extends BaseValueVector implements
     return buffers;
   }
 
-//  @Override
-//  public void load(UserBitShared.SerializedField metadata, DrillBuf buffer) {
-//    final UserBitShared.SerializedField offsetMetadata = metadata.getChild(0);
-//    offsets.load(offsetMetadata, buffer);
-//
-//    final UserBitShared.SerializedField vectorMetadata = metadata.getChild(1);
-//    if (getDataVector() == DEFAULT_DATA_VECTOR) {
-//      addOrGetVector(VectorDescriptor.create(vectorMetadata.getMajorType()));
-//    }
-//
-//    final int offsetLength = offsetMetadata.getBufferLength();
-//    final int vectorLength = vectorMetadata.getBufferLength();
-//    vector.load(vectorMetadata, buffer.slice(offsetLength, vectorLength));
-//  }
-
   /**
    * Returns 1 if inner vector is explicitly set via #addOrGetVector else 0
-   *
-   * @see {@link ContainerVectorLike#size}
    */
-  @Override
   public int size() {
     return vector == DEFAULT_DATA_VECTOR ? 0:1;
   }
 
-  @Override
-  public <T extends ValueVector> AddOrGetResult<T> addOrGetVector(VectorDescriptor descriptor) {
+  public <T extends ValueVector> AddOrGetResult<T> addOrGetVector(MinorType minorType) {
     boolean created = false;
-    if (vector == DEFAULT_DATA_VECTOR && descriptor.getType().getMinorType() != MinorType.LATE) {
-      final MaterializedField field = descriptor.withName(DATA_VECTOR_NAME).getField();
-      vector = BasicTypeHelper.getNewVector(field, allocator);
+    if (vector instanceof ZeroVector) {
+      vector = minorType.getNewVector(DATA_VECTOR_NAME, allocator, null);
       // returned vector must have the same field
-      assert field.equals(vector.getField());
-      getField().addChild(field);
       created = true;
     }
 
-    final MajorType actual = vector.getField().getType();
-    if (!actual.equals(descriptor.getType())) {
+    if (vector.getField().getType().getTypeType() != minorType.getType().getTypeType()) {
       final String msg = String.format("Inner vector type mismatch. Requested type: [%s], actual type: [%s]",
-          descriptor.getType(), actual);
+          Type.name(minorType.getType().getTypeType()), Type.name(vector.getField().getType().getTypeType()));
       throw new SchemaChangeRuntimeException(msg);
     }
 

http://git-wip-us.apache.org/repos/asf/arrow/blob/e7e399db/java/vector/src/main/java/org/apache/arrow/vector/complex/ContainerVectorLike.java
----------------------------------------------------------------------
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/ContainerVectorLike.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/ContainerVectorLike.java
deleted file mode 100644
index 655b55a..0000000
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/ContainerVectorLike.java
+++ /dev/null
@@ -1,43 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.arrow.vector.complex;
-
-import org.apache.arrow.vector.AddOrGetResult;
-import org.apache.arrow.vector.ValueVector;
-import org.apache.arrow.vector.VectorDescriptor;
-
-/**
- * A mix-in used for introducing container vector-like behaviour.
- */
-public interface ContainerVectorLike {
-
-  /**
-   * Creates and adds a child vector if none with the same name exists, else returns the vector instance.
-   *
-   * @param  descriptor vector descriptor
-   * @return  result of operation wrapping vector corresponding to the given descriptor and whether it's newly created
-   * @throws org.apache.arrow.vector.util.SchemaChangeRuntimeException
-   *    if schema change is not permissible between the given and existing data vector types.
-   */
-  <T extends ValueVector> AddOrGetResult<T> addOrGetVector(VectorDescriptor descriptor);
-
-  /**
-   * Returns the number of child vectors in this container vector-like instance.
-   */
-  int size();
-}

http://git-wip-us.apache.org/repos/asf/arrow/blob/e7e399db/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java
----------------------------------------------------------------------
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java
index 3e60c76..c6c6b09 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java
@@ -18,6 +18,8 @@
  ******************************************************************************/
 package org.apache.arrow.vector.complex;
 
+import com.google.common.collect.ImmutableList;
+import com.google.flatbuffers.FlatBufferBuilder;
 import io.netty.buffer.ArrowBuf;
 
 import java.util.List;
@@ -28,17 +30,14 @@ import org.apache.arrow.vector.AddOrGetResult;
 import org.apache.arrow.vector.UInt1Vector;
 import org.apache.arrow.vector.UInt4Vector;
 import org.apache.arrow.vector.ValueVector;
-import org.apache.arrow.vector.VectorDescriptor;
 import org.apache.arrow.vector.ZeroVector;
 import org.apache.arrow.vector.complex.impl.ComplexCopier;
 import org.apache.arrow.vector.complex.impl.UnionListReader;
 import org.apache.arrow.vector.complex.impl.UnionListWriter;
 import org.apache.arrow.vector.complex.reader.FieldReader;
 import org.apache.arrow.vector.complex.writer.FieldWriter;
-import org.apache.arrow.vector.types.MaterializedField;
-import org.apache.arrow.vector.types.Types.DataMode;
-import org.apache.arrow.vector.types.Types.MajorType;
 import org.apache.arrow.vector.types.Types.MinorType;
+import org.apache.arrow.vector.types.pojo.Field;
 import org.apache.arrow.vector.util.CallBack;
 import org.apache.arrow.vector.util.JsonStringArrayList;
 import org.apache.arrow.vector.util.TransferPair;
@@ -55,11 +54,10 @@ public class ListVector extends BaseRepeatedValueVector {
   private UnionListReader reader;
   private CallBack callBack;
 
-  public ListVector(MaterializedField field, BufferAllocator allocator, CallBack callBack) {
-    super(field, allocator);
-    this.bits = new UInt1Vector(MaterializedField.create("$bits$", new MajorType(MinorType.UINT1, DataMode.REQUIRED)), allocator);
+  public ListVector(String name, BufferAllocator allocator, CallBack callBack) {
+    super(name, allocator);
+    this.bits = new UInt1Vector("$bits$", allocator);
     offsets = getOffsetVector();
-    this.field.addChild(getDataVector().getField());
     this.writer = new UnionListWriter(this);
     this.reader = new UnionListReader(this);
     this.callBack = callBack;
@@ -75,15 +73,6 @@ public class ListVector extends BaseRepeatedValueVector {
     bits.allocateNewSafe();
   }
 
-  public void transferTo(ListVector target) {
-    offsets.makeTransferPair(target.offsets).transfer();
-    bits.makeTransferPair(target.bits).transfer();
-    if (target.getDataVector() instanceof ZeroVector) {
-      target.addOrGetVector(new VectorDescriptor(vector.getField().getType()));
-    }
-    getDataVector().makeTransferPair(target.getDataVector()).transfer();
-  }
-
   public void copyFromSafe(int inIndex, int outIndex, ListVector from) {
     copyFrom(inIndex, outIndex, from);
   }
@@ -103,7 +92,7 @@ public class ListVector extends BaseRepeatedValueVector {
 
   @Override
   public TransferPair getTransferPair(String ref, BufferAllocator allocator) {
-    return new TransferImpl(field.withPath(ref), allocator);
+    return new TransferImpl(ref, allocator);
   }
 
   @Override
@@ -114,20 +103,28 @@ public class ListVector extends BaseRepeatedValueVector {
   private class TransferImpl implements TransferPair {
 
     ListVector to;
+    TransferPair pairs[] = new TransferPair[3];
 
-    public TransferImpl(MaterializedField field, BufferAllocator allocator) {
-      to = new ListVector(field, allocator, null);
-      to.addOrGetVector(new VectorDescriptor(vector.getField().getType()));
+    public TransferImpl(String name, BufferAllocator allocator) {
+      this(new ListVector(name, allocator, null));
     }
 
     public TransferImpl(ListVector to) {
       this.to = to;
-      to.addOrGetVector(new VectorDescriptor(vector.getField().getType()));
+      to.addOrGetVector(vector.getMinorType());
+      pairs[0] = offsets.makeTransferPair(to.offsets);
+      pairs[1] = bits.makeTransferPair(to.bits);
+      if (to.getDataVector() instanceof ZeroVector) {
+        to.addOrGetVector(vector.getMinorType());
+      }
+      pairs[2] = getDataVector().makeTransferPair(to.getDataVector());
     }
 
     @Override
     public void transfer() {
-      transferTo(to);
+      for (TransferPair pair : pairs) {
+        pair.transfer();
+      }
     }
 
     @Override
@@ -190,17 +187,8 @@ public class ListVector extends BaseRepeatedValueVector {
     return success;
   }
 
-//  @Override
-//  protected UserBitShared.SerializedField.Builder getMetadataBuilder() {
-//    return getField().getAsBuilder()
-//            .setValueCount(getAccessor().getValueCount())
-//            .setBufferLength(getBufferSize())
-//            .addChild(offsets.getMetadata())
-//            .addChild(bits.getMetadata())
-//            .addChild(vector.getMetadata());
-//  }
-  public <T extends ValueVector> AddOrGetResult<T> addOrGetVector(VectorDescriptor descriptor) {
-    AddOrGetResult<T> result = super.addOrGetVector(descriptor);
+  public <T extends ValueVector> AddOrGetResult<T> addOrGetVector(MinorType minorType) {
+    AddOrGetResult<T> result = super.addOrGetVector(minorType);
     reader = new UnionListReader(this);
     return result;
   }
@@ -214,6 +202,17 @@ public class ListVector extends BaseRepeatedValueVector {
   }
 
   @Override
+  public Field getField() {
+    return new Field(name, true, new org.apache.arrow.vector.types.pojo.ArrowType.List(),
+            ImmutableList.of(getDataVector().getField()));
+  }
+
+  @Override
+  public MinorType getMinorType() {
+    return MinorType.LIST;
+  }
+
+  @Override
   public void clear() {
     offsets.clear();
     vector.clear();
@@ -235,28 +234,8 @@ public class ListVector extends BaseRepeatedValueVector {
     return buffers;
   }
 
-//  @Override
-//  public void load(UserBitShared.SerializedField metadata, DrillBuf buffer) {
-//    final UserBitShared.SerializedField offsetMetadata = metadata.getChild(0);
-//    offsets.load(offsetMetadata, buffer);
-//
-//    final int offsetLength = offsetMetadata.getBufferLength();
-//    final UserBitShared.SerializedField bitMetadata = metadata.getChild(1);
-//    final int bitLength = bitMetadata.getBufferLength();
-//    bits.load(bitMetadata, buffer.slice(offsetLength, bitLength));
-//
-//    final UserBitShared.SerializedField vectorMetadata = metadata.getChild(2);
-//    if (getDataVector() == DEFAULT_DATA_VECTOR) {
-//      addOrGetVector(VectorDescriptor.create(vectorMetadata.getMajorType()));
-//    }
-//
-//    final int vectorLength = vectorMetadata.getBufferLength();
-//    vector.load(vectorMetadata, buffer.slice(offsetLength + bitLength, vectorLength));
-//  }
-
   public UnionVector promoteToUnion() {
-    MaterializedField newField = MaterializedField.create(getField().getPath(), new MajorType(MinorType.UNION, DataMode.OPTIONAL));
-    UnionVector vector = new UnionVector(newField, allocator, null);
+    UnionVector vector = new UnionVector(name, allocator, null);
     replaceDataVector(vector);
     reader = new UnionListReader(this);
     return vector;

http://git-wip-us.apache.org/repos/asf/arrow/blob/e7e399db/java/vector/src/main/java/org/apache/arrow/vector/complex/MapVector.java
----------------------------------------------------------------------
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/MapVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/MapVector.java
index cc0953a..0cb613e 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/MapVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/MapVector.java
@@ -19,8 +19,10 @@ package org.apache.arrow.vector.complex;
 
 import io.netty.buffer.ArrowBuf;
 
+import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Iterator;
+import java.util.List;
 import java.util.Map;
 
 import javax.annotation.Nullable;
@@ -28,14 +30,13 @@ import javax.annotation.Nullable;
 import org.apache.arrow.memory.BufferAllocator;
 import org.apache.arrow.vector.BaseValueVector;
 import org.apache.arrow.vector.ValueVector;
-import org.apache.arrow.vector.complex.RepeatedMapVector.MapSingleCopier;
 import org.apache.arrow.vector.complex.impl.SingleMapReaderImpl;
 import org.apache.arrow.vector.complex.reader.FieldReader;
 import org.apache.arrow.vector.holders.ComplexHolder;
-import org.apache.arrow.vector.types.MaterializedField;
-import org.apache.arrow.vector.types.Types.DataMode;
-import org.apache.arrow.vector.types.Types.MajorType;
 import org.apache.arrow.vector.types.Types.MinorType;
+import org.apache.arrow.vector.types.pojo.ArrowType;
+import org.apache.arrow.vector.types.pojo.ArrowType.Tuple;
+import org.apache.arrow.vector.types.pojo.Field;
 import org.apache.arrow.vector.util.CallBack;
 import org.apache.arrow.vector.util.JsonStringHashMap;
 import org.apache.arrow.vector.util.TransferPair;
@@ -47,19 +48,13 @@ import com.google.common.primitives.Ints;
 public class MapVector extends AbstractMapVector {
   //private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(MapVector.class);
 
-  public final static MajorType TYPE = new MajorType(MinorType.MAP, DataMode.OPTIONAL);
-
   private final SingleMapReaderImpl reader = new SingleMapReaderImpl(MapVector.this);
   private final Accessor accessor = new Accessor();
   private final Mutator mutator = new Mutator();
   int valueCount;
 
-  public MapVector(String path, BufferAllocator allocator, CallBack callBack){
-    this(MaterializedField.create(path, TYPE), allocator, callBack);
-  }
-
-  public MapVector(MaterializedField field, BufferAllocator allocator, CallBack callBack){
-    super(field, allocator, callBack);
+  public MapVector(String name, BufferAllocator allocator, CallBack callBack){
+    super(name, allocator, callBack);
   }
 
   @Override
@@ -69,7 +64,6 @@ public class MapVector extends AbstractMapVector {
   }
 
   transient private MapTransferPair ephPair;
-  transient private MapSingleCopier ephPair2;
 
   public void copyFromSafe(int fromIndex, int thisIndex, MapVector from) {
     if(ephPair == null || ephPair.from != from) {
@@ -78,13 +72,6 @@ public class MapVector extends AbstractMapVector {
     ephPair.copyValueSafe(fromIndex, thisIndex);
   }
 
-  public void copyFromSafe(int fromSubIndex, int thisIndex, RepeatedMapVector from) {
-    if(ephPair2 == null || ephPair2.from != from) {
-      ephPair2 = from.makeSingularCopier(this);
-    }
-    ephPair2.copySafe(fromSubIndex, thisIndex);
-  }
-
   @Override
   protected boolean supportsDirectRead() {
     return true;
@@ -139,7 +126,7 @@ public class MapVector extends AbstractMapVector {
 
   @Override
   public TransferPair getTransferPair(BufferAllocator allocator) {
-    return new MapTransferPair(this, getField().getPath(), allocator);
+    return new MapTransferPair(this, name, allocator);
   }
 
   @Override
@@ -157,8 +144,8 @@ public class MapVector extends AbstractMapVector {
     private final MapVector from;
     private final MapVector to;
 
-    public MapTransferPair(MapVector from, String path, BufferAllocator allocator) {
-      this(from, new MapVector(MaterializedField.create(path, TYPE), allocator, from.callBack), false);
+    public MapTransferPair(MapVector from, String name, BufferAllocator allocator) {
+      this(from, new MapVector(name, allocator, from.callBack), false);
     }
 
     public MapTransferPair(MapVector from, MapVector to) {
@@ -170,7 +157,6 @@ public class MapVector extends AbstractMapVector {
       this.to = to;
       this.pairs = new TransferPair[from.size()];
       this.to.ephPair = null;
-      this.to.ephPair2 = null;
 
       int i = 0;
       ValueVector vector;
@@ -189,7 +175,7 @@ public class MapVector extends AbstractMapVector {
         // (This is similar to what happens in ScanBatch where the children cannot be added till they are
         // read). To take care of this, we ensure that the hashCode of the MaterializedField does not
         // include the hashCode of the children but is based only on MaterializedField$key.
-        final ValueVector newVector = to.addOrGet(child, vector.getField().getType(), vector.getClass());
+        final ValueVector newVector = to.addOrGet(child, vector.getMinorType(), vector.getClass());
         if (allocate && to.size() != preSize) {
           newVector.allocateNew();
         }
@@ -251,46 +237,6 @@ public class MapVector extends AbstractMapVector {
     return accessor;
   }
 
-//  @Override
-//  public void load(SerializedField metadata, DrillBuf buf) {
-//    final List<SerializedField> fields = metadata.getChildList();
-//    valueCount = metadata.getValueCount();
-//
-//    int bufOffset = 0;
-//    for (final SerializedField child : fields) {
-//      final MaterializedField fieldDef = SerializedFieldHelper.create(child);
-//
-//      ValueVector vector = getChild(fieldDef.getLastName());
-//      if (vector == null) {
-//         if we arrive here, we didn't have a matching vector.
-//        vector = BasicTypeHelper.getNewVector(fieldDef, allocator);
-//        putChild(fieldDef.getLastName(), vector);
-//      }
-//      if (child.getValueCount() == 0) {
-//        vector.clear();
-//      } else {
-//        vector.load(child, buf.slice(bufOffset, child.getBufferLength()));
-//      }
-//      bufOffset += child.getBufferLength();
-//    }
-//
-//    assert bufOffset == buf.capacity();
-//  }
-//
-//  @Override
-//  public SerializedField getMetadata() {
-//    SerializedField.Builder b = getField() //
-//        .getAsBuilder() //
-//        .setBufferLength(getBufferSize()) //
-//        .setValueCount(valueCount);
-//
-//
-//    for(ValueVector v : getChildren()) {
-//      b.addChild(v.getMetadata());
-//    }
-//    return b.build();
-//  }
-
   @Override
   public Mutator getMutator() {
     return mutator;
@@ -303,13 +249,6 @@ public class MapVector extends AbstractMapVector {
       Map<String, Object> vv = new JsonStringHashMap<>();
       for (String child:getChildFieldNames()) {
         ValueVector v = getChild(child);
-        // TODO(DRILL-4001):  Resolve this hack:
-        // The index/value count check in the following if statement is a hack
-        // to work around the current fact that RecordBatchLoader.load and
-        // MapVector.load leave child vectors with a length of zero (as opposed
-        // to matching the lengths of siblings and the parent map vector)
-        // because they don't remove (or set the lengths of) vectors from
-        // previous batches that aren't in the current batch.
         if (v != null && index < v.getAccessor().getValueCount()) {
           Object value = v.getAccessor().getObject(index);
           if (value != null) {
@@ -361,6 +300,20 @@ public class MapVector extends AbstractMapVector {
   }
 
   @Override
+  public Field getField() {
+    List<Field> children = new ArrayList<>();
+    for (ValueVector child : getChildren()) {
+      children.add(child.getField());
+    }
+    return new Field(name, false, Tuple.INSTANCE, children);
+  }
+
+  @Override
+  public MinorType getMinorType() {
+    return MinorType.MAP;
+  }
+
+  @Override
   public void close() {
     final Collection<ValueVector> vectors = getChildren();
     for (final ValueVector v : vectors) {

http://git-wip-us.apache.org/repos/asf/arrow/blob/e7e399db/java/vector/src/main/java/org/apache/arrow/vector/complex/RepeatedListVector.java
----------------------------------------------------------------------
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/RepeatedListVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/RepeatedListVector.java
deleted file mode 100644
index f337f9c..0000000
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/RepeatedListVector.java
+++ /dev/null
@@ -1,427 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.arrow.vector.complex;
-
-import io.netty.buffer.ArrowBuf;
-
-import java.util.Iterator;
-import java.util.List;
-
-import org.apache.arrow.memory.BufferAllocator;
-import org.apache.arrow.memory.OutOfMemoryException;
-import org.apache.arrow.vector.AddOrGetResult;
-import org.apache.arrow.vector.UInt4Vector;
-import org.apache.arrow.vector.ValueVector;
-import org.apache.arrow.vector.VectorDescriptor;
-import org.apache.arrow.vector.complex.impl.NullReader;
-import org.apache.arrow.vector.complex.impl.RepeatedListReaderImpl;
-import org.apache.arrow.vector.complex.reader.FieldReader;
-import org.apache.arrow.vector.holders.ComplexHolder;
-import org.apache.arrow.vector.holders.RepeatedListHolder;
-import org.apache.arrow.vector.types.MaterializedField;
-import org.apache.arrow.vector.types.Types.DataMode;
-import org.apache.arrow.vector.types.Types.MajorType;
-import org.apache.arrow.vector.types.Types.MinorType;
-import org.apache.arrow.vector.util.CallBack;
-import org.apache.arrow.vector.util.JsonStringArrayList;
-import org.apache.arrow.vector.util.TransferPair;
-
-import com.google.common.base.Preconditions;
-import com.google.common.collect.Lists;
-
-public class RepeatedListVector extends AbstractContainerVector
-    implements RepeatedValueVector, RepeatedFixedWidthVectorLike {
-
-  public final static MajorType TYPE = new MajorType(MinorType.LIST, DataMode.REPEATED);
-  private final RepeatedListReaderImpl reader = new RepeatedListReaderImpl(null, this);
-  final DelegateRepeatedVector delegate;
-
-  protected static class DelegateRepeatedVector extends BaseRepeatedValueVector {
-
-    private final RepeatedListAccessor accessor = new RepeatedListAccessor();
-    private final RepeatedListMutator mutator = new RepeatedListMutator();
-    private final EmptyValuePopulator emptyPopulator;
-    private transient DelegateTransferPair ephPair;
-
-    public class RepeatedListAccessor extends BaseRepeatedValueVector.BaseRepeatedAccessor {
-
-      @Override
-      public Object getObject(int index) {
-        final List<Object> list = new JsonStringArrayList<>();
-        final int start = offsets.getAccessor().get(index);
-        final int until = offsets.getAccessor().get(index+1);
-        for (int i = start; i < until; i++) {
-          list.add(vector.getAccessor().getObject(i));
-        }
-        return list;
-      }
-
-      public void get(int index, RepeatedListHolder holder) {
-        assert index <= getValueCapacity();
-        holder.start = getOffsetVector().getAccessor().get(index);
-        holder.end = getOffsetVector().getAccessor().get(index+1);
-      }
-
-      public void get(int index, ComplexHolder holder) {
-        final FieldReader reader = getReader();
-        reader.setPosition(index);
-        holder.reader = reader;
-      }
-
-      public void get(int index, int arrayIndex, ComplexHolder holder) {
-        final RepeatedListHolder listHolder = new RepeatedListHolder();
-        get(index, listHolder);
-        int offset = listHolder.start + arrayIndex;
-        if (offset >= listHolder.end) {
-          holder.reader = NullReader.INSTANCE;
-        } else {
-          FieldReader r = getDataVector().getReader();
-          r.setPosition(offset);
-          holder.reader = r;
-        }
-      }
-    }
-
-    public class RepeatedListMutator extends BaseRepeatedValueVector.BaseRepeatedMutator {
-
-      public int add(int index) {
-        final int curEnd = getOffsetVector().getAccessor().get(index+1);
-        getOffsetVector().getMutator().setSafe(index + 1, curEnd + 1);
-        return curEnd;
-      }
-
-      @Override
-      public void startNewValue(int index) {
-        emptyPopulator.populate(index+1);
-        super.startNewValue(index);
-      }
-
-      @Override
-      public void setValueCount(int valueCount) {
-        emptyPopulator.populate(valueCount);
-        super.setValueCount(valueCount);
-      }
-    }
-
-
-    public class DelegateTransferPair implements TransferPair {
-      private final DelegateRepeatedVector target;
-      private final TransferPair[] children;
-
-      public DelegateTransferPair(DelegateRepeatedVector target) {
-        this.target = Preconditions.checkNotNull(target);
-        if (target.getDataVector() == DEFAULT_DATA_VECTOR) {
-          target.addOrGetVector(VectorDescriptor.create(getDataVector().getField()));
-          target.getDataVector().allocateNew();
-        }
-        this.children = new TransferPair[] {
-            getOffsetVector().makeTransferPair(target.getOffsetVector()),
-            getDataVector().makeTransferPair(target.getDataVector())
-        };
-      }
-
-      @Override
-      public void transfer() {
-        for (TransferPair child:children) {
-          child.transfer();
-        }
-      }
-
-      @Override
-      public ValueVector getTo() {
-        return target;
-      }
-
-      @Override
-      public void splitAndTransfer(int startIndex, int length) {
-        target.allocateNew();
-        for (int i = 0; i < length; i++) {
-          copyValueSafe(startIndex + i, i);
-        }
-      }
-
-      @Override
-      public void copyValueSafe(int srcIndex, int destIndex) {
-        final RepeatedListHolder holder = new RepeatedListHolder();
-        getAccessor().get(srcIndex, holder);
-        target.emptyPopulator.populate(destIndex+1);
-        final TransferPair vectorTransfer = children[1];
-        int newIndex = target.getOffsetVector().getAccessor().get(destIndex);
-        //todo: make this a bulk copy.
-        for (int i = holder.start; i < holder.end; i++, newIndex++) {
-          vectorTransfer.copyValueSafe(i, newIndex);
-        }
-        target.getOffsetVector().getMutator().setSafe(destIndex + 1, newIndex);
-      }
-    }
-
-    public DelegateRepeatedVector(String path, BufferAllocator allocator) {
-      this(MaterializedField.create(path, TYPE), allocator);
-    }
-
-    public DelegateRepeatedVector(MaterializedField field, BufferAllocator allocator) {
-      super(field, allocator);
-      emptyPopulator = new EmptyValuePopulator(getOffsetVector());
-    }
-
-    @Override
-    public void allocateNew() throws OutOfMemoryException {
-      if (!allocateNewSafe()) {
-        throw new OutOfMemoryException();
-      }
-    }
-
-    @Override
-    public TransferPair getTransferPair(String ref, BufferAllocator allocator) {
-      return makeTransferPair(new DelegateRepeatedVector(ref, allocator));
-    }
-
-    @Override
-    public TransferPair makeTransferPair(ValueVector target) {
-      return new DelegateTransferPair(DelegateRepeatedVector.class.cast(target));
-    }
-
-    @Override
-    public RepeatedListAccessor getAccessor() {
-      return accessor;
-    }
-
-    @Override
-    public RepeatedListMutator getMutator() {
-      return mutator;
-    }
-
-    @Override
-    public FieldReader getReader() {
-      throw new UnsupportedOperationException();
-    }
-
-    public void copyFromSafe(int fromIndex, int thisIndex, DelegateRepeatedVector from) {
-      if(ephPair == null || ephPair.target != from) {
-        ephPair = DelegateTransferPair.class.cast(from.makeTransferPair(this));
-      }
-      ephPair.copyValueSafe(fromIndex, thisIndex);
-    }
-
-  }
-
-  protected class RepeatedListTransferPair implements TransferPair {
-    private final TransferPair delegate;
-
-    public RepeatedListTransferPair(TransferPair delegate) {
-      this.delegate = delegate;
-    }
-
-    public void transfer() {
-      delegate.transfer();
-    }
-
-    @Override
-    public void splitAndTransfer(int startIndex, int length) {
-      delegate.splitAndTransfer(startIndex, length);
-    }
-
-    @Override
-    public ValueVector getTo() {
-      final DelegateRepeatedVector delegateVector = DelegateRepeatedVector.class.cast(delegate.getTo());
-      return new RepeatedListVector(getField(), allocator, callBack, delegateVector);
-    }
-
-    @Override
-    public void copyValueSafe(int from, int to) {
-      delegate.copyValueSafe(from, to);
-    }
-  }
-
-  public RepeatedListVector(String path, BufferAllocator allocator, CallBack callBack) {
-    this(MaterializedField.create(path, TYPE), allocator, callBack);
-  }
-
-  public RepeatedListVector(MaterializedField field, BufferAllocator allocator, CallBack callBack) {
-    this(field, allocator, callBack, new DelegateRepeatedVector(field, allocator));
-  }
-
-  protected RepeatedListVector(MaterializedField field, BufferAllocator allocator, CallBack callBack, DelegateRepeatedVector delegate) {
-    super(field, allocator, callBack);
-    this.delegate = Preconditions.checkNotNull(delegate);
-
-    final List<MaterializedField> children = Lists.newArrayList(field.getChildren());
-    final int childSize = children.size();
-    assert childSize < 3;
-    final boolean hasChild = childSize > 0;
-    if (hasChild) {
-      // the last field is data field
-      final MaterializedField child = children.get(childSize-1);
-      addOrGetVector(VectorDescriptor.create(child));
-    }
-  }
-
-
-    @Override
-  public RepeatedListReaderImpl getReader() {
-    return reader;
-  }
-
-  @Override
-  public DelegateRepeatedVector.RepeatedListAccessor getAccessor() {
-    return delegate.getAccessor();
-  }
-
-  @Override
-  public DelegateRepeatedVector.RepeatedListMutator getMutator() {
-    return delegate.getMutator();
-  }
-
-  @Override
-  public UInt4Vector getOffsetVector() {
-    return delegate.getOffsetVector();
-  }
-
-  @Override
-  public ValueVector getDataVector() {
-    return delegate.getDataVector();
-  }
-
-  @Override
-  public void allocateNew() throws OutOfMemoryException {
-    delegate.allocateNew();
-  }
-
-  @Override
-  public boolean allocateNewSafe() {
-    return delegate.allocateNewSafe();
-  }
-
-  @Override
-  public <T extends ValueVector> AddOrGetResult<T> addOrGetVector(VectorDescriptor descriptor) {
-    final AddOrGetResult<T> result = delegate.addOrGetVector(descriptor);
-    if (result.isCreated() && callBack != null) {
-      callBack.doWork();
-    }
-    return result;
-  }
-
-  @Override
-  public int size() {
-    return delegate.size();
-  }
-
-  @Override
-  public int getBufferSize() {
-    return delegate.getBufferSize();
-  }
-
-  @Override
-  public int getBufferSizeFor(final int valueCount) {
-    return delegate.getBufferSizeFor(valueCount);
-  }
-
-  @Override
-  public void close() {
-    delegate.close();
-  }
-
-  @Override
-  public void clear() {
-    delegate.clear();
-  }
-
-  @Override
-  public TransferPair getTransferPair(BufferAllocator allocator) {
-    return new RepeatedListTransferPair(delegate.getTransferPair(allocator));
-  }
-
-  @Override
-  public TransferPair getTransferPair(String ref, BufferAllocator allocator) {
-    return new RepeatedListTransferPair(delegate.getTransferPair(ref, allocator));
-  }
-
-  @Override
-  public TransferPair makeTransferPair(ValueVector to) {
-    final RepeatedListVector target = RepeatedListVector.class.cast(to);
-    return new RepeatedListTransferPair(delegate.makeTransferPair(target.delegate));
-  }
-
-  @Override
-  public int getValueCapacity() {
-    return delegate.getValueCapacity();
-  }
-
-  @Override
-  public ArrowBuf[] getBuffers(boolean clear) {
-    return delegate.getBuffers(clear);
-  }
-
-
-//  @Override
-//  public void load(SerializedField metadata, DrillBuf buf) {
-//    delegate.load(metadata, buf);
-//  }
-
-//  @Override
-//  public SerializedField getMetadata() {
-//    return delegate.getMetadata();
-//  }
-
-  @Override
-  public Iterator<ValueVector> iterator() {
-    return delegate.iterator();
-  }
-
-  @Override
-  public void setInitialCapacity(int numRecords) {
-    delegate.setInitialCapacity(numRecords);
-  }
-
-  /**
-   * @deprecated
-   *   prefer using {@link #addOrGetVector(org.apache.arrow.vector.VectorDescriptor)} instead.
-   */
-  @Override
-  public <T extends ValueVector> T addOrGet(String name, MajorType type, Class<T> clazz) {
-    final AddOrGetResult<T> result = addOrGetVector(VectorDescriptor.create(type));
-    return result.getVector();
-  }
-
-  @Override
-  public <T extends ValueVector> T getChild(String name, Class<T> clazz) {
-    if (name != null) {
-      return null;
-    }
-    return typeify(delegate.getDataVector(), clazz);
-  }
-
-  @Override
-  public void allocateNew(int valueCount, int innerValueCount) {
-    clear();
-    getOffsetVector().allocateNew(valueCount + 1);
-    getMutator().reset();
-  }
-
-  @Override
-  public VectorWithOrdinal getChildVectorWithOrdinal(String name) {
-    if (name != null) {
-      return null;
-    }
-    return new VectorWithOrdinal(delegate.getDataVector(), 0);
-  }
-
-  public void copyFromSafe(int fromIndex, int thisIndex, RepeatedListVector from) {
-    delegate.copyFromSafe(fromIndex, thisIndex, from.delegate);
-  }
-}

http://git-wip-us.apache.org/repos/asf/arrow/blob/e7e399db/java/vector/src/main/java/org/apache/arrow/vector/complex/RepeatedMapVector.java
----------------------------------------------------------------------
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/RepeatedMapVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/RepeatedMapVector.java
deleted file mode 100644
index 686414e..0000000
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/RepeatedMapVector.java
+++ /dev/null
@@ -1,584 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.arrow.vector.complex;
-
-import io.netty.buffer.ArrowBuf;
-
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-
-import org.apache.arrow.memory.BufferAllocator;
-import org.apache.arrow.memory.OutOfMemoryException;
-import org.apache.arrow.vector.AddOrGetResult;
-import org.apache.arrow.vector.AllocationHelper;
-import org.apache.arrow.vector.UInt4Vector;
-import org.apache.arrow.vector.ValueVector;
-import org.apache.arrow.vector.VectorDescriptor;
-import org.apache.arrow.vector.complex.impl.NullReader;
-import org.apache.arrow.vector.complex.impl.RepeatedMapReaderImpl;
-import org.apache.arrow.vector.complex.reader.FieldReader;
-import org.apache.arrow.vector.holders.ComplexHolder;
-import org.apache.arrow.vector.holders.RepeatedMapHolder;
-import org.apache.arrow.vector.types.MaterializedField;
-import org.apache.arrow.vector.types.Types.DataMode;
-import org.apache.arrow.vector.types.Types.MajorType;
-import org.apache.arrow.vector.types.Types.MinorType;
-import org.apache.arrow.vector.util.CallBack;
-import org.apache.arrow.vector.util.JsonStringArrayList;
-import org.apache.arrow.vector.util.TransferPair;
-import org.apache.commons.lang3.ArrayUtils;
-
-import com.google.common.base.Preconditions;
-import com.google.common.collect.Maps;
-
-public class RepeatedMapVector extends AbstractMapVector
-    implements RepeatedValueVector, RepeatedFixedWidthVectorLike {
-  //private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(RepeatedMapVector.class);
-
-  public final static MajorType TYPE = new MajorType(MinorType.MAP, DataMode.REPEATED);
-
-  final UInt4Vector offsets;   // offsets to start of each record (considering record indices are 0-indexed)
-  private final RepeatedMapReaderImpl reader = new RepeatedMapReaderImpl(RepeatedMapVector.this);
-  private final RepeatedMapAccessor accessor = new RepeatedMapAccessor();
-  private final Mutator mutator = new Mutator();
-  private final EmptyValuePopulator emptyPopulator;
-
-  public RepeatedMapVector(MaterializedField field, BufferAllocator allocator, CallBack callBack){
-    super(field, allocator, callBack);
-    this.offsets = new UInt4Vector(BaseRepeatedValueVector.OFFSETS_FIELD, allocator);
-    this.emptyPopulator = new EmptyValuePopulator(offsets);
-  }
-
-  @Override
-  public UInt4Vector getOffsetVector() {
-    return offsets;
-  }
-
-  @Override
-  public ValueVector getDataVector() {
-    throw new UnsupportedOperationException();
-  }
-
-  @Override
-  public <T extends ValueVector> AddOrGetResult<T> addOrGetVector(VectorDescriptor descriptor) {
-    throw new UnsupportedOperationException();
-  }
-
-  @Override
-  public void setInitialCapacity(int numRecords) {
-    offsets.setInitialCapacity(numRecords + 1);
-    for(final ValueVector v : (Iterable<ValueVector>) this) {
-      v.setInitialCapacity(numRecords * RepeatedValueVector.DEFAULT_REPEAT_PER_RECORD);
-    }
-  }
-
-  @Override
-  public RepeatedMapReaderImpl getReader() {
-    return reader;
-  }
-
-  @Override
-  public void allocateNew(int groupCount, int innerValueCount) {
-    clear();
-    try {
-      offsets.allocateNew(groupCount + 1);
-      for (ValueVector v : getChildren()) {
-        AllocationHelper.allocatePrecomputedChildCount(v, groupCount, 50, innerValueCount);
-      }
-    } catch (OutOfMemoryException e){
-      clear();
-      throw e;
-    }
-    offsets.zeroVector();
-    mutator.reset();
-  }
-
-  public Iterator<String> fieldNameIterator() {
-    return getChildFieldNames().iterator();
-  }
-
-  @Override
-  public List<ValueVector> getPrimitiveVectors() {
-    final List<ValueVector> primitiveVectors = super.getPrimitiveVectors();
-    primitiveVectors.add(offsets);
-    return primitiveVectors;
-  }
-
-  @Override
-  public int getBufferSize() {
-    if (getAccessor().getValueCount() == 0) {
-      return 0;
-    }
-    long bufferSize = offsets.getBufferSize();
-    for (final ValueVector v : (Iterable<ValueVector>) this) {
-      bufferSize += v.getBufferSize();
-    }
-    return (int) bufferSize;
-  }
-
-  @Override
-  public int getBufferSizeFor(final int valueCount) {
-    if (valueCount == 0) {
-      return 0;
-    }
-
-    long bufferSize = 0;
-    for (final ValueVector v : (Iterable<ValueVector>) this) {
-      bufferSize += v.getBufferSizeFor(valueCount);
-    }
-
-    return (int) bufferSize;
-  }
-
-  @Override
-  public void close() {
-    offsets.close();
-    super.close();
-  }
-
-  @Override
-  public TransferPair getTransferPair(BufferAllocator allocator) {
-    return new RepeatedMapTransferPair(this, getField().getPath(), allocator);
-  }
-
-  @Override
-  public TransferPair makeTransferPair(ValueVector to) {
-    return new RepeatedMapTransferPair(this, (RepeatedMapVector)to);
-  }
-
-  MapSingleCopier makeSingularCopier(MapVector to) {
-    return new MapSingleCopier(this, to);
-  }
-
-  protected static class MapSingleCopier {
-    private final TransferPair[] pairs;
-    public final RepeatedMapVector from;
-
-    public MapSingleCopier(RepeatedMapVector from, MapVector to) {
-      this.from = from;
-      this.pairs = new TransferPair[from.size()];
-
-      int i = 0;
-      ValueVector vector;
-      for (final String child:from.getChildFieldNames()) {
-        int preSize = to.size();
-        vector = from.getChild(child);
-        if (vector == null) {
-          continue;
-        }
-        final ValueVector newVector = to.addOrGet(child, vector.getField().getType(), vector.getClass());
-        if (to.size() != preSize) {
-          newVector.allocateNew();
-        }
-        pairs[i++] = vector.makeTransferPair(newVector);
-      }
-    }
-
-    public void copySafe(int fromSubIndex, int toIndex) {
-      for (TransferPair p : pairs) {
-        p.copyValueSafe(fromSubIndex, toIndex);
-      }
-    }
-  }
-
-  public TransferPair getTransferPairToSingleMap(String reference, BufferAllocator allocator) {
-    return new SingleMapTransferPair(this, reference, allocator);
-  }
-
-  @Override
-  public TransferPair getTransferPair(String ref, BufferAllocator allocator) {
-    return new RepeatedMapTransferPair(this, ref, allocator);
-  }
-
-  @Override
-  public boolean allocateNewSafe() {
-    /* boolean to keep track if all the memory allocation were successful
-     * Used in the case of composite vectors when we need to allocate multiple
-     * buffers for multiple vectors. If one of the allocations failed we need to
-     * clear all the memory that we allocated
-     */
-    boolean success = false;
-    try {
-      if (!offsets.allocateNewSafe()) {
-        return false;
-      }
-      success =  super.allocateNewSafe();
-    } finally {
-      if (!success) {
-        clear();
-      }
-    }
-    offsets.zeroVector();
-    return success;
-  }
-
-  protected static class SingleMapTransferPair implements TransferPair {
-    private final TransferPair[] pairs;
-    private final RepeatedMapVector from;
-    private final MapVector to;
-    private static final MajorType MAP_TYPE = new MajorType(MinorType.MAP, DataMode.REQUIRED);
-
-    public SingleMapTransferPair(RepeatedMapVector from, String path, BufferAllocator allocator) {
-      this(from, new MapVector(MaterializedField.create(path, MAP_TYPE), allocator, from.callBack), false);
-    }
-
-    public SingleMapTransferPair(RepeatedMapVector from, MapVector to) {
-      this(from, to, true);
-    }
-
-    public SingleMapTransferPair(RepeatedMapVector from, MapVector to, boolean allocate) {
-      this.from = from;
-      this.to = to;
-      this.pairs = new TransferPair[from.size()];
-      int i = 0;
-      ValueVector vector;
-      for (final String child : from.getChildFieldNames()) {
-        int preSize = to.size();
-        vector = from.getChild(child);
-        if (vector == null) {
-          continue;
-        }
-        final ValueVector newVector = to.addOrGet(child, vector.getField().getType(), vector.getClass());
-        if (allocate && to.size() != preSize) {
-          newVector.allocateNew();
-        }
-        pairs[i++] = vector.makeTransferPair(newVector);
-      }
-    }
-
-
-    @Override
-    public void transfer() {
-      for (TransferPair p : pairs) {
-        p.transfer();
-      }
-      to.getMutator().setValueCount(from.getAccessor().getValueCount());
-      from.clear();
-    }
-
-    @Override
-    public ValueVector getTo() {
-      return to;
-    }
-
-    @Override
-    public void copyValueSafe(int from, int to) {
-      for (TransferPair p : pairs) {
-        p.copyValueSafe(from, to);
-      }
-    }
-
-    @Override
-    public void splitAndTransfer(int startIndex, int length) {
-      for (TransferPair p : pairs) {
-        p.splitAndTransfer(startIndex, length);
-      }
-      to.getMutator().setValueCount(length);
-    }
-  }
-
-  private static class RepeatedMapTransferPair implements TransferPair{
-
-    private final TransferPair[] pairs;
-    private final RepeatedMapVector to;
-    private final RepeatedMapVector from;
-
-    public RepeatedMapTransferPair(RepeatedMapVector from, String path, BufferAllocator allocator) {
-      this(from, new RepeatedMapVector(MaterializedField.create(path, TYPE), allocator, from.callBack), false);
-    }
-
-    public RepeatedMapTransferPair(RepeatedMapVector from, RepeatedMapVector to) {
-      this(from, to, true);
-    }
-
-    public RepeatedMapTransferPair(RepeatedMapVector from, RepeatedMapVector to, boolean allocate) {
-      this.from = from;
-      this.to = to;
-      this.pairs = new TransferPair[from.size()];
-      this.to.ephPair = null;
-
-      int i = 0;
-      ValueVector vector;
-      for (final String child : from.getChildFieldNames()) {
-        final int preSize = to.size();
-        vector = from.getChild(child);
-        if (vector == null) {
-          continue;
-        }
-
-        final ValueVector newVector = to.addOrGet(child, vector.getField().getType(), vector.getClass());
-        if (to.size() != preSize) {
-          newVector.allocateNew();
-        }
-
-        pairs[i++] = vector.makeTransferPair(newVector);
-      }
-    }
-
-    @Override
-    public void transfer() {
-      from.offsets.transferTo(to.offsets);
-      for (TransferPair p : pairs) {
-        p.transfer();
-      }
-      from.clear();
-    }
-
-    @Override
-    public ValueVector getTo() {
-      return to;
-    }
-
-    @Override
-    public void copyValueSafe(int srcIndex, int destIndex) {
-      RepeatedMapHolder holder = new RepeatedMapHolder();
-      from.getAccessor().get(srcIndex, holder);
-      to.emptyPopulator.populate(destIndex + 1);
-      int newIndex = to.offsets.getAccessor().get(destIndex);
-      //todo: make these bulk copies
-      for (int i = holder.start; i < holder.end; i++, newIndex++) {
-        for (TransferPair p : pairs) {
-          p.copyValueSafe(i, newIndex);
-        }
-      }
-      to.offsets.getMutator().setSafe(destIndex + 1, newIndex);
-    }
-
-    @Override
-    public void splitAndTransfer(final int groupStart, final int groups) {
-      final UInt4Vector.Accessor a = from.offsets.getAccessor();
-      final UInt4Vector.Mutator m = to.offsets.getMutator();
-
-      final int startPos = a.get(groupStart);
-      final int endPos = a.get(groupStart + groups);
-      final int valuesToCopy = endPos - startPos;
-
-      to.offsets.clear();
-      to.offsets.allocateNew(groups + 1);
-
-      int normalizedPos;
-      for (int i = 0; i < groups + 1; i++) {
-        normalizedPos = a.get(groupStart + i) - startPos;
-        m.set(i, normalizedPos);
-      }
-
-      m.setValueCount(groups + 1);
-      to.emptyPopulator.populate(groups);
-
-      for (final TransferPair p : pairs) {
-        p.splitAndTransfer(startPos, valuesToCopy);
-      }
-    }
-  }
-
-
-  transient private RepeatedMapTransferPair ephPair;
-
-  public void copyFromSafe(int fromIndex, int thisIndex, RepeatedMapVector from) {
-    if (ephPair == null || ephPair.from != from) {
-      ephPair = (RepeatedMapTransferPair) from.makeTransferPair(this);
-    }
-    ephPair.copyValueSafe(fromIndex, thisIndex);
-  }
-
-  @Override
-  public int getValueCapacity() {
-    return Math.max(offsets.getValueCapacity() - 1, 0);
-  }
-
-  @Override
-  public RepeatedMapAccessor getAccessor() {
-    return accessor;
-  }
-
-  @Override
-  public ArrowBuf[] getBuffers(boolean clear) {
-    final int expectedBufferSize = getBufferSize();
-    final int actualBufferSize = super.getBufferSize();
-
-    Preconditions.checkArgument(expectedBufferSize == actualBufferSize + offsets.getBufferSize());
-    return ArrayUtils.addAll(offsets.getBuffers(clear), super.getBuffers(clear));
-  }
-
-
-//  @Override
-//  public void load(SerializedField metadata, DrillBuf buffer) {
-//    final List<SerializedField> children = metadata.getChildList();
-//
-//    final SerializedField offsetField = children.get(0);
-//    offsets.load(offsetField, buffer);
-//    int bufOffset = offsetField.getBufferLength();
-//
-//    for (int i = 1; i < children.size(); i++) {
-//      final SerializedField child = children.get(i);
-//      final MaterializedField fieldDef = SerializedFieldHelper.create(child);
-//      ValueVector vector = getChild(fieldDef.getLastName());
-//      if (vector == null) {
-        // if we arrive here, we didn't have a matching vector.
-//        vector = BasicTypeHelper.getNewVector(fieldDef, allocator);
-//        putChild(fieldDef.getLastName(), vector);
-//      }
-//      final int vectorLength = child.getBufferLength();
-//      vector.load(child, buffer.slice(bufOffset, vectorLength));
-//      bufOffset += vectorLength;
-//    }
-//
-//    assert bufOffset == buffer.capacity();
-//  }
-//
-//
-//  @Override
-//  public SerializedField getMetadata() {
-//    SerializedField.Builder builder = getField() //
-//        .getAsBuilder() //
-//        .setBufferLength(getBufferSize()) //
-        // while we don't need to actually read this on load, we need it to make sure we don't skip deserialization of this vector
-//        .setValueCount(accessor.getValueCount());
-//    builder.addChild(offsets.getMetadata());
-//    for (final ValueVector child : getChildren()) {
-//      builder.addChild(child.getMetadata());
-//    }
-//    return builder.build();
-//  }
-
-  @Override
-  public Mutator getMutator() {
-    return mutator;
-  }
-
-  public class RepeatedMapAccessor implements RepeatedAccessor {
-    @Override
-    public Object getObject(int index) {
-      final List<Object> list = new JsonStringArrayList<>();
-      final int end = offsets.getAccessor().get(index+1);
-      String fieldName;
-      for (int i =  offsets.getAccessor().get(index); i < end; i++) {
-        final Map<String, Object> vv = Maps.newLinkedHashMap();
-        for (final MaterializedField field : getField().getChildren()) {
-          if (!field.equals(BaseRepeatedValueVector.OFFSETS_FIELD)) {
-            fieldName = field.getLastName();
-            final Object value = getChild(fieldName).getAccessor().getObject(i);
-            if (value != null) {
-              vv.put(fieldName, value);
-            }
-          }
-        }
-        list.add(vv);
-      }
-      return list;
-    }
-
-    @Override
-    public int getValueCount() {
-      return Math.max(offsets.getAccessor().getValueCount() - 1, 0);
-    }
-
-    @Override
-    public int getInnerValueCount() {
-      final int valueCount = getValueCount();
-      if (valueCount == 0) {
-        return 0;
-      }
-      return offsets.getAccessor().get(valueCount);
-    }
-
-    @Override
-    public int getInnerValueCountAt(int index) {
-      return offsets.getAccessor().get(index+1) - offsets.getAccessor().get(index);
-    }
-
-    @Override
-    public boolean isEmpty(int index) {
-      return false;
-    }
-
-    @Override
-    public boolean isNull(int index) {
-      return false;
-    }
-
-    public void get(int index, RepeatedMapHolder holder) {
-      assert index < getValueCapacity() :
-        String.format("Attempted to access index %d when value capacity is %d",
-            index, getValueCapacity());
-      final UInt4Vector.Accessor offsetsAccessor = offsets.getAccessor();
-      holder.start = offsetsAccessor.get(index);
-      holder.end = offsetsAccessor.get(index + 1);
-    }
-
-    public void get(int index, ComplexHolder holder) {
-      final FieldReader reader = getReader();
-      reader.setPosition(index);
-      holder.reader = reader;
-    }
-
-    public void get(int index, int arrayIndex, ComplexHolder holder) {
-      final RepeatedMapHolder h = new RepeatedMapHolder();
-      get(index, h);
-      final int offset = h.start + arrayIndex;
-
-      if (offset >= h.end) {
-        holder.reader = NullReader.INSTANCE;
-      } else {
-        reader.setSinglePosition(index, arrayIndex);
-        holder.reader = reader;
-      }
-    }
-  }
-
-  public class Mutator implements RepeatedMutator {
-    @Override
-    public void startNewValue(int index) {
-      emptyPopulator.populate(index + 1);
-      offsets.getMutator().setSafe(index + 1, offsets.getAccessor().get(index));
-    }
-
-    @Override
-    public void setValueCount(int topLevelValueCount) {
-      emptyPopulator.populate(topLevelValueCount);
-      offsets.getMutator().setValueCount(topLevelValueCount == 0 ? 0 : topLevelValueCount + 1);
-      int childValueCount = offsets.getAccessor().get(topLevelValueCount);
-      for (final ValueVector v : getChildren()) {
-        v.getMutator().setValueCount(childValueCount);
-      }
-    }
-
-    @Override
-    public void reset() {}
-
-    @Override
-    public void generateTestData(int values) {}
-
-    public int add(int index) {
-      final int prevEnd = offsets.getAccessor().get(index + 1);
-      offsets.getMutator().setSafe(index + 1, prevEnd + 1);
-      return prevEnd;
-    }
-  }
-
-  @Override
-  public void clear() {
-    getMutator().reset();
-
-    offsets.clear();
-    for(final ValueVector vector : getChildren()) {
-      vector.clear();
-    }
-  }
-}

http://git-wip-us.apache.org/repos/asf/arrow/blob/e7e399db/java/vector/src/main/java/org/apache/arrow/vector/complex/RepeatedValueVector.java
----------------------------------------------------------------------
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/RepeatedValueVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/RepeatedValueVector.java
index 99c0a0a..54db393 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/RepeatedValueVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/RepeatedValueVector.java
@@ -28,7 +28,7 @@ import org.apache.arrow.vector.ValueVector;
  * uses the offset vector to determine the sequence of cells pertaining to an individual value.
  *
  */
-public interface RepeatedValueVector extends ValueVector, ContainerVectorLike {
+public interface RepeatedValueVector extends ValueVector {
 
   final static int DEFAULT_REPEAT_PER_RECORD = 5;
 

http://git-wip-us.apache.org/repos/asf/arrow/blob/e7e399db/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/AbstractBaseReader.java
----------------------------------------------------------------------
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/AbstractBaseReader.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/AbstractBaseReader.java
index 264e241..259a954 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/AbstractBaseReader.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/AbstractBaseReader.java
@@ -19,20 +19,20 @@ package org.apache.arrow.vector.complex.impl;
 
 import java.util.Iterator;
 
+import com.google.flatbuffers.FlatBufferBuilder;
+import org.apache.arrow.flatbuf.Type;
+import org.apache.arrow.flatbuf.Union;
+import org.apache.arrow.flatbuf.UnionMode;
 import org.apache.arrow.vector.complex.reader.FieldReader;
 import org.apache.arrow.vector.complex.writer.BaseWriter.ListWriter;
 import org.apache.arrow.vector.complex.writer.FieldWriter;
 import org.apache.arrow.vector.holders.UnionHolder;
-import org.apache.arrow.vector.types.MaterializedField;
-import org.apache.arrow.vector.types.Types.DataMode;
-import org.apache.arrow.vector.types.Types.MajorType;
-import org.apache.arrow.vector.types.Types.MinorType;
+import org.apache.arrow.vector.types.pojo.Field;
 
 
 abstract class AbstractBaseReader implements FieldReader{
 
   static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(AbstractBaseReader.class);
-  private static final MajorType LATE_BIND_TYPE = new MajorType(MinorType.LATE, DataMode.OPTIONAL);
 
   private int index;
 
@@ -58,15 +58,6 @@ abstract class AbstractBaseReader implements FieldReader{
     throw new IllegalStateException("The current reader doesn't support reading as a map.");
   }
 
-  public MajorType getType(){
-    throw new IllegalStateException("The current reader doesn't support getting type information.");
-  }
-
-  @Override
-  public MaterializedField getField() {
-    return MaterializedField.create("unknown", LATE_BIND_TYPE);
-  }
-
   @Override
   public boolean next() {
     throw new IllegalStateException("The current reader doesn't support getting next information.");

http://git-wip-us.apache.org/repos/asf/arrow/blob/e7e399db/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/AbstractBaseWriter.java
----------------------------------------------------------------------
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/AbstractBaseWriter.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/AbstractBaseWriter.java
index 4e1e103..e6cf098 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/AbstractBaseWriter.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/AbstractBaseWriter.java
@@ -23,25 +23,11 @@ import org.apache.arrow.vector.complex.writer.FieldWriter;
 abstract class AbstractBaseWriter implements FieldWriter {
   //private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(AbstractBaseWriter.class);
 
-  final FieldWriter parent;
   private int index;
 
-  public AbstractBaseWriter(FieldWriter parent) {
-    this.parent = parent;
-  }
-
   @Override
   public String toString() {
-    return super.toString() + "[index = " + index + ", parent = " + parent + "]";
-  }
-
-  @Override
-  public FieldWriter getParent() {
-    return parent;
-  }
-
-  public boolean isRoot() {
-    return parent == null;
+    return super.toString() + "[index = " + index + "]";
   }
 
   int idx() {

http://git-wip-us.apache.org/repos/asf/arrow/blob/e7e399db/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/ComplexWriterImpl.java
----------------------------------------------------------------------
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/ComplexWriterImpl.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/ComplexWriterImpl.java
index 4e2051f..4d2adfb 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/ComplexWriterImpl.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/ComplexWriterImpl.java
@@ -17,20 +17,20 @@
  */
 package org.apache.arrow.vector.complex.impl;
 
+import org.apache.arrow.vector.complex.ListVector;
 import org.apache.arrow.vector.complex.MapVector;
 import org.apache.arrow.vector.complex.StateTool;
 import org.apache.arrow.vector.complex.writer.BaseWriter.ComplexWriter;
-import org.apache.arrow.vector.types.MaterializedField;
-import org.apache.arrow.vector.types.Types;
 import org.apache.arrow.vector.types.Types.MinorType;
 
 import com.google.common.base.Preconditions;
+import org.apache.arrow.vector.types.pojo.Field;
 
 public class ComplexWriterImpl extends AbstractFieldWriter implements ComplexWriter {
 //  private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ComplexWriterImpl.class);
 
   private SingleMapWriter mapRoot;
-  private SingleListWriter listRoot;
+  private UnionListWriter listRoot;
   private final MapVector container;
 
   Mode mode = Mode.INIT;
@@ -40,7 +40,6 @@ public class ComplexWriterImpl extends AbstractFieldWriter implements ComplexWri
   private enum Mode { INIT, MAP, LIST };
 
   public ComplexWriterImpl(String name, MapVector container, boolean unionEnabled){
-    super(null);
     this.name = name;
     this.container = container;
     this.unionEnabled = unionEnabled;
@@ -51,7 +50,7 @@ public class ComplexWriterImpl extends AbstractFieldWriter implements ComplexWri
   }
 
   @Override
-  public MaterializedField getField() {
+  public Field getField() {
     return container.getField();
   }
 
@@ -123,7 +122,7 @@ public class ComplexWriterImpl extends AbstractFieldWriter implements ComplexWri
 
     case INIT:
       MapVector map = (MapVector) container;
-      mapRoot = new SingleMapWriter(map, this, unionEnabled);
+      mapRoot = new SingleMapWriter(map);
       mapRoot.setPosition(idx());
       mode = Mode.MAP;
       break;
@@ -143,8 +142,8 @@ public class ComplexWriterImpl extends AbstractFieldWriter implements ComplexWri
     switch(mode){
 
     case INIT:
-      MapVector map = container.addOrGet(name, Types.required(MinorType.MAP), MapVector.class);
-      mapRoot = new SingleMapWriter(map, this, unionEnabled);
+      MapVector map = container.addOrGet(name, MinorType.MAP, MapVector.class);
+      mapRoot = new SingleMapWriter(map);
       mapRoot.setPosition(idx());
       mode = Mode.MAP;
       break;
@@ -174,7 +173,12 @@ public class ComplexWriterImpl extends AbstractFieldWriter implements ComplexWri
     switch(mode){
 
     case INIT:
-      listRoot = new SingleListWriter(name, container, this);
+      int vectorCount = container.size();
+      ListVector listVector = container.addOrGet(name, MinorType.LIST, ListVector.class);
+      if (container.size() > vectorCount) {
+        listVector.allocateNew();
+      }
+      listRoot = new UnionListWriter(listVector);
       listRoot.setPosition(idx());
       mode = Mode.LIST;
       break;

http://git-wip-us.apache.org/repos/asf/arrow/blob/e7e399db/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/PromotableWriter.java
----------------------------------------------------------------------
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/PromotableWriter.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/PromotableWriter.java
index 462ec9d..586b128 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/PromotableWriter.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/PromotableWriter.java
@@ -17,20 +17,14 @@
  */
 package org.apache.arrow.vector.complex.impl;
 
-import java.lang.reflect.Constructor;
-
 import org.apache.arrow.vector.ValueVector;
-import org.apache.arrow.vector.VectorDescriptor;
 import org.apache.arrow.vector.ZeroVector;
 import org.apache.arrow.vector.complex.AbstractMapVector;
 import org.apache.arrow.vector.complex.ListVector;
 import org.apache.arrow.vector.complex.UnionVector;
 import org.apache.arrow.vector.complex.writer.FieldWriter;
-import org.apache.arrow.vector.types.MaterializedField;
-import org.apache.arrow.vector.types.Types.DataMode;
-import org.apache.arrow.vector.types.Types.MajorType;
 import org.apache.arrow.vector.types.Types.MinorType;
-import org.apache.arrow.vector.util.BasicTypeHelper;
+import org.apache.arrow.vector.types.pojo.Field;
 import org.apache.arrow.vector.util.TransferPair;
 
 /**
@@ -56,14 +50,12 @@ public class PromotableWriter extends AbstractPromotableFieldWriter {
   private FieldWriter writer;
 
   public PromotableWriter(ValueVector v, AbstractMapVector parentContainer) {
-    super(null);
     this.parentContainer = parentContainer;
     this.listVector = null;
     init(v);
   }
 
   public PromotableWriter(ValueVector v, ListVector listVector) {
-    super(null);
     this.listVector = listVector;
     this.parentContainer = null;
     init(v);
@@ -84,30 +76,8 @@ public class PromotableWriter extends AbstractPromotableFieldWriter {
   private void setWriter(ValueVector v) {
     state = State.SINGLE;
     vector = v;
-    type = v.getField().getType().getMinorType();
-    Class<?> writerClass = BasicTypeHelper
-        .getWriterImpl(v.getField().getType().getMinorType(), v.getField().getDataMode());
-    if (writerClass.equals(SingleListWriter.class)) {
-      writerClass = UnionListWriter.class;
-    }
-    Class<?> vectorClass = BasicTypeHelper.getValueVectorClass(v.getField().getType().getMinorType(), v.getField()
-        .getDataMode());
-    try {
-      Constructor<?> constructor = null;
-      for (Constructor<?> c : writerClass.getConstructors()) {
-        if (c.getParameterTypes().length == 3) {
-          constructor = c;
-        }
-      }
-      if (constructor == null) {
-        constructor = writerClass.getConstructor(vectorClass, AbstractFieldWriter.class);
-        writer = (FieldWriter) constructor.newInstance(vector, null);
-      } else {
-        writer = (FieldWriter) constructor.newInstance(vector, null, true);
-      }
-    } catch (ReflectiveOperationException e) {
-      throw new RuntimeException(e);
-    }
+    type = v.getMinorType();
+    writer = type.getNewFieldWriter(vector);
   }
 
   @Override
@@ -129,7 +99,7 @@ public class PromotableWriter extends AbstractPromotableFieldWriter {
       if (type == null) {
         return null;
       }
-      ValueVector v = listVector.addOrGetVector(new VectorDescriptor(new MajorType(type, DataMode.OPTIONAL))).getVector();
+      ValueVector v = listVector.addOrGetVector(type).getVector();
       v.allocateNew();
       setWriter(v);
       writer.setPosition(position);
@@ -150,11 +120,11 @@ public class PromotableWriter extends AbstractPromotableFieldWriter {
   }
 
   private FieldWriter promoteToUnion() {
-    String name = vector.getField().getLastName();
-    TransferPair tp = vector.getTransferPair(vector.getField().getType().getMinorType().name().toLowerCase(), vector.getAllocator());
+    String name = vector.getField().getName();
+    TransferPair tp = vector.getTransferPair(vector.getMinorType().name().toLowerCase(), vector.getAllocator());
     tp.transfer();
     if (parentContainer != null) {
-      unionVector = parentContainer.addOrGet(name, new MajorType(MinorType.UNION, DataMode.OPTIONAL), UnionVector.class);
+      unionVector = parentContainer.addOrGet(name, MinorType.UNION, UnionVector.class);
       unionVector.allocateNew();
     } else if (listVector != null) {
       unionVector = listVector.promoteToUnion();
@@ -163,7 +133,7 @@ public class PromotableWriter extends AbstractPromotableFieldWriter {
     writer = new UnionWriter(unionVector);
     writer.setPosition(idx());
     for (int i = 0; i < idx(); i++) {
-      unionVector.getMutator().setType(i, vector.getField().getType().getMinorType());
+      unionVector.getMutator().setType(i, vector.getMinorType());
     }
     vector = null;
     state = State.UNION;
@@ -181,7 +151,7 @@ public class PromotableWriter extends AbstractPromotableFieldWriter {
   }
 
   @Override
-  public MaterializedField getField() {
+  public Field getField() {
     return getWriter().getField();
   }
 

http://git-wip-us.apache.org/repos/asf/arrow/blob/e7e399db/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/RepeatedListReaderImpl.java
----------------------------------------------------------------------
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/RepeatedListReaderImpl.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/RepeatedListReaderImpl.java
deleted file mode 100644
index dd1a152..0000000
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/RepeatedListReaderImpl.java
+++ /dev/null
@@ -1,145 +0,0 @@
-/*******************************************************************************
-
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- ******************************************************************************/
-package org.apache.arrow.vector.complex.impl;
-
-
-import org.apache.arrow.vector.ValueVector;
-import org.apache.arrow.vector.complex.RepeatedListVector;
-import org.apache.arrow.vector.complex.reader.FieldReader;
-import org.apache.arrow.vector.complex.writer.BaseWriter.ListWriter;
-import org.apache.arrow.vector.complex.writer.BaseWriter.MapWriter;
-import org.apache.arrow.vector.holders.RepeatedListHolder;
-import org.apache.arrow.vector.types.Types.DataMode;
-import org.apache.arrow.vector.types.Types.MajorType;
-import org.apache.arrow.vector.types.Types.MinorType;
-
-public class RepeatedListReaderImpl extends AbstractFieldReader{
-  private static final int NO_VALUES = Integer.MAX_VALUE - 1;
-  private static final MajorType TYPE = new MajorType(MinorType.LIST, DataMode.REPEATED);
-  private final String name;
-  private final RepeatedListVector container;
-  private FieldReader reader;
-
-  public RepeatedListReaderImpl(String name, RepeatedListVector container) {
-    super();
-    this.name = name;
-    this.container = container;
-  }
-
-  @Override
-  public MajorType getType() {
-    return TYPE;
-  }
-
-  @Override
-  public void copyAsValue(ListWriter writer) {
-    if (currentOffset == NO_VALUES) {
-      return;
-    }
-    RepeatedListWriter impl = (RepeatedListWriter) writer;
-    impl.container.copyFromSafe(idx(), impl.idx(), container);
-  }
-
-  @Override
-  public void copyAsField(String name, MapWriter writer) {
-    if (currentOffset == NO_VALUES) {
-      return;
-    }
-    RepeatedListWriter impl = (RepeatedListWriter) writer.list(name);
-    impl.container.copyFromSafe(idx(), impl.idx(), container);
-  }
-
-  private int currentOffset;
-  private int maxOffset;
-
-  @Override
-  public void reset() {
-    super.reset();
-    currentOffset = 0;
-    maxOffset = 0;
-    if (reader != null) {
-      reader.reset();
-    }
-    reader = null;
-  }
-
-  @Override
-  public int size() {
-    return maxOffset - currentOffset;
-  }
-
-  @Override
-  public void setPosition(int index) {
-    if (index < 0 || index == NO_VALUES) {
-      currentOffset = NO_VALUES;
-      return;
-    }
-
-    super.setPosition(index);
-    RepeatedListHolder h = new RepeatedListHolder();
-    container.getAccessor().get(index, h);
-    if (h.start == h.end) {
-      currentOffset = NO_VALUES;
-    } else {
-      currentOffset = h.start-1;
-      maxOffset = h.end;
-      if(reader != null) {
-        reader.setPosition(currentOffset);
-      }
-    }
-  }
-
-  @Override
-  public boolean next() {
-    if (currentOffset +1 < maxOffset) {
-      currentOffset++;
-      if (reader != null) {
-        reader.setPosition(currentOffset);
-      }
-      return true;
-    } else {
-      currentOffset = NO_VALUES;
-      return false;
-    }
-  }
-
-  @Override
-  public Object readObject() {
-    return container.getAccessor().getObject(idx());
-  }
-
-  @Override
-  public FieldReader reader() {
-    if (reader == null) {
-      ValueVector child = container.getChild(name);
-      if (child == null) {
-        reader = NullReader.INSTANCE;
-      } else {
-        reader = child.getReader();
-      }
-      reader.setPosition(currentOffset);
-    }
-    return reader;
-  }
-
-  public boolean isSet() {
-    return true;
-  }
-
-}

http://git-wip-us.apache.org/repos/asf/arrow/blob/e7e399db/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/RepeatedMapReaderImpl.java
----------------------------------------------------------------------
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/RepeatedMapReaderImpl.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/RepeatedMapReaderImpl.java
deleted file mode 100644
index 09a831d..0000000
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/RepeatedMapReaderImpl.java
+++ /dev/null
@@ -1,192 +0,0 @@
-/*******************************************************************************
-
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- ******************************************************************************/
-package org.apache.arrow.vector.complex.impl;
-
-import java.util.Map;
-
-import org.apache.arrow.vector.ValueVector;
-import org.apache.arrow.vector.complex.RepeatedMapVector;
-import org.apache.arrow.vector.complex.reader.FieldReader;
-import org.apache.arrow.vector.complex.writer.BaseWriter.MapWriter;
-import org.apache.arrow.vector.holders.RepeatedMapHolder;
-import org.apache.arrow.vector.types.Types.MajorType;
-
-import com.google.common.collect.Maps;
-
-@SuppressWarnings("unused")
-public class RepeatedMapReaderImpl extends AbstractFieldReader{
-  private static final int NO_VALUES = Integer.MAX_VALUE - 1;
-
-  private final RepeatedMapVector vector;
-  private final Map<String, FieldReader> fields = Maps.newHashMap();
-
-  public RepeatedMapReaderImpl(RepeatedMapVector vector) {
-    this.vector = vector;
-  }
-
-  private void setChildrenPosition(int index) {
-    for (FieldReader r : fields.values()) {
-      r.setPosition(index);
-    }
-  }
-
-  @Override
-  public FieldReader reader(String name) {
-    FieldReader reader = fields.get(name);
-    if (reader == null) {
-      ValueVector child = vector.getChild(name);
-      if (child == null) {
-        reader = NullReader.INSTANCE;
-      } else {
-        reader = child.getReader();
-      }
-      fields.put(name, reader);
-      reader.setPosition(currentOffset);
-    }
-    return reader;
-  }
-
-  @Override
-  public FieldReader reader() {
-    if (currentOffset == NO_VALUES) {
-      return NullReader.INSTANCE;
-    }
-
-    setChildrenPosition(currentOffset);
-    return new SingleLikeRepeatedMapReaderImpl(vector, this);
-  }
-
-  private int currentOffset;
-  private int maxOffset;
-
-  @Override
-  public void reset() {
-    super.reset();
-    currentOffset = 0;
-    maxOffset = 0;
-    for (FieldReader reader:fields.values()) {
-      reader.reset();
-    }
-    fields.clear();
-  }
-
-  @Override
-  public int size() {
-    if (isNull()) {
-      return 0;
-    }
-    return maxOffset - (currentOffset < 0 ? 0 : currentOffset);
-  }
-
-  @Override
-  public void setPosition(int index) {
-    if (index < 0 || index == NO_VALUES) {
-      currentOffset = NO_VALUES;
-      return;
-    }
-
-    super.setPosition(index);
-    RepeatedMapHolder h = new RepeatedMapHolder();
-    vector.getAccessor().get(index, h);
-    if (h.start == h.end) {
-      currentOffset = NO_VALUES;
-    } else {
-      currentOffset = h.start-1;
-      maxOffset = h.end;
-      setChildrenPosition(currentOffset);
-    }
-  }
-
-  public void setSinglePosition(int index, int childIndex) {
-    super.setPosition(index);
-    RepeatedMapHolder h = new RepeatedMapHolder();
-    vector.getAccessor().get(index, h);
-    if (h.start == h.end) {
-      currentOffset = NO_VALUES;
-    } else {
-      int singleOffset = h.start + childIndex;
-      assert singleOffset < h.end;
-      currentOffset = singleOffset;
-      maxOffset = singleOffset + 1;
-      setChildrenPosition(singleOffset);
-    }
-  }
-
-  @Override
-  public boolean next() {
-    if (currentOffset +1 < maxOffset) {
-      setChildrenPosition(++currentOffset);
-      return true;
-    } else {
-      currentOffset = NO_VALUES;
-      return false;
-    }
-  }
-
-  public boolean isNull() {
-    return currentOffset == NO_VALUES;
-  }
-
-  @Override
-  public Object readObject() {
-    return vector.getAccessor().getObject(idx());
-  }
-
-  @Override
-  public MajorType getType() {
-    return vector.getField().getType();
-  }
-
-  @Override
-  public java.util.Iterator<String> iterator() {
-    return vector.fieldNameIterator();
-  }
-
-  @Override
-  public boolean isSet() {
-    return true;
-  }
-
-  @Override
-  public void copyAsValue(MapWriter writer) {
-    if (currentOffset == NO_VALUES) {
-      return;
-    }
-    RepeatedMapWriter impl = (RepeatedMapWriter) writer;
-    impl.container.copyFromSafe(idx(), impl.idx(), vector);
-  }
-
-  public void copyAsValueSingle(MapWriter writer) {
-    if (currentOffset == NO_VALUES) {
-      return;
-    }
-    SingleMapWriter impl = (SingleMapWriter) writer;
-    impl.container.copyFromSafe(currentOffset, impl.idx(), vector);
-  }
-
-  @Override
-  public void copyAsField(String name, MapWriter writer) {
-    if (currentOffset == NO_VALUES) {
-      return;
-    }
-    RepeatedMapWriter impl = (RepeatedMapWriter) writer.map(name);
-    impl.container.copyFromSafe(idx(), impl.idx(), vector);
-  }
-
-}

http://git-wip-us.apache.org/repos/asf/arrow/blob/e7e399db/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/SingleLikeRepeatedMapReaderImpl.java
----------------------------------------------------------------------
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/SingleLikeRepeatedMapReaderImpl.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/SingleLikeRepeatedMapReaderImpl.java
deleted file mode 100644
index 086d26e..0000000
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/SingleLikeRepeatedMapReaderImpl.java
+++ /dev/null
@@ -1,89 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.arrow.vector.complex.impl;
-
-import java.util.Iterator;
-
-import org.apache.arrow.vector.complex.RepeatedMapVector;
-import org.apache.arrow.vector.complex.reader.FieldReader;
-import org.apache.arrow.vector.complex.writer.BaseWriter.MapWriter;
-import org.apache.arrow.vector.types.Types;
-import org.apache.arrow.vector.types.Types.MajorType;
-import org.apache.arrow.vector.types.Types.MinorType;
-
-public class SingleLikeRepeatedMapReaderImpl extends AbstractFieldReader{
-
-  private RepeatedMapReaderImpl delegate;
-
-  public SingleLikeRepeatedMapReaderImpl(RepeatedMapVector vector, FieldReader delegate) {
-    this.delegate = (RepeatedMapReaderImpl) delegate;
-  }
-
-  @Override
-  public int size() {
-    throw new UnsupportedOperationException("You can't call size on a single map reader.");
-  }
-
-  @Override
-  public boolean next() {
-    throw new UnsupportedOperationException("You can't call next on a single map reader.");
-  }
-
-  @Override
-  public MajorType getType() {
-    return Types.required(MinorType.MAP);
-  }
-
-
-  @Override
-  public void copyAsValue(MapWriter writer) {
-    delegate.copyAsValueSingle(writer);
-  }
-
-  public void copyAsValueSingle(MapWriter writer){
-    delegate.copyAsValueSingle(writer);
-  }
-
-  @Override
-  public FieldReader reader(String name) {
-    return delegate.reader(name);
-  }
-
-  @Override
-  public void setPosition(int index) {
-    delegate.setPosition(index);
-  }
-
-  @Override
-  public Object readObject() {
-    return delegate.readObject();
-  }
-
-  @Override
-  public Iterator<String> iterator() {
-    return delegate.iterator();
-  }
-
-  @Override
-  public boolean isSet() {
-    return ! delegate.isNull();
-  }
-
-
-}


Mime
View raw message