drill-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jacq...@apache.org
Subject [04/45] drill git commit: DRILL-3987: (MOVE) Extract key vector, field reader, complex/field writer classes.
Date Fri, 13 Nov 2015 02:37:34 GMT
http://git-wip-us.apache.org/repos/asf/drill/blob/9969d8bd/exec/vector/src/main/java/org/apache/drill/exec/expr/holders/ComplexHolder.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/java/org/apache/drill/exec/expr/holders/ComplexHolder.java b/exec/vector/src/main/java/org/apache/drill/exec/expr/holders/ComplexHolder.java
new file mode 100644
index 0000000..e1025df
--- /dev/null
+++ b/exec/vector/src/main/java/org/apache/drill/exec/expr/holders/ComplexHolder.java
@@ -0,0 +1,25 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.expr.holders;
+
+import org.apache.drill.exec.vector.complex.reader.FieldReader;
+
+public class ComplexHolder implements ValueHolder {
+  public FieldReader reader;
+  public int isSet;
+}

http://git-wip-us.apache.org/repos/asf/drill/blob/9969d8bd/exec/vector/src/main/java/org/apache/drill/exec/expr/holders/ObjectHolder.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/java/org/apache/drill/exec/expr/holders/ObjectHolder.java b/exec/vector/src/main/java/org/apache/drill/exec/expr/holders/ObjectHolder.java
new file mode 100644
index 0000000..391a795
--- /dev/null
+++ b/exec/vector/src/main/java/org/apache/drill/exec/expr/holders/ObjectHolder.java
@@ -0,0 +1,37 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.drill.exec.expr.holders;
+
+import org.apache.drill.common.types.TypeProtos;
+import org.apache.drill.common.types.Types;
+
+/*
+ * Holder class for the vector ObjectVector. This holder internally stores a
+ * reference to an object. The ObjectVector maintains an array of these objects.
+ * This holder can be used only as workspace variables in aggregate functions.
+ * Using this holder should be avoided and we should stick to native holder types.
+ */
+@Deprecated
+public class ObjectHolder implements ValueHolder {
+  public static final TypeProtos.MajorType TYPE = Types.required(TypeProtos.MinorType.GENERIC_OBJECT);
+
+  public TypeProtos.MajorType getType() {return TYPE;}
+
+  public Object obj;
+}

http://git-wip-us.apache.org/repos/asf/drill/blob/9969d8bd/exec/vector/src/main/java/org/apache/drill/exec/expr/holders/RepeatedListHolder.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/java/org/apache/drill/exec/expr/holders/RepeatedListHolder.java b/exec/vector/src/main/java/org/apache/drill/exec/expr/holders/RepeatedListHolder.java
new file mode 100644
index 0000000..09746da
--- /dev/null
+++ b/exec/vector/src/main/java/org/apache/drill/exec/expr/holders/RepeatedListHolder.java
@@ -0,0 +1,23 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.expr.holders;
+
+public final class RepeatedListHolder implements ValueHolder{
+  public int start;
+  public int end;
+}

http://git-wip-us.apache.org/repos/asf/drill/blob/9969d8bd/exec/vector/src/main/java/org/apache/drill/exec/expr/holders/RepeatedMapHolder.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/java/org/apache/drill/exec/expr/holders/RepeatedMapHolder.java b/exec/vector/src/main/java/org/apache/drill/exec/expr/holders/RepeatedMapHolder.java
new file mode 100644
index 0000000..247f75e
--- /dev/null
+++ b/exec/vector/src/main/java/org/apache/drill/exec/expr/holders/RepeatedMapHolder.java
@@ -0,0 +1,23 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.expr.holders;
+
+public final class RepeatedMapHolder implements ValueHolder{
+  public int start;
+  public int end;
+}

http://git-wip-us.apache.org/repos/asf/drill/blob/9969d8bd/exec/vector/src/main/java/org/apache/drill/exec/expr/holders/UnionHolder.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/java/org/apache/drill/exec/expr/holders/UnionHolder.java b/exec/vector/src/main/java/org/apache/drill/exec/expr/holders/UnionHolder.java
new file mode 100644
index 0000000..84cdefb
--- /dev/null
+++ b/exec/vector/src/main/java/org/apache/drill/exec/expr/holders/UnionHolder.java
@@ -0,0 +1,37 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.expr.holders;
+
+import org.apache.drill.common.types.TypeProtos.MajorType;
+import org.apache.drill.common.types.TypeProtos.MinorType;
+import org.apache.drill.common.types.Types;
+import org.apache.drill.exec.vector.complex.reader.FieldReader;
+
+public class UnionHolder implements ValueHolder {
+  public static final MajorType TYPE = Types.optional(MinorType.UNION);
+  public FieldReader reader;
+  public int isSet;
+
+  public MajorType getType() {
+    return reader.getType();
+  }
+
+  public boolean isSet() {
+    return isSet == 1;
+  }
+}

http://git-wip-us.apache.org/repos/asf/drill/blob/9969d8bd/exec/vector/src/main/java/org/apache/drill/exec/expr/holders/ValueHolder.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/java/org/apache/drill/exec/expr/holders/ValueHolder.java b/exec/vector/src/main/java/org/apache/drill/exec/expr/holders/ValueHolder.java
new file mode 100644
index 0000000..2431dcb
--- /dev/null
+++ b/exec/vector/src/main/java/org/apache/drill/exec/expr/holders/ValueHolder.java
@@ -0,0 +1,31 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.expr.holders;
+
+/**
+ * Wrapper object for an individual value in Drill.
+ *
+ * ValueHolders are designed to be mutable wrapper objects for defining clean
+ * APIs that access data in Drill. For performance, object creation is avoided
+ * at all costs throughout execution. For this reason, ValueHolders are
+ * disallowed from implementing any methods, this allows for them to be
+ * replaced by their java primitive inner members during optimization of
+ * run-time generated code.
+ */
+public interface ValueHolder {
+}

http://git-wip-us.apache.org/repos/asf/drill/blob/9969d8bd/exec/vector/src/main/java/org/apache/drill/exec/memory/BufferAllocator.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/java/org/apache/drill/exec/memory/BufferAllocator.java b/exec/vector/src/main/java/org/apache/drill/exec/memory/BufferAllocator.java
new file mode 100644
index 0000000..b01534e
--- /dev/null
+++ b/exec/vector/src/main/java/org/apache/drill/exec/memory/BufferAllocator.java
@@ -0,0 +1,136 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.memory;
+
+import io.netty.buffer.ByteBufAllocator;
+import io.netty.buffer.DrillBuf;
+import io.netty.buffer.UnsafeDirectLittleEndian;
+
+import java.io.Closeable;
+
+import org.apache.drill.exec.ops.FragmentContext;
+import org.apache.drill.exec.util.Pointer;
+
+/**
+ * Wrapper class to deal with byte buffer allocation. Ensures users only use designated methods. Also allows inser
+ */
+public interface BufferAllocator extends Closeable {
+  static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(BufferAllocator.class);
+
+  /**
+   * Allocate a new or reused buffer of the provided size. Note that the buffer may technically be larger than the
+   * requested size for rounding purposes. However, the buffers capacity will be set to the configured size.
+   *
+   * @param size
+   *          The size in bytes.
+   * @return A new ByteBuf.
+   * @throws OutOfMemoryRuntimeException if buffer cannot be allocated
+   */
+  public abstract DrillBuf buffer(int size);
+
+  /**
+   * Allocate a new or reused buffer within provided range. Note that the buffer may technically be larger than the
+   * requested size for rounding purposes. However, the buffers capacity will be set to the configured size.
+   *
+   * @param minSize The minimum size in bytes.
+   * @param maxSize The maximum size in bytes.
+   * @return A new ByteBuf.
+   * @throws OutOfMemoryRuntimeException if buffer cannot be allocated
+   */
+  public abstract DrillBuf buffer(int minSize, int maxSize);
+
+  public abstract ByteBufAllocator getUnderlyingAllocator();
+
+  /**
+   * Create a child allocator nested below this one.
+   *
+   * @param context - owning fragment for this allocator
+   * @param initialReservation - specified in bytes
+   * @param maximumReservation - specified in bytes
+   * @param applyFragmentLimit - flag to conditionally enable fragment memory limits
+   * @return - a new buffer allocator owned by the parent it was spawned from
+   * @throws OutOfMemoryException - when off-heap memory has been exhausted
+   */
+  public abstract BufferAllocator getChildAllocator(FragmentContext context, long initialReservation,
+      long maximumReservation, boolean applyFragmentLimit) throws OutOfMemoryException;
+
+  /**
+   * Take over ownership of fragment accounting.  Always takes over ownership.
+   * @param buf
+   * @return false if over allocation.
+   */
+  public boolean takeOwnership(DrillBuf buf) ;
+
+  /**
+   * Take over ownership of fragment accounting.  Always takes over ownership.
+   * @param buf
+   * @return false if over allocation.
+   */
+  public boolean takeOwnership(DrillBuf buf, Pointer<DrillBuf> bufOut);
+
+  public PreAllocator getNewPreAllocator();
+
+  //public void addFragmentContext(FragmentContext c);
+
+  /**
+   * For Top Level Allocators. Reset the fragment limits for all allocators
+   */
+  public void resetFragmentLimits();
+
+  /**
+   * For Child allocators to set the Fragment limit for the corresponding fragment allocator.
+   * @param l the new fragment limit
+   */
+  public void setFragmentLimit(long l);
+
+  public long getFragmentLimit();
+
+
+  /**
+   * Not thread safe.
+   *
+   * WARNING: unclaimed pre-allocations leak memory. If you call preAllocate(), you must
+   * make sure to ultimately try to get the buffer and release it.
+   */
+  public interface PreAllocator {
+    public boolean preAllocate(int bytes);
+
+    public DrillBuf getAllocation();
+  }
+
+  /**
+   * @param bytes
+   * @return
+   */
+
+  /**
+   *
+   */
+
+  /**
+   * Close and release all buffers generated from this buffer pool.
+   */
+  @Override
+  public abstract void close();
+
+  public abstract long getAllocatedMemory();
+
+  public abstract long getPeakMemoryAllocation();
+
+  public DrillBuf getEmpty();
+}

http://git-wip-us.apache.org/repos/asf/drill/blob/9969d8bd/exec/vector/src/main/java/org/apache/drill/exec/memory/OutOfMemoryRuntimeException.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/java/org/apache/drill/exec/memory/OutOfMemoryRuntimeException.java b/exec/vector/src/main/java/org/apache/drill/exec/memory/OutOfMemoryRuntimeException.java
new file mode 100644
index 0000000..305eabd
--- /dev/null
+++ b/exec/vector/src/main/java/org/apache/drill/exec/memory/OutOfMemoryRuntimeException.java
@@ -0,0 +1,49 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.memory;
+
+import org.apache.drill.common.exceptions.DrillRuntimeException;
+
+public class OutOfMemoryRuntimeException extends DrillRuntimeException{
+
+  public OutOfMemoryRuntimeException() {
+    super();
+
+  }
+
+  public OutOfMemoryRuntimeException(String message, Throwable cause, boolean enableSuppression,
+      boolean writableStackTrace) {
+    super(message, cause, enableSuppression, writableStackTrace);
+
+  }
+
+  public OutOfMemoryRuntimeException(String message, Throwable cause) {
+    super(message, cause);
+
+  }
+
+  public OutOfMemoryRuntimeException(String message) {
+    super(message);
+
+  }
+
+  public OutOfMemoryRuntimeException(Throwable cause) {
+    super(cause);
+
+  }
+}

http://git-wip-us.apache.org/repos/asf/drill/blob/9969d8bd/exec/vector/src/main/java/org/apache/drill/exec/record/MaterializedField.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/java/org/apache/drill/exec/record/MaterializedField.java b/exec/vector/src/main/java/org/apache/drill/exec/record/MaterializedField.java
new file mode 100644
index 0000000..26e1257
--- /dev/null
+++ b/exec/vector/src/main/java/org/apache/drill/exec/record/MaterializedField.java
@@ -0,0 +1,324 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.record;
+
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.LinkedHashSet;
+import java.util.Map;
+
+import org.apache.drill.common.expression.PathSegment;
+import org.apache.drill.common.expression.SchemaPath;
+import org.apache.drill.common.types.TypeProtos.DataMode;
+import org.apache.drill.common.types.TypeProtos.MajorType;
+import org.apache.drill.exec.expr.TypeHelper;
+import org.apache.drill.exec.proto.UserBitShared.SerializedField;
+
+
+public class MaterializedField {
+  private final Key key;
+  // use an ordered set as existing code relies on order (e,g. parquet writer)
+  private final LinkedHashSet<MaterializedField> children;
+
+  private MaterializedField(SchemaPath path, MajorType type) {
+    this(path, type, new LinkedHashSet<MaterializedField>());
+  }
+
+  private MaterializedField(SchemaPath path, MajorType type, LinkedHashSet<MaterializedField> children) {
+    this.key = new Key(path, type);
+    this.children = children;
+  }
+
+  public static MaterializedField create(SerializedField serField){
+    MaterializedField field = new MaterializedField(SchemaPath.create(serField.getNamePart()), serField.getMajorType());
+    for (SerializedField sf:serField.getChildList()) {
+      field.addChild(MaterializedField.create(sf));
+    }
+    return field;
+  }
+
+  /**
+   * Create and return a serialized field based on the current state.
+   */
+  public SerializedField getSerializedField() {
+    SerializedField.Builder serializedFieldBuilder = getAsBuilder();
+    for(MaterializedField childMaterializedField : getChildren()) {
+      serializedFieldBuilder.addChild(childMaterializedField.getSerializedField());
+    }
+    return serializedFieldBuilder.build();
+  }
+
+
+  public SerializedField.Builder getAsBuilder(){
+    return SerializedField.newBuilder() //
+        .setMajorType(key.type) //
+        .setNamePart(key.path.getAsNamePart());
+  }
+
+  public Collection<MaterializedField> getChildren() {
+    return children;
+  }
+
+  public void addChild(MaterializedField field){
+    children.add(field);
+  }
+
+  public MaterializedField clone() {
+    return withPathAndType(getPath(), getType());
+  }
+
+  public MaterializedField withType(MajorType type) {
+    return withPathAndType(getPath(), type);
+  }
+
+  public MaterializedField withPath(SchemaPath path) {
+    return withPathAndType(path, getType());
+  }
+
+  public MaterializedField withPathAndType(final SchemaPath path, final MajorType type) {
+    final LinkedHashSet<MaterializedField> newChildren = new LinkedHashSet<>(children.size());
+    final MaterializedField clone = new MaterializedField(path, type, newChildren);
+    for (final MaterializedField child:children) {
+      newChildren.add(child.clone());
+    }
+    return clone;
+  }
+
+  public String getLastName(){
+    PathSegment seg = key.path.getRootSegment();
+    while (seg.getChild() != null) {
+      seg = seg.getChild();
+    }
+    return seg.getNameSegment().getPath();
+  }
+
+
+  // TODO: rewrite without as direct match rather than conversion then match.
+  public boolean matches(SerializedField field){
+    MaterializedField f = create(field);
+    return f.equals(this);
+  }
+
+  public static MaterializedField create(String path, MajorType type){
+    SchemaPath p = SchemaPath.getSimplePath(path);
+    return create(p, type);
+  }
+
+  public static MaterializedField create(SchemaPath path, MajorType type) {
+    return new MaterializedField(path, type);
+  }
+
+  public SchemaPath getPath(){
+    return key.path;
+  }
+
+  /**
+   * Get the schema path.  Deprecated, use getPath() instead.
+   * @return the SchemaPath of this field.
+   */
+  @Deprecated
+  public SchemaPath getAsSchemaPath(){
+    return getPath();
+  }
+
+//  public String getName(){
+//    StringBuilder sb = new StringBuilder();
+//    boolean first = true;
+//    for(NamePart np : def.getNameList()){
+//      if(np.getType() == Type.ARRAY){
+//        sb.append("[]");
+//      }else{
+//        if(first){
+//          first = false;
+//        }else{
+//          sb.append(".");
+//        }
+//        sb.append('`');
+//        sb.append(np.getName());
+//        sb.append('`');
+//
+//      }
+//    }
+//    return sb.toString();
+//  }
+
+  public int getWidth() {
+    return key.type.getWidth();
+  }
+
+  public MajorType getType() {
+    return key.type;
+  }
+
+  public int getScale() {
+      return key.type.getScale();
+  }
+  public int getPrecision() {
+      return key.type.getPrecision();
+  }
+  public boolean isNullable() {
+    return key.type.getMode() == DataMode.OPTIONAL;
+  }
+
+  public DataMode getDataMode() {
+    return key.type.getMode();
+  }
+
+  public MaterializedField getOtherNullableVersion(){
+    MajorType mt = key.type;
+    DataMode newDataMode = null;
+    switch (mt.getMode()){
+    case OPTIONAL:
+      newDataMode = DataMode.REQUIRED;
+      break;
+    case REQUIRED:
+      newDataMode = DataMode.OPTIONAL;
+      break;
+    default:
+      throw new UnsupportedOperationException();
+    }
+    return new MaterializedField(key.path, mt.toBuilder().setMode(newDataMode).build());
+  }
+
+  public Class<?> getValueClass() {
+    return TypeHelper.getValueVectorClass(getType().getMinorType(), getDataMode());
+  }
+
+  public boolean matches(SchemaPath path) {
+    if (!path.isSimplePath()) {
+      return false;
+    }
+
+    return key.path.equals(path);
+  }
+
+
+  @Override
+  public int hashCode() {
+    int result = 1;
+    // DRILL-1872: Compute hashCode only on key. See also the comment
+    // in MapVector$MapTransferPair
+    result = ((key == null) ? 0 : key.hashCode());
+    return result;
+  }
+
+  @Override
+  public boolean equals(Object obj) {
+    if (this == obj) {
+      return true;
+    }
+    if (obj == null) {
+      return false;
+    }
+    if (getClass() != obj.getClass()) {
+      return false;
+    }
+    MaterializedField other = (MaterializedField) obj;
+    // DRILL-1872: Compute equals only on key. See also the comment
+    // in MapVector$MapTransferPair
+
+    if (key == null) {
+      if (other.key != null) {
+        return false;
+      }
+    } else if (!key.equals(other.key)) {
+      return false;
+    }
+    return true;
+  }
+
+
+  @Override
+  public String toString() {
+    final int maxLen = 10;
+    String childStr = children != null && !children.isEmpty() ? toString(children, maxLen) : "";
+    return key.path + "(" + key.type.getMinorType().name() + ":" + key.type.getMode().name() + ")" + childStr;
+  }
+
+
+  private String toString(Collection<?> collection, int maxLen) {
+    StringBuilder builder = new StringBuilder();
+    builder.append("[");
+    int i = 0;
+    for (Iterator<?> iterator = collection.iterator(); iterator.hasNext() && i < maxLen; i++) {
+      if (i > 0){
+        builder.append(", ");
+      }
+      builder.append(iterator.next());
+    }
+    builder.append("]");
+    return builder.toString();
+  }
+
+  public Key key() {
+    return key;
+  }
+
+  public String toExpr(){
+    return key.path.toExpr();
+  }
+
+  /**
+   * Since the {@code MaterializedField) itself is mutable, in certain cases, it is not suitable
+   * as a key of a {@link Map}. This inner class allows the {@link MaterializedField} object to be
+   * used for this purpose.
+   */
+  public class Key {
+
+    private final SchemaPath path;
+    private final MajorType type;
+
+    private Key(SchemaPath path, MajorType type) {
+      this.path = path;
+      this.type = type;
+    }
+
+    @Override
+    public int hashCode() {
+      final int prime = 31;
+      int result = 1;
+      result = prime * result + ((path == null) ? 0 : path.hashCode());
+      return result;
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+      if (this == obj) {
+        return true;
+      }
+      if (obj == null) {
+        return false;
+      }
+      if (getClass() != obj.getClass()) {
+        return false;
+      }
+      Key other = (Key) obj;
+      if (path == null) {
+        if (other.path != null) {
+          return false;
+        }
+      } else if (!path.equals(other.path)) {
+        return false;
+      }
+
+      return true;
+    }
+
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/drill/blob/9969d8bd/exec/vector/src/main/java/org/apache/drill/exec/record/TransferPair.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/java/org/apache/drill/exec/record/TransferPair.java b/exec/vector/src/main/java/org/apache/drill/exec/record/TransferPair.java
new file mode 100644
index 0000000..3585a8c
--- /dev/null
+++ b/exec/vector/src/main/java/org/apache/drill/exec/record/TransferPair.java
@@ -0,0 +1,27 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.record;
+
+import org.apache.drill.exec.vector.ValueVector;
+
+public interface TransferPair {
+  public void transfer();
+  public void splitAndTransfer(int startIndex, int length);
+  public ValueVector getTo();
+  public void copyValueSafe(int from, int to);
+}

http://git-wip-us.apache.org/repos/asf/drill/blob/9969d8bd/exec/vector/src/main/java/org/apache/drill/exec/record/TypedFieldId.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/java/org/apache/drill/exec/record/TypedFieldId.java b/exec/vector/src/main/java/org/apache/drill/exec/record/TypedFieldId.java
new file mode 100644
index 0000000..931f981
--- /dev/null
+++ b/exec/vector/src/main/java/org/apache/drill/exec/record/TypedFieldId.java
@@ -0,0 +1,272 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.record;
+
+import java.util.Arrays;
+
+import org.apache.commons.lang3.ArrayUtils;
+import org.apache.drill.common.expression.PathSegment;
+import org.apache.drill.common.types.TypeProtos.MajorType;
+import org.apache.drill.exec.expr.TypeHelper;
+import org.apache.drill.exec.vector.ValueVector;
+
+import com.carrotsearch.hppc.IntArrayList;
+import com.google.common.base.Preconditions;
+
+public class TypedFieldId {
+  final MajorType finalType;
+  final MajorType secondaryFinal;
+  final MajorType intermediateType;
+  final int[] fieldIds;
+  final boolean isHyperReader;
+  final boolean isListVector;
+  final PathSegment remainder;
+
+  public TypedFieldId(MajorType type, int... fieldIds) {
+    this(type, type, type, false, null, fieldIds);
+  }
+
+  public TypedFieldId(MajorType type, IntArrayList breadCrumb, PathSegment remainder) {
+    this(type, type, type, false, remainder, breadCrumb.toArray());
+  }
+
+  public TypedFieldId(MajorType type, boolean isHyper, int... fieldIds) {
+    this(type, type, type, isHyper, null, fieldIds);
+  }
+
+  public TypedFieldId(MajorType intermediateType, MajorType secondaryFinal, MajorType finalType, boolean isHyper, PathSegment remainder, int... fieldIds) {
+    this(intermediateType, secondaryFinal, finalType, isHyper, false, remainder, fieldIds);
+  }
+
+  public TypedFieldId(MajorType intermediateType, MajorType secondaryFinal, MajorType finalType, boolean isHyper, boolean isListVector, PathSegment remainder, int... fieldIds) {
+    super();
+    this.intermediateType = intermediateType;
+    this.finalType = finalType;
+    this.secondaryFinal = secondaryFinal;
+    this.fieldIds = fieldIds;
+    this.isHyperReader = isHyper;
+    this.isListVector = isListVector;
+    this.remainder = remainder;
+  }
+
+  public TypedFieldId cloneWithChild(int id) {
+    int[] fieldIds = ArrayUtils.add(this.fieldIds, id);
+    return new TypedFieldId(intermediateType, secondaryFinal, finalType, isHyperReader, remainder, fieldIds);
+  }
+
+  public PathSegment getLastSegment() {
+    if (remainder == null) {
+      return null;
+    }
+    PathSegment seg = remainder;
+    while (seg.getChild() != null) {
+      seg = seg.getChild();
+    }
+    return seg;
+  }
+
+  public TypedFieldId cloneWithRemainder(PathSegment remainder) {
+    return new TypedFieldId(intermediateType, secondaryFinal, finalType, isHyperReader, remainder, fieldIds);
+  }
+
+  public boolean hasRemainder() {
+    return remainder != null;
+  }
+
+  public PathSegment getRemainder() {
+    return remainder;
+  }
+
+  public boolean isHyperReader() {
+    return isHyperReader;
+  }
+
+  public boolean isListVector() {
+    return isListVector;
+  }
+
+  public MajorType getIntermediateType() {
+    return intermediateType;
+  }
+
+  public Class<? extends ValueVector> getIntermediateClass() {
+    return (Class<? extends ValueVector>) TypeHelper.getValueVectorClass(intermediateType.getMinorType(), intermediateType.getMode());
+  }
+
+  public MajorType getFinalType() {
+    return finalType;
+  }
+
+  public int[] getFieldIds() {
+    return fieldIds;
+  }
+
+  public MajorType getSecondaryFinal() {
+    return secondaryFinal;
+  }
+
+  public static Builder newBuilder() {
+    return new Builder();
+  }
+
+  public static class Builder{
+    final IntArrayList ids = new IntArrayList();
+    MajorType finalType;
+    MajorType intermediateType;
+    MajorType secondaryFinal;
+    PathSegment remainder;
+    boolean hyperReader = false;
+    boolean withIndex = false;
+    boolean isListVector = false;
+
+    public Builder addId(int id) {
+      ids.add(id);
+      return this;
+    }
+
+    public Builder withIndex() {
+      withIndex = true;
+      return this;
+    }
+
+    public Builder remainder(PathSegment remainder) {
+      this.remainder = remainder;
+      return this;
+    }
+
+    public Builder hyper() {
+      this.hyperReader = true;
+      return this;
+    }
+
+    public Builder listVector() {
+      this.isListVector = true;
+      return this;
+    }
+
+    public Builder finalType(MajorType finalType) {
+      this.finalType = finalType;
+      return this;
+    }
+
+    public Builder secondaryFinal(MajorType secondaryFinal) {
+      this.secondaryFinal = secondaryFinal;
+      return this;
+    }
+
+    public Builder intermediateType(MajorType intermediateType) {
+      this.intermediateType = intermediateType;
+      return this;
+    }
+
+    public TypedFieldId build() {
+      Preconditions.checkNotNull(intermediateType);
+      Preconditions.checkNotNull(finalType);
+
+      if (intermediateType == null) {
+        intermediateType = finalType;
+      }
+      if (secondaryFinal == null) {
+        secondaryFinal = finalType;
+      }
+
+      MajorType actualFinalType = finalType;
+      //MajorType secondaryFinal = finalType;
+
+      // if this has an index, switch to required type for output
+      //if(withIndex && intermediateType == finalType) actualFinalType = finalType.toBuilder().setMode(DataMode.REQUIRED).build();
+
+      // if this isn't a direct access, switch the final type to nullable as offsets may be null.
+      // TODO: there is a bug here with some things.
+      //if(intermediateType != finalType) actualFinalType = finalType.toBuilder().setMode(DataMode.OPTIONAL).build();
+
+      return new TypedFieldId(intermediateType, secondaryFinal, actualFinalType, hyperReader, isListVector, remainder, ids.toArray());
+    }
+  }
+
+  @Override
+  public int hashCode() {
+    final int prime = 31;
+    int result = 1;
+    result = prime * result + Arrays.hashCode(fieldIds);
+    result = prime * result + ((finalType == null) ? 0 : finalType.hashCode());
+    result = prime * result + ((intermediateType == null) ? 0 : intermediateType.hashCode());
+    result = prime * result + (isHyperReader ? 1231 : 1237);
+    result = prime * result + ((remainder == null) ? 0 : remainder.hashCode());
+    result = prime * result + ((secondaryFinal == null) ? 0 : secondaryFinal.hashCode());
+    return result;
+  }
+
+  @Override
+  public boolean equals(Object obj) {
+    if (this == obj) {
+      return true;
+    }
+    if (obj == null) {
+      return false;
+    }
+    if (getClass() != obj.getClass()) {
+      return false;
+    }
+    TypedFieldId other = (TypedFieldId) obj;
+    if (!Arrays.equals(fieldIds, other.fieldIds)) {
+      return false;
+    }
+    if (finalType == null) {
+      if (other.finalType != null) {
+        return false;
+      }
+    } else if (!finalType.equals(other.finalType)) {
+      return false;
+    }
+    if (intermediateType == null) {
+      if (other.intermediateType != null) {
+        return false;
+      }
+    } else if (!intermediateType.equals(other.intermediateType)) {
+      return false;
+    }
+    if (isHyperReader != other.isHyperReader) {
+      return false;
+    }
+    if (remainder == null) {
+      if (other.remainder != null) {
+        return false;
+      }
+    } else if (!remainder.equals(other.remainder)) {
+      return false;
+    }
+    if (secondaryFinal == null) {
+      if (other.secondaryFinal != null) {
+        return false;
+      }
+    } else if (!secondaryFinal.equals(other.secondaryFinal)) {
+      return false;
+    }
+    return true;
+  }
+
+  @Override
+  public String toString() {
+    final int maxLen = 10;
+    return "TypedFieldId [fieldIds="
+        + (fieldIds != null ? Arrays.toString(Arrays.copyOf(fieldIds, Math.min(fieldIds.length, maxLen))) : null)
+        + ", remainder=" + remainder + "]";
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/drill/blob/9969d8bd/exec/vector/src/main/java/org/apache/drill/exec/util/CallBack.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/java/org/apache/drill/exec/util/CallBack.java b/exec/vector/src/main/java/org/apache/drill/exec/util/CallBack.java
new file mode 100644
index 0000000..0243f8a
--- /dev/null
+++ b/exec/vector/src/main/java/org/apache/drill/exec/util/CallBack.java
@@ -0,0 +1,23 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.util;
+
+
+public interface CallBack {
+  public void doWork();
+}

http://git-wip-us.apache.org/repos/asf/drill/blob/9969d8bd/exec/vector/src/main/java/org/apache/drill/exec/util/JsonStringArrayList.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/java/org/apache/drill/exec/util/JsonStringArrayList.java b/exec/vector/src/main/java/org/apache/drill/exec/util/JsonStringArrayList.java
new file mode 100644
index 0000000..dd5145d
--- /dev/null
+++ b/exec/vector/src/main/java/org/apache/drill/exec/util/JsonStringArrayList.java
@@ -0,0 +1,63 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.util;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.hadoop.io.Text;
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.module.SimpleModule;
+
+public class JsonStringArrayList<E> extends ArrayList<E> {
+
+  private static ObjectMapper mapper;
+
+  static {
+    mapper = new ObjectMapper();
+    SimpleModule serializer = new SimpleModule("TextSerializer")
+        .addSerializer(Text.class, new TextSerializer());
+    mapper.registerModule(serializer);
+  }
+
+  @Override
+  public boolean equals(Object obj) {
+    if (this == obj) {
+      return true;
+    }
+    if (obj == null) {
+      return false;
+    }
+    if (!(obj instanceof List)) {
+      return false;
+    }
+    List other = (List) obj;
+    return this.size() == other.size() && this.containsAll(other);
+  }
+
+  @Override
+  public final String toString() {
+    try {
+      return mapper.writeValueAsString(this);
+    } catch(JsonProcessingException e) {
+      throw new IllegalStateException("Cannot serialize array list to JSON string", e);
+    }
+  }
+}

http://git-wip-us.apache.org/repos/asf/drill/blob/9969d8bd/exec/vector/src/main/java/org/apache/drill/exec/util/JsonStringHashMap.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/java/org/apache/drill/exec/util/JsonStringHashMap.java b/exec/vector/src/main/java/org/apache/drill/exec/util/JsonStringHashMap.java
new file mode 100644
index 0000000..ac980eb
--- /dev/null
+++ b/exec/vector/src/main/java/org/apache/drill/exec/util/JsonStringHashMap.java
@@ -0,0 +1,82 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.util;
+
+import java.util.LinkedHashMap;
+import java.util.Map;
+
+import org.apache.hadoop.io.Text;
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.module.SimpleModule;
+
+/*
+ * Simple class that extends the regular java.util.HashMap but overrides the
+ * toString() method of the HashMap class to produce a JSON string instead
+ */
+public class JsonStringHashMap<K, V> extends LinkedHashMap<K, V> {
+
+  private static ObjectMapper mapper;
+
+  static {
+    mapper = new ObjectMapper();
+    SimpleModule serializer = new SimpleModule("TextSerializer")
+        .addSerializer(Text.class, new TextSerializer());
+    mapper.registerModule(serializer);
+  }
+
+  @Override
+  public boolean equals(Object obj) {
+    if (this == obj) {
+      return true;
+    }
+    if (obj == null) {
+      return false;
+    }
+    if (!(obj instanceof Map)) {
+      return false;
+    }
+    Map other = (Map) obj;
+    if (this.size() != other.size()) {
+      return false;
+    }
+    for (K key : this.keySet()) {
+      if (this.get(key) == null ) {
+        if (other.get(key) == null) {
+          continue;
+        } else {
+          return false;
+        }
+      }
+      if ( ! this.get(key).equals(other.get(key))) {
+        return false;
+      }
+    }
+    return true;
+  }
+
+  @Override
+  public final String toString() {
+    try {
+      return mapper.writeValueAsString(this);
+    } catch(JsonProcessingException e) {
+      throw new IllegalStateException("Cannot serialize hash map to JSON string", e);
+    }
+  }
+}

http://git-wip-us.apache.org/repos/asf/drill/blob/9969d8bd/exec/vector/src/main/java/org/apache/drill/exec/util/Pointer.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/java/org/apache/drill/exec/util/Pointer.java b/exec/vector/src/main/java/org/apache/drill/exec/util/Pointer.java
new file mode 100644
index 0000000..074ed76
--- /dev/null
+++ b/exec/vector/src/main/java/org/apache/drill/exec/util/Pointer.java
@@ -0,0 +1,28 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.util;
+
+public class Pointer<T> {
+  public T value;
+
+  public Pointer(){}
+
+  public Pointer(T value){
+    this.value = value;
+  }
+}

http://git-wip-us.apache.org/repos/asf/drill/blob/9969d8bd/exec/vector/src/main/java/org/apache/drill/exec/util/TextSerializer.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/java/org/apache/drill/exec/util/TextSerializer.java b/exec/vector/src/main/java/org/apache/drill/exec/util/TextSerializer.java
new file mode 100644
index 0000000..87df102
--- /dev/null
+++ b/exec/vector/src/main/java/org/apache/drill/exec/util/TextSerializer.java
@@ -0,0 +1,39 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.util;
+
+import java.io.IOException;
+
+import org.apache.hadoop.io.Text;
+
+import com.fasterxml.jackson.core.JsonGenerationException;
+import com.fasterxml.jackson.core.JsonGenerator;
+import com.fasterxml.jackson.databind.SerializerProvider;
+import com.fasterxml.jackson.databind.ser.std.StdSerializer;
+
+public class TextSerializer extends StdSerializer<Text> {
+
+  public TextSerializer() {
+    super(Text.class);
+  }
+
+  @Override
+  public void serialize(Text text, JsonGenerator jsonGenerator, SerializerProvider serializerProvider) throws IOException, JsonGenerationException {
+    jsonGenerator.writeString(text.toString());
+  }
+}

http://git-wip-us.apache.org/repos/asf/drill/blob/9969d8bd/exec/vector/src/main/java/org/apache/drill/exec/vector/AddOrGetResult.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/java/org/apache/drill/exec/vector/AddOrGetResult.java b/exec/vector/src/main/java/org/apache/drill/exec/vector/AddOrGetResult.java
new file mode 100644
index 0000000..7d1f08d
--- /dev/null
+++ b/exec/vector/src/main/java/org/apache/drill/exec/vector/AddOrGetResult.java
@@ -0,0 +1,38 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.vector;
+
+import com.google.common.base.Preconditions;
+
+public class AddOrGetResult<V extends ValueVector> {
+  private final V vector;
+  private final boolean created;
+
+  public AddOrGetResult(V vector, boolean created) {
+    this.vector = Preconditions.checkNotNull(vector);
+    this.created = created;
+  }
+
+  public V getVector() {
+    return vector;
+  }
+
+  public boolean isCreated() {
+    return created;
+  }
+}

http://git-wip-us.apache.org/repos/asf/drill/blob/9969d8bd/exec/vector/src/main/java/org/apache/drill/exec/vector/AllocationHelper.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/java/org/apache/drill/exec/vector/AllocationHelper.java b/exec/vector/src/main/java/org/apache/drill/exec/vector/AllocationHelper.java
new file mode 100644
index 0000000..622e2d2
--- /dev/null
+++ b/exec/vector/src/main/java/org/apache/drill/exec/vector/AllocationHelper.java
@@ -0,0 +1,61 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.vector;
+
+import org.apache.drill.exec.vector.complex.RepeatedFixedWidthVectorLike;
+import org.apache.drill.exec.vector.complex.RepeatedVariableWidthVectorLike;
+
+public class AllocationHelper {
+//  private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(AllocationHelper.class);
+
+  public static void allocate(ValueVector v, int valueCount, int bytesPerValue) {
+    allocate(v, valueCount, bytesPerValue, 5);
+  }
+
+  public static void allocatePrecomputedChildCount(ValueVector v, int valueCount, int bytesPerValue, int childValCount){
+    if(v instanceof FixedWidthVector) {
+      ((FixedWidthVector) v).allocateNew(valueCount);
+    } else if (v instanceof VariableWidthVector) {
+      ((VariableWidthVector) v).allocateNew(valueCount * bytesPerValue, valueCount);
+    } else if(v instanceof RepeatedFixedWidthVectorLike) {
+      ((RepeatedFixedWidthVectorLike) v).allocateNew(valueCount, childValCount);
+    } else if(v instanceof RepeatedVariableWidthVectorLike) {
+      ((RepeatedVariableWidthVectorLike) v).allocateNew(childValCount * bytesPerValue, valueCount, childValCount);
+    } else {
+      v.allocateNew();
+    }
+  }
+
+  public static void allocate(ValueVector v, int valueCount, int bytesPerValue, int repeatedPerTop){
+    allocatePrecomputedChildCount(v, valueCount, bytesPerValue, repeatedPerTop * valueCount);
+  }
+
+  /**
+   * Allocates the exact amount if v is fixed width, otherwise falls back to dynamic allocation
+   * @param v value vector we are trying to allocate
+   * @param valueCount  size we are trying to allocate
+   * @throws org.apache.drill.exec.memory.OutOfMemoryRuntimeException if it can't allocate the memory
+   */
+  public static void allocateNew(ValueVector v, int valueCount) {
+    if (v instanceof  FixedWidthVector) {
+      ((FixedWidthVector) v).allocateNew(valueCount);
+    } else {
+      v.allocateNew();
+    }
+  }
+}

http://git-wip-us.apache.org/repos/asf/drill/blob/9969d8bd/exec/vector/src/main/java/org/apache/drill/exec/vector/BaseDataValueVector.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/java/org/apache/drill/exec/vector/BaseDataValueVector.java b/exec/vector/src/main/java/org/apache/drill/exec/vector/BaseDataValueVector.java
new file mode 100644
index 0000000..2fc741c
--- /dev/null
+++ b/exec/vector/src/main/java/org/apache/drill/exec/vector/BaseDataValueVector.java
@@ -0,0 +1,90 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.vector;
+
+import io.netty.buffer.DrillBuf;
+import org.apache.drill.exec.memory.BufferAllocator;
+import org.apache.drill.exec.record.MaterializedField;
+
+
+public abstract class BaseDataValueVector extends BaseValueVector {
+
+  protected final static byte[] emptyByteArray = new byte[]{}; // Nullable vectors use this
+
+  protected DrillBuf data;
+
+  public BaseDataValueVector(MaterializedField field, BufferAllocator allocator) {
+    super(field, allocator);
+    data = allocator.getEmpty();
+  }
+
+  @Override
+  public void clear() {
+    if (data != null) {
+      data.release();
+    }
+    data = allocator.getEmpty();
+    super.clear();
+  }
+
+  @Override
+  public void close() {
+    clear();
+    if (data != null) {
+      data.release();
+      data = null;
+    }
+    super.close();
+  }
+
+  @Override
+  public DrillBuf[] getBuffers(boolean clear) {
+    DrillBuf[] out;
+    if (getBufferSize() == 0) {
+      out = new DrillBuf[0];
+    } else {
+      out = new DrillBuf[]{data};
+      if (clear) {
+        data.readerIndex(0);
+        data.retain(1);
+      }
+    }
+    if (clear) {
+      clear();
+    }
+    return out;
+  }
+
+  @Override
+  public int getBufferSize() {
+    if (getAccessor().getValueCount() == 0) {
+      return 0;
+    }
+    return data.writerIndex();
+  }
+
+  public DrillBuf getBuffer() {
+    return data;
+  }
+
+  /**
+   * This method has a similar effect of allocateNew() without actually clearing and reallocating
+   * the value vector. The purpose is to move the value vector to a "mutate" state
+   */
+  public void reset() {}
+}

http://git-wip-us.apache.org/repos/asf/drill/blob/9969d8bd/exec/vector/src/main/java/org/apache/drill/exec/vector/BaseValueVector.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/java/org/apache/drill/exec/vector/BaseValueVector.java b/exec/vector/src/main/java/org/apache/drill/exec/vector/BaseValueVector.java
new file mode 100644
index 0000000..eb5dbcd
--- /dev/null
+++ b/exec/vector/src/main/java/org/apache/drill/exec/vector/BaseValueVector.java
@@ -0,0 +1,123 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.vector;
+
+import io.netty.buffer.DrillBuf;
+
+import java.util.Iterator;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Iterators;
+
+import org.apache.drill.common.expression.FieldReference;
+import org.apache.drill.exec.memory.BufferAllocator;
+import org.apache.drill.exec.proto.UserBitShared.SerializedField;
+import org.apache.drill.exec.record.MaterializedField;
+import org.apache.drill.exec.record.TransferPair;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public abstract class BaseValueVector implements ValueVector {
+  private static final Logger logger = LoggerFactory.getLogger(BaseValueVector.class);
+
+  public static final int MAX_ALLOCATION_SIZE = Integer.MAX_VALUE;
+  public static final int INITIAL_VALUE_ALLOCATION = 4096;
+
+  protected final BufferAllocator allocator;
+  protected final MaterializedField field;
+
+  protected BaseValueVector(MaterializedField field, BufferAllocator allocator) {
+    this.field = Preconditions.checkNotNull(field, "field cannot be null");
+    this.allocator = Preconditions.checkNotNull(allocator, "allocator cannot be null");
+  }
+
+  @Override
+  public String toString() {
+    return super.toString() + "[field = " + field + ", ...]";
+  }
+
+  @Override
+  public void clear() {
+    getMutator().reset();
+  }
+
+  @Override
+  public void close() {
+    clear();
+  }
+
+  @Override
+  public MaterializedField getField() {
+    return field;
+  }
+
+  public MaterializedField getField(FieldReference ref){
+    return getField().withPath(ref);
+  }
+
+  @Override
+  public TransferPair getTransferPair() {
+    return getTransferPair(new FieldReference(getField().getPath()));
+  }
+
+  @Override
+  public SerializedField getMetadata() {
+    return getMetadataBuilder().build();
+  }
+
+  protected SerializedField.Builder getMetadataBuilder() {
+    return getField().getAsBuilder()
+        .setValueCount(getAccessor().getValueCount())
+        .setBufferLength(getBufferSize());
+  }
+
+  public abstract static class BaseAccessor implements ValueVector.Accessor {
+    protected BaseAccessor() { }
+
+    @Override
+    public boolean isNull(int index) {
+      return false;
+    }
+  }
+
+  public abstract static class BaseMutator implements ValueVector.Mutator {
+    protected BaseMutator() { }
+
+    @Override
+    public void generateTestData(int values) {}
+
+    //TODO: consider making mutator stateless(if possible) on another issue.
+    public void reset() {}
+  }
+
+  @Override
+  public Iterator<ValueVector> iterator() {
+    return Iterators.emptyIterator();
+  }
+
+  public static boolean checkBufRefs(final ValueVector vv) {
+    for(final DrillBuf buffer : vv.getBuffers(false)) {
+      if (buffer.refCnt() <= 0) {
+        throw new IllegalStateException("zero refcount");
+      }
+    }
+
+    return true;
+  }
+}
+

http://git-wip-us.apache.org/repos/asf/drill/blob/9969d8bd/exec/vector/src/main/java/org/apache/drill/exec/vector/BitVector.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/java/org/apache/drill/exec/vector/BitVector.java b/exec/vector/src/main/java/org/apache/drill/exec/vector/BitVector.java
new file mode 100644
index 0000000..67ba5c4
--- /dev/null
+++ b/exec/vector/src/main/java/org/apache/drill/exec/vector/BitVector.java
@@ -0,0 +1,448 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.vector;
+
+import com.google.common.base.Preconditions;
+import io.netty.buffer.DrillBuf;
+
+import org.apache.drill.common.expression.FieldReference;
+import org.apache.drill.exec.exception.OversizedAllocationException;
+import org.apache.drill.exec.expr.holders.BitHolder;
+import org.apache.drill.exec.expr.holders.NullableBitHolder;
+import org.apache.drill.exec.memory.BufferAllocator;
+import org.apache.drill.exec.memory.OutOfMemoryRuntimeException;
+import org.apache.drill.exec.proto.UserBitShared.SerializedField;
+import org.apache.drill.exec.record.MaterializedField;
+import org.apache.drill.exec.record.TransferPair;
+import org.apache.drill.exec.vector.complex.impl.BitReaderImpl;
+import org.apache.drill.exec.vector.complex.reader.FieldReader;
+
+/**
+ * Bit implements a vector of bit-width values. Elements in the vector are accessed by position from the logical start
+ * of the vector. The width of each element is 1 bit. The equivalent Java primitive is an int containing the value '0'
+ * or '1'.
+ */
+public final class BitVector extends BaseDataValueVector implements FixedWidthVector {
+  static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(BitVector.class);
+
+  private final FieldReader reader = new BitReaderImpl(BitVector.this);
+  private final Accessor accessor = new Accessor();
+  private final Mutator mutator = new Mutator();
+
+  private int valueCount;
+  private int allocationSizeInBytes = INITIAL_VALUE_ALLOCATION;
+  private int allocationMonitor = 0;
+
+  public BitVector(MaterializedField field, BufferAllocator allocator) {
+    super(field, allocator);
+  }
+
+  @Override
+  public FieldReader getReader() {
+    return reader;
+  }
+
+  @Override
+  public int getBufferSize() {
+    return getSizeFromCount(valueCount);
+  }
+
+  @Override
+  public int getBufferSizeFor(final int valueCount) {
+    return getSizeFromCount(valueCount);
+  }
+
+  private int getSizeFromCount(int valueCount) {
+    return (int) Math.ceil(valueCount / 8.0);
+  }
+
+  @Override
+  public int getValueCapacity() {
+    return (int)Math.min((long)Integer.MAX_VALUE, data.capacity() * 8L);
+  }
+
+  private int getByteIndex(int index) {
+    return (int) Math.floor(index / 8.0);
+  }
+
+  @Override
+  public void setInitialCapacity(final int valueCount) {
+    allocationSizeInBytes = getSizeFromCount(valueCount);
+  }
+
+  @Override
+  public void allocateNew() {
+    if (!allocateNewSafe()) {
+      throw new OutOfMemoryRuntimeException();
+    }
+  }
+
+  @Override
+  public boolean allocateNewSafe() {
+    long curAllocationSize = allocationSizeInBytes;
+    if (allocationMonitor > 10) {
+      curAllocationSize = Math.max(8, allocationSizeInBytes / 2);
+      allocationMonitor = 0;
+    } else if (allocationMonitor < -2) {
+      curAllocationSize = allocationSizeInBytes * 2L;
+      allocationMonitor = 0;
+    }
+
+    try {
+      allocateBytes(curAllocationSize);
+    } catch (OutOfMemoryRuntimeException ex) {
+      return false;
+    }
+    return true;
+  }
+
+  @Override
+  public void reset() {
+    valueCount = 0;
+    allocationSizeInBytes = INITIAL_VALUE_ALLOCATION;
+    allocationMonitor = 0;
+    zeroVector();
+    super.reset();
+  }
+
+  /**
+   * Allocate a new memory space for this vector. Must be called prior to using the ValueVector.
+   *
+   * @param valueCount
+   *          The number of values which can be contained within this vector.
+   */
+  @Override
+  public void allocateNew(int valueCount) {
+    final int size = getSizeFromCount(valueCount);
+    allocateBytes(size);
+  }
+
+  private void allocateBytes(final long size) {
+    if (size > MAX_ALLOCATION_SIZE) {
+      throw new OversizedAllocationException("Requested amount of memory is more than max allowed allocation size");
+    }
+
+    final int curSize = (int) size;
+    clear();
+    data = allocator.buffer(curSize);
+    zeroVector();
+    allocationSizeInBytes = curSize;
+  }
+
+  /**
+   * Allocate new buffer with double capacity, and copy data into the new buffer. Replace vector's buffer with new buffer, and release old one
+   */
+  public void reAlloc() {
+    final long newAllocationSize = allocationSizeInBytes * 2L;
+    if (newAllocationSize > MAX_ALLOCATION_SIZE) {
+      throw new OversizedAllocationException("Requested amount of memory is more than max allowed allocation size");
+    }
+
+    final int curSize = (int)newAllocationSize;
+    final DrillBuf newBuf = allocator.buffer(curSize);
+    newBuf.setZero(0, newBuf.capacity());
+    newBuf.setBytes(0, data, 0, data.capacity());
+    data.release();
+    data = newBuf;
+    allocationSizeInBytes = curSize;
+  }
+
+  /**
+   * {@inheritDoc}
+   */
+  @Override
+  public void zeroVector() {
+    data.setZero(0, data.capacity());
+  }
+
+  public void copyFrom(int inIndex, int outIndex, BitVector from) {
+    this.mutator.set(outIndex, from.accessor.get(inIndex));
+  }
+
+  public boolean copyFromSafe(int inIndex, int outIndex, BitVector from) {
+    if (outIndex >= this.getValueCapacity()) {
+      decrementAllocationMonitor();
+      return false;
+    }
+    copyFrom(inIndex, outIndex, from);
+    return true;
+  }
+
+  @Override
+  public void load(SerializedField metadata, DrillBuf buffer) {
+    Preconditions.checkArgument(this.field.matches(metadata), "The field %s doesn't match the provided metadata %s.", this.field, metadata);
+    final int valueCount = metadata.getValueCount();
+    final int expectedLength = getSizeFromCount(valueCount);
+    final int actualLength = metadata.getBufferLength();
+    assert expectedLength == actualLength: "expected and actual buffer sizes do not match";
+
+    clear();
+    data = buffer.slice(0, actualLength);
+    data.retain();
+    this.valueCount = valueCount;
+  }
+
+  @Override
+  public Mutator getMutator() {
+    return new Mutator();
+  }
+
+  @Override
+  public Accessor getAccessor() {
+    return new Accessor();
+  }
+
+  @Override
+  public TransferPair getTransferPair() {
+    return new TransferImpl(getField());
+  }
+
+  @Override
+  public TransferPair getTransferPair(FieldReference ref) {
+    return new TransferImpl(getField().withPath(ref));
+  }
+
+  @Override
+  public TransferPair makeTransferPair(ValueVector to) {
+    return new TransferImpl((BitVector) to);
+  }
+
+
+  public void transferTo(BitVector target) {
+    target.clear();
+    if (target.data != null) {
+      target.data.release();
+    }
+    target.data = data;
+    target.data.retain(1);
+    target.valueCount = valueCount;
+    clear();
+  }
+
+  public void splitAndTransferTo(int startIndex, int length, BitVector target) {
+    assert startIndex + length <= valueCount;
+    int firstByte = getByteIndex(startIndex);
+    int byteSize = getSizeFromCount(length);
+    int offset = startIndex % 8;
+    if (offset == 0) {
+      target.clear();
+      // slice
+      if (target.data != null) {
+        target.data.release();
+      }
+      target.data = (DrillBuf) data.slice(firstByte, byteSize);
+      target.data.retain(1);
+    } else {
+      // Copy data
+      // When the first bit starts from the middle of a byte (offset != 0), copy data from src BitVector.
+      // Each byte in the target is composed by a part in i-th byte, another part in (i+1)-th byte.
+      // The last byte copied to target is a bit tricky :
+      //   1) if length requires partly byte (length % 8 !=0), copy the remaining bits only.
+      //   2) otherwise, copy the last byte in the same way as to the prior bytes.
+      target.clear();
+      target.allocateNew(length);
+      // TODO maybe do this one word at a time, rather than byte?
+      for(int i = 0; i < byteSize - 1; i++) {
+        target.data.setByte(i, (((this.data.getByte(firstByte + i) & 0xFF) >>> offset) + (this.data.getByte(firstByte + i + 1) <<  (8 - offset))));
+      }
+      if (length % 8 != 0) {
+        target.data.setByte(byteSize - 1, ((this.data.getByte(firstByte + byteSize - 1) & 0xFF) >>> offset));
+      } else {
+        target.data.setByte(byteSize - 1,
+            (((this.data.getByte(firstByte + byteSize - 1) & 0xFF) >>> offset) + (this.data.getByte(firstByte + byteSize) <<  (8 - offset))));
+      }
+    }
+    target.getMutator().setValueCount(length);
+  }
+
+  private class TransferImpl implements TransferPair {
+    BitVector to;
+
+    public TransferImpl(MaterializedField field) {
+      this.to = new BitVector(field, allocator);
+    }
+
+    public TransferImpl(BitVector to) {
+      this.to = to;
+    }
+
+    @Override
+    public BitVector getTo() {
+      return to;
+    }
+
+    @Override
+    public void transfer() {
+      transferTo(to);
+    }
+
+    @Override
+    public void splitAndTransfer(int startIndex, int length) {
+      splitAndTransferTo(startIndex, length, to);
+    }
+
+    @Override
+    public void copyValueSafe(int fromIndex, int toIndex) {
+      to.copyFromSafe(fromIndex, toIndex, BitVector.this);
+    }
+  }
+
+  private void decrementAllocationMonitor() {
+    if (allocationMonitor > 0) {
+      allocationMonitor = 0;
+    }
+    --allocationMonitor;
+  }
+
+  private void incrementAllocationMonitor() {
+    ++allocationMonitor;
+  }
+
+  public class Accessor extends BaseAccessor {
+
+    /**
+     * Get the byte holding the desired bit, then mask all other bits. Iff the result is 0, the bit was not set.
+     *
+     * @param index
+     *          position of the bit in the vector
+     * @return 1 if set, otherwise 0
+     */
+    public final int get(int index) {
+      int byteIndex = index >> 3;
+      byte b = data.getByte(byteIndex);
+      int bitIndex = index & 7;
+      return Long.bitCount(b &  (1L << bitIndex));
+    }
+
+    @Override
+    public boolean isNull(int index) {
+      return false;
+    }
+
+    @Override
+    public final Boolean getObject(int index) {
+      return new Boolean(get(index) != 0);
+    }
+
+    @Override
+    public final int getValueCount() {
+      return valueCount;
+    }
+
+    public final void get(int index, BitHolder holder) {
+      holder.value = get(index);
+    }
+
+    public final void get(int index, NullableBitHolder holder) {
+      holder.isSet = 1;
+      holder.value = get(index);
+    }
+  }
+
+  /**
+   * MutableBit implements a vector of bit-width values. Elements in the vector are accessed by position from the
+   * logical start of the vector. Values should be pushed onto the vector sequentially, but may be randomly accessed.
+   *
+   * NB: this class is automatically generated from ValueVectorTypes.tdd using FreeMarker.
+   */
+  public class Mutator extends BaseMutator {
+
+    private Mutator() {
+    }
+
+    /**
+     * Set the bit at the given index to the specified value.
+     *
+     * @param index
+     *          position of the bit to set
+     * @param value
+     *          value to set (either 1 or 0)
+     */
+    public final void set(int index, int value) {
+      int byteIndex = index >> 3;
+      int bitIndex = index & 7;
+      byte currentByte = data.getByte(byteIndex);
+      byte bitMask = (byte) (1L << bitIndex);
+      if (value != 0) {
+        currentByte |= bitMask;
+      } else {
+        currentByte -= (bitMask & currentByte);
+      }
+
+      data.setByte(byteIndex, currentByte);
+    }
+
+    public final void set(int index, BitHolder holder) {
+      set(index, holder.value);
+    }
+
+    final void set(int index, NullableBitHolder holder) {
+      set(index, holder.value);
+    }
+
+    public void setSafe(int index, int value) {
+      while(index >= getValueCapacity()) {
+        reAlloc();
+      }
+      set(index, value);
+    }
+
+    public void setSafe(int index, BitHolder holder) {
+      while(index >= getValueCapacity()) {
+        reAlloc();
+      }
+      set(index, holder.value);
+    }
+
+    public void setSafe(int index, NullableBitHolder holder) {
+      while(index >= getValueCapacity()) {
+        reAlloc();
+      }
+      set(index, holder.value);
+    }
+
+    @Override
+    public final void setValueCount(int valueCount) {
+      int currentValueCapacity = getValueCapacity();
+      BitVector.this.valueCount = valueCount;
+      int idx = getSizeFromCount(valueCount);
+      while(valueCount > getValueCapacity()) {
+        reAlloc();
+      }
+      if (valueCount > 0 && currentValueCapacity > valueCount * 2) {
+        incrementAllocationMonitor();
+      } else if (allocationMonitor > 0) {
+        allocationMonitor = 0;
+      }
+      VectorTrimmer.trim(data, idx);
+    }
+
+    @Override
+    public final void generateTestData(int values) {
+      boolean even = true;
+      for(int i = 0; i < values; i++, even = !even) {
+        if (even) {
+          set(i, 1);
+        }
+      }
+      setValueCount(values);
+    }
+
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/drill/blob/9969d8bd/exec/vector/src/main/java/org/apache/drill/exec/vector/FixedWidthVector.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/java/org/apache/drill/exec/vector/FixedWidthVector.java b/exec/vector/src/main/java/org/apache/drill/exec/vector/FixedWidthVector.java
new file mode 100644
index 0000000..c2781eb
--- /dev/null
+++ b/exec/vector/src/main/java/org/apache/drill/exec/vector/FixedWidthVector.java
@@ -0,0 +1,35 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.vector;
+
+
+public interface FixedWidthVector extends ValueVector{
+
+  /**
+   * Allocate a new memory space for this vector.  Must be called prior to using the ValueVector.
+   *
+   * @param valueCount   Number of values in the vector.
+   */
+  void allocateNew(int valueCount);
+
+/**
+ * Zero out the underlying buffer backing this vector.
+ */
+  void zeroVector();
+
+}

http://git-wip-us.apache.org/repos/asf/drill/blob/9969d8bd/exec/vector/src/main/java/org/apache/drill/exec/vector/NullableVector.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/java/org/apache/drill/exec/vector/NullableVector.java b/exec/vector/src/main/java/org/apache/drill/exec/vector/NullableVector.java
new file mode 100644
index 0000000..8091c4c
--- /dev/null
+++ b/exec/vector/src/main/java/org/apache/drill/exec/vector/NullableVector.java
@@ -0,0 +1,23 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.vector;
+
+public interface NullableVector extends ValueVector{
+
+  ValueVector getValuesVector();
+}

http://git-wip-us.apache.org/repos/asf/drill/blob/9969d8bd/exec/vector/src/main/java/org/apache/drill/exec/vector/NullableVectorDefinitionSetter.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/java/org/apache/drill/exec/vector/NullableVectorDefinitionSetter.java b/exec/vector/src/main/java/org/apache/drill/exec/vector/NullableVectorDefinitionSetter.java
new file mode 100644
index 0000000..c418751
--- /dev/null
+++ b/exec/vector/src/main/java/org/apache/drill/exec/vector/NullableVectorDefinitionSetter.java
@@ -0,0 +1,23 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.vector;
+
+public interface NullableVectorDefinitionSetter {
+
+  public void setIndexDefined(int index);
+}

http://git-wip-us.apache.org/repos/asf/drill/blob/9969d8bd/exec/vector/src/main/java/org/apache/drill/exec/vector/ObjectVector.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/java/org/apache/drill/exec/vector/ObjectVector.java b/exec/vector/src/main/java/org/apache/drill/exec/vector/ObjectVector.java
new file mode 100644
index 0000000..a7a3609
--- /dev/null
+++ b/exec/vector/src/main/java/org/apache/drill/exec/vector/ObjectVector.java
@@ -0,0 +1,222 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.vector;
+
+import io.netty.buffer.DrillBuf;
+
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+
+import org.apache.drill.common.expression.FieldReference;
+import org.apache.drill.exec.expr.holders.ObjectHolder;
+import org.apache.drill.exec.memory.BufferAllocator;
+import org.apache.drill.exec.memory.OutOfMemoryRuntimeException;
+import org.apache.drill.exec.proto.UserBitShared;
+import org.apache.drill.exec.record.MaterializedField;
+import org.apache.drill.exec.record.TransferPair;
+import org.apache.drill.exec.vector.complex.reader.FieldReader;
+
+public class ObjectVector extends BaseValueVector {
+  private final Accessor accessor = new Accessor();
+  private final Mutator mutator = new Mutator();
+  private int maxCount = 0;
+  private int count = 0;
+  private int allocationSize = 4096;
+
+  private List<Object[]> objectArrayList = new ArrayList<>();
+
+  public ObjectVector(MaterializedField field, BufferAllocator allocator) {
+    super(field, allocator);
+  }
+
+  public void addNewArray() {
+    objectArrayList.add(new Object[allocationSize]);
+    maxCount += allocationSize;
+  }
+
+  @Override
+  public FieldReader getReader() {
+    throw new UnsupportedOperationException("ObjectVector does not support this");
+  }
+
+  public final class Mutator implements ValueVector.Mutator {
+
+    public void set(int index, Object obj) {
+      int listOffset = index / allocationSize;
+      if (listOffset >= objectArrayList.size()) {
+        addNewArray();
+      }
+      objectArrayList.get(listOffset)[index % allocationSize] = obj;
+    }
+
+    public boolean setSafe(int index, long value) {
+      set(index, value);
+      return true;
+    }
+
+    protected void set(int index, ObjectHolder holder) {
+      set(index, holder.obj);
+    }
+
+    public boolean setSafe(int index, ObjectHolder holder){
+      set(index, holder);
+      return true;
+    }
+
+    @Override
+    public void setValueCount(int valueCount) {
+      count = valueCount;
+    }
+
+    @Override
+    public void reset() {
+      count = 0;
+      maxCount = 0;
+      objectArrayList = new ArrayList<>();
+      addNewArray();
+    }
+
+    @Override
+    public void generateTestData(int values) {
+    }
+  }
+
+  @Override
+  public void setInitialCapacity(int numRecords) {
+    // NoOp
+  }
+
+  @Override
+  public void allocateNew() throws OutOfMemoryRuntimeException {
+    addNewArray();
+  }
+
+  public void allocateNew(int valueCount) throws OutOfMemoryRuntimeException {
+    while (maxCount < valueCount) {
+      addNewArray();
+    }
+  }
+
+  @Override
+  public boolean allocateNewSafe() {
+    allocateNew();
+    return true;
+  }
+
+  @Override
+  public int getBufferSize() {
+    throw new UnsupportedOperationException("ObjectVector does not support this");
+  }
+
+  @Override
+  public int getBufferSizeFor(final int valueCount) {
+    throw new UnsupportedOperationException("ObjectVector does not support this");
+  }
+
+  @Override
+  public void close() {
+    clear();
+  }
+
+  @Override
+  public void clear() {
+    objectArrayList.clear();
+    maxCount = 0;
+    count = 0;
+  }
+
+  @Override
+  public MaterializedField getField() {
+    return field;
+  }
+
+  @Override
+  public TransferPair getTransferPair() {
+    throw new UnsupportedOperationException("ObjectVector does not support this");
+  }
+
+  @Override
+  public TransferPair makeTransferPair(ValueVector to) {
+    throw new UnsupportedOperationException("ObjectVector does not support this");
+  }
+
+  @Override
+  public TransferPair getTransferPair(FieldReference ref) {
+    throw new UnsupportedOperationException("ObjectVector does not support this");
+  }
+
+  @Override
+  public int getValueCapacity() {
+    return maxCount;
+  }
+
+  @Override
+  public Accessor getAccessor() {
+    return accessor;
+  }
+
+  @Override
+  public DrillBuf[] getBuffers(boolean clear) {
+    throw new UnsupportedOperationException("ObjectVector does not support this");
+  }
+
+  @Override
+  public void load(UserBitShared.SerializedField metadata, DrillBuf buffer) {
+    throw new UnsupportedOperationException("ObjectVector does not support this");
+  }
+
+  @Override
+  public UserBitShared.SerializedField getMetadata() {
+    throw new UnsupportedOperationException("ObjectVector does not support this");
+  }
+
+  @Override
+  public Mutator getMutator() {
+    return mutator;
+  }
+
+  @Override
+  public Iterator<ValueVector> iterator() {
+    throw new UnsupportedOperationException("ObjectVector does not support this");
+  }
+
+  public final class Accessor extends BaseAccessor {
+    @Override
+    public Object getObject(int index) {
+      int listOffset = index / allocationSize;
+      if (listOffset >= objectArrayList.size()) {
+        addNewArray();
+      }
+      return objectArrayList.get(listOffset)[index % allocationSize];
+    }
+
+    @Override
+    public int getValueCount() {
+      return count;
+    }
+
+    public Object get(int index) {
+      return getObject(index);
+    }
+
+    public void get(int index, ObjectHolder holder){
+      holder.obj = getObject(index);
+    }
+  }
+}

http://git-wip-us.apache.org/repos/asf/drill/blob/9969d8bd/exec/vector/src/main/java/org/apache/drill/exec/vector/SchemaChangeCallBack.java
----------------------------------------------------------------------
diff --git a/exec/vector/src/main/java/org/apache/drill/exec/vector/SchemaChangeCallBack.java b/exec/vector/src/main/java/org/apache/drill/exec/vector/SchemaChangeCallBack.java
new file mode 100644
index 0000000..4c2491c
--- /dev/null
+++ b/exec/vector/src/main/java/org/apache/drill/exec/vector/SchemaChangeCallBack.java
@@ -0,0 +1,52 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.drill.exec.vector;
+
+import org.apache.drill.exec.util.CallBack;
+
+
+public class SchemaChangeCallBack implements CallBack {
+  private boolean schemaChanged = false;
+
+  /**
+   * Constructs a schema-change callback with the schema-changed state set to
+   * {@code false}.
+   */
+  public SchemaChangeCallBack() {
+  }
+
+  /**
+   * Sets the schema-changed state to {@code true}.
+   */
+  @Override
+  public void doWork() {
+    schemaChanged = true;
+  }
+
+  /**
+   * Returns the value of schema-changed state, <strong>resetting</strong> the
+   * schema-changed state to {@code false}.
+   */
+  public boolean getSchemaChangedAndReset() {
+    final boolean current = schemaChanged;
+    schemaChanged = false;
+    return current;
+  }
+}
+


Mime
View raw message