drill-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From meh...@apache.org
Subject [2/2] drill git commit: DRILL-2142: Refactor complex vector classes. Move the logic that applies specifically to maps in a separate base class that will be used by RepeatedMap and Map vectors.
Date Thu, 05 Feb 2015 20:19:21 GMT
DRILL-2142: Refactor complex vector classes. Move the logic that applies specifically to maps
in a separate base class that will be used by RepeatedMap and Map vectors.


Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/9d088f53
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/9d088f53
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/9d088f53

Branch: refs/heads/master
Commit: 9d088f53c7b817f2d8fd5a8a6c30d2aa605b0aa3
Parents: a0d1006
Author: Mehant Baid <mehantr@gmail.com>
Authored: Mon Feb 2 13:42:04 2015 -0800
Committer: Mehant Baid <mehantr@gmail.com>
Committed: Thu Feb 5 10:34:37 2015 -0800

----------------------------------------------------------------------
 .../vector/complex/AbstractContainerVector.java | 197 +---------------
 .../exec/vector/complex/AbstractMapVector.java  | 225 +++++++++++++++++++
 .../drill/exec/vector/complex/MapVector.java    |   2 +-
 .../exec/vector/complex/RepeatedMapVector.java  |   2 +-
 4 files changed, 238 insertions(+), 188 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/drill/blob/9d088f53/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/AbstractContainerVector.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/AbstractContainerVector.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/AbstractContainerVector.java
index fdfddfb..3bc7d11 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/AbstractContainerVector.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/AbstractContainerVector.java
@@ -18,23 +18,17 @@
 package org.apache.drill.exec.vector.complex;
 
 import java.util.Collection;
-import java.util.Iterator;
-import java.util.List;
 
 import javax.annotation.Nullable;
 
 import com.google.common.base.Function;
 import com.google.common.base.Preconditions;
 import com.google.common.collect.Iterables;
-import com.google.common.collect.Lists;
 import com.google.common.collect.Sets;
-import io.netty.buffer.DrillBuf;
-import org.apache.drill.common.collections.MapWithOrdinal;
 import org.apache.drill.common.expression.PathSegment;
 import org.apache.drill.common.types.TypeProtos.DataMode;
 import org.apache.drill.common.types.TypeProtos.MajorType;
 import org.apache.drill.common.types.TypeProtos.MinorType;
-import org.apache.drill.exec.expr.TypeHelper;
 import org.apache.drill.exec.memory.BufferAllocator;
 import org.apache.drill.exec.memory.OutOfMemoryRuntimeException;
 import org.apache.drill.exec.record.MaterializedField;
@@ -50,8 +44,7 @@ import org.apache.drill.exec.vector.ValueVector;
 public abstract class AbstractContainerVector implements ValueVector {
   static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(AbstractContainerVector.class);
 
-  private final MapWithOrdinal<String, ValueVector> vectors =  new MapWithOrdinal<>();
-  private final MaterializedField field;
+  protected final MaterializedField field;
   protected final BufferAllocator allocator;
   protected final CallBack callBack;
 
@@ -68,16 +61,6 @@ public abstract class AbstractContainerVector implements ValueVector {
     }
   }
 
-  @Override
-  public boolean allocateNewSafe() {
-    for (ValueVector v : vectors.values()) {
-      if (!v.allocateNewSafe()) {
-        return false;
-      }
-    }
-    return true;
-  }
-
   /**
    * Returns the field definition of this instance.
    */
@@ -87,73 +70,6 @@ public abstract class AbstractContainerVector implements ValueVector {
   }
 
   /**
-   * Adds a new field with the given parameters or replaces the existing one and consequently
returns the resultant
-   * {@link org.apache.drill.exec.vector.ValueVector}.
-   *
-   * Execution takes place in the following order:
-   * <ul>
-   *   <li>
-   *     if field is new, create and insert a new vector of desired type.
-   *   </li>
-   *   <li>
-   *     if field exists and existing vector is of desired vector type, return the vector.
-   *   </li>
-   *   <li>
-   *     if field exists and null filled, clear the existing vector; create and insert a
new vector of desired type.
-   *   </li>
-   *   <li>
-   *     otherwise, throw an {@link java.lang.IllegalStateException}
-   *   </li>
-   * </ul>
-   *
-   * @param name name of the field
-   * @param type type of the field
-   * @param clazz class of expected vector type
-   * @param <T> class type of expected vector type
-   * @throws java.lang.IllegalStateException raised if there is a hard schema change
-   *
-   * @return resultant {@link org.apache.drill.exec.vector.ValueVector}
-   */
-  public <T extends ValueVector> T addOrGet(String name, MajorType type, Class<T>
clazz) {
-    final ValueVector existing = getChild(name);
-    boolean create = false;
-    if (existing == null) {
-      create = true;
-    } else if (clazz.isAssignableFrom(existing.getClass())) {
-      return (T)existing;
-    } else if (nullFilled(existing)) {
-      existing.clear();
-      create = true;
-    }
-    if (create) {
-      final T vector = (T)TypeHelper.getNewVector(field.getPath(), name, allocator, type);
-      putChild(name, vector);
-      if (callBack!=null) {
-        callBack.doWork();
-      }
-      return vector;
-    }
-    final String message = "Drill does not support schema change yet. Existing[{}] and desired[{}]
vector types mismatch";
-    throw new IllegalStateException(String.format(message, existing.getClass().getSimpleName(),
clazz.getSimpleName()));
-  }
-
-  private boolean nullFilled(ValueVector vector) {
-    for (int r=0; r<vector.getAccessor().getValueCount(); r++) {
-      if (!vector.getAccessor().isNull(r)) {
-        return false;
-      }
-    }
-    return true;
-  }
-
-  /**
-   * Returns a {@link org.apache.drill.exec.vector.ValueVector} corresponding to the given
ordinal identifier.
-   */
-  public ValueVector getChildByOrdinal(int id) {
-    return vectors.getByOrdinal(id);
-  }
-
-  /**
    * Returns a {@link org.apache.drill.exec.vector.ValueVector} corresponding to the given
field name if exists or null.
    */
   public ValueVector getChild(String name) {
@@ -161,36 +77,6 @@ public abstract class AbstractContainerVector implements ValueVector {
   }
 
   /**
-   * Returns a {@link org.apache.drill.exec.vector.ValueVector} instance of subtype of <T>
corresponding to the given
-   * field name if exists or null.
-   */
-  public <T extends ValueVector> T getChild(String name, Class<T> clazz) {
-    ValueVector v = vectors.get(name.toLowerCase());
-    if (v == null) {
-      return null;
-    }
-    return typeify(v, clazz);
-  }
-
-  /**
-   * Inserts the vector with the given name if it does not exist else replaces it with the
new value.
-   *
-   * Note that this method does not enforce any vector type check nor throws a schema change
exception.
-   */
-  protected void putChild(String name, ValueVector vector) {
-    ValueVector old = vectors.put(
-        Preconditions.checkNotNull(name, "field name cannot be null").toLowerCase(),
-        Preconditions.checkNotNull(vector, "vector cannot be null")
-    );
-    if (old != null && old != vector) {
-      logger.debug("Field [%s] mutated from [%s] to [%s]", name, old.getClass().getSimpleName(),
-          vector.getClass().getSimpleName());
-    }
-
-    field.addChild(vector.getField());
-  }
-
-  /**
    * Returns a sequence of field names in the order that they show up in the schema.
    */
   protected Collection<String> getChildFieldNames() {
@@ -204,20 +90,6 @@ public abstract class AbstractContainerVector implements ValueVector {
   }
 
   /**
-   * Returns a sequence of underlying child vectors.
-   */
-  protected Collection<ValueVector> getChildren() {
-    return vectors.values();
-  }
-
-  /**
-   * Returns the number of underlying child vectors.
-   */
-  public int size() {
-    return vectors.size();
-  }
-
-  /**
    * Clears out all underlying child vectors.
    */
  @Override
@@ -227,11 +99,6 @@ public abstract class AbstractContainerVector implements ValueVector {
     }
   }
 
-  @Override
-  public Iterator<ValueVector> iterator() {
-    return vectors.values().iterator();
-  }
-
   protected <T extends ValueVector> T typeify(ValueVector v, Class<T> clazz)
{
     if (clazz.isAssignableFrom(v.getClass())) {
       return (T) v;
@@ -239,34 +106,6 @@ public abstract class AbstractContainerVector implements ValueVector
{
     throw new IllegalStateException(String.format("Vector requested [%s] was different than
type stored [%s].  Drill doesn't yet support hetergenous types.", clazz.getSimpleName(), v.getClass().getSimpleName()));
   }
 
-  /**
-   * Returns a list of scalar child vectors recursing the entire vector hierarchy.
-   */
-  public List<ValueVector> getPrimitiveVectors() {
-    List<ValueVector> primitiveVectors = Lists.newArrayList();
-    for (ValueVector v : vectors.values()) {
-      if (v instanceof AbstractContainerVector) {
-        AbstractContainerVector av = (AbstractContainerVector) v;
-        primitiveVectors.addAll(av.getPrimitiveVectors());
-      } else {
-        primitiveVectors.add(v);
-      }
-    }
-    return primitiveVectors;
-  }
-
-  /**
-   * Returns a vector with its corresponding ordinal mapping if field exists or null.
-   */
-  public VectorWithOrdinal getChildVectorWithOrdinal(String name) {
-    final int ordinal = vectors.getOrdinal(name.toLowerCase());
-    if (ordinal < 0) {
-      return null;
-    }
-    final ValueVector vector = vectors.getByOrdinal(ordinal);
-    return new VectorWithOrdinal(vector, ordinal);
-  }
-
   public TypedFieldId getFieldIdIfMatches(TypedFieldId.Builder builder, boolean addToBreadCrumb,
PathSegment seg) {
     if (seg == null) {
       if (addToBreadCrumb) {
@@ -360,33 +199,19 @@ public abstract class AbstractContainerVector implements ValueVector
{
     return this.getField().getType();
   }
 
-  @Override
-  public DrillBuf[] getBuffers(boolean clear) {
-    List<DrillBuf> buffers = Lists.newArrayList();
-
-    for (ValueVector v : vectors.values()) {
-      for (DrillBuf buf : v.getBuffers(clear)) {
-        buffers.add(buf);
-      }
-    }
-
-    return buffers.toArray(new DrillBuf[buffers.size()]);
+  protected boolean supportsDirectRead() {
+    return false;
   }
 
-  @Override
-  public int getBufferSize() {
-    int actualBufSize = 0 ;
+  // return the number of child vectors
+  public abstract int size();
 
-    for (ValueVector v : vectors.values()) {
-      for (DrillBuf buf : v.getBuffers(false)) {
-        actualBufSize += buf.writerIndex();
-      }
-    }
-    return actualBufSize;
-  }
+  // add a new vector with the input MajorType or return the existing vector if we already
added one with the same type
+  public abstract <T extends ValueVector> T addOrGet(String name, MajorType type, Class<T>
clazz);
 
-  protected boolean supportsDirectRead() {
-    return false;
-  }
+  // return the child vector with the input name
+  public abstract <T extends ValueVector> T getChild(String name, Class<T> clazz);
 
+  // return the child vector's ordinal in the composite container
+  public abstract VectorWithOrdinal getChildVectorWithOrdinal(String name);
 }
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/drill/blob/9d088f53/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/AbstractMapVector.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/AbstractMapVector.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/AbstractMapVector.java
new file mode 100644
index 0000000..dd1fff3
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/AbstractMapVector.java
@@ -0,0 +1,225 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.vector.complex;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Lists;
+import io.netty.buffer.DrillBuf;
+import org.apache.drill.common.collections.MapWithOrdinal;
+import org.apache.drill.common.types.TypeProtos;
+import org.apache.drill.exec.expr.TypeHelper;
+import org.apache.drill.exec.memory.BufferAllocator;
+import org.apache.drill.exec.record.MaterializedField;
+import org.apache.drill.exec.util.CallBack;
+import org.apache.drill.exec.vector.ValueVector;
+
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.List;
+
+/*
+ * Base class for MapVectors. Currently used by RepeatedMapVector and MapVector
+ */
+public abstract class AbstractMapVector extends AbstractContainerVector {
+
+  // Maintains a map with key as field name and value is the vector itself
+  private final MapWithOrdinal<String, ValueVector> vectors =  new MapWithOrdinal<>();
+
+  protected AbstractMapVector(MaterializedField field, BufferAllocator allocator, CallBack
callBack) {
+    super(field, allocator, callBack);
+  }
+
+  @Override
+  public boolean allocateNewSafe() {
+    for (ValueVector v : vectors.values()) {
+      if (!v.allocateNewSafe()) {
+        return false;
+      }
+    }
+    return true;
+  }
+
+  /**
+   * Adds a new field with the given parameters or replaces the existing one and consequently
returns the resultant
+   * {@link org.apache.drill.exec.vector.ValueVector}.
+   *
+   * Execution takes place in the following order:
+   * <ul>
+   *   <li>
+   *     if field is new, create and insert a new vector of desired type.
+   *   </li>
+   *   <li>
+   *     if field exists and existing vector is of desired vector type, return the vector.
+   *   </li>
+   *   <li>
+   *     if field exists and null filled, clear the existing vector; create and insert a
new vector of desired type.
+   *   </li>
+   *   <li>
+   *     otherwise, throw an {@link java.lang.IllegalStateException}
+   *   </li>
+   * </ul>
+   *
+   * @param name name of the field
+   * @param type type of the field
+   * @param clazz class of expected vector type
+   * @param <T> class type of expected vector type
+   * @throws java.lang.IllegalStateException raised if there is a hard schema change
+   *
+   * @return resultant {@link org.apache.drill.exec.vector.ValueVector}
+   */
+  public <T extends ValueVector> T addOrGet(String name, TypeProtos.MajorType type,
Class<T> clazz) {
+    final ValueVector existing = getChild(name);
+    boolean create = false;
+    if (existing == null) {
+      create = true;
+    } else if (clazz.isAssignableFrom(existing.getClass())) {
+      return (T)existing;
+    } else if (nullFilled(existing)) {
+      existing.clear();
+      create = true;
+    }
+    if (create) {
+      final T vector = (T) TypeHelper.getNewVector(field.getPath(), name, allocator, type);
+      putChild(name, vector);
+      if (callBack!=null) {
+        callBack.doWork();
+      }
+      return vector;
+    }
+    final String message = "Drill does not support schema change yet. Existing[{}] and desired[{}]
vector types mismatch";
+    throw new IllegalStateException(String.format(message, existing.getClass().getSimpleName(),
clazz.getSimpleName()));
+  }
+
+  private boolean nullFilled(ValueVector vector) {
+    for (int r=0; r<vector.getAccessor().getValueCount(); r++) {
+      if (!vector.getAccessor().isNull(r)) {
+        return false;
+      }
+    }
+    return true;
+  }
+
+  /**
+   * Returns a {@link org.apache.drill.exec.vector.ValueVector} corresponding to the given
ordinal identifier.
+   */
+  public ValueVector getChildByOrdinal(int id) {
+    return vectors.getByOrdinal(id);
+  }
+
+  /**
+   * Returns a {@link org.apache.drill.exec.vector.ValueVector} instance of subtype of <T>
corresponding to the given
+   * field name if exists or null.
+   */
+  public <T extends ValueVector> T getChild(String name, Class<T> clazz) {
+    ValueVector v = vectors.get(name.toLowerCase());
+    if (v == null) {
+      return null;
+    }
+    return typeify(v, clazz);
+  }
+
+  /**
+   * Inserts the vector with the given name if it does not exist else replaces it with the
new value.
+   *
+   * Note that this method does not enforce any vector type check nor throws a schema change
exception.
+   */
+  protected void putChild(String name, ValueVector vector) {
+    ValueVector old = vectors.put(
+        Preconditions.checkNotNull(name, "field name cannot be null").toLowerCase(),
+        Preconditions.checkNotNull(vector, "vector cannot be null")
+    );
+    if (old != null && old != vector) {
+      logger.debug("Field [%s] mutated from [%s] to [%s]", name, old.getClass().getSimpleName(),
+          vector.getClass().getSimpleName());
+    }
+
+    field.addChild(vector.getField());
+  }
+
+  /**
+   * Returns a sequence of underlying child vectors.
+   */
+  protected Collection<ValueVector> getChildren() {
+    return vectors.values();
+  }
+
+  /**
+   * Returns the number of underlying child vectors.
+   */
+  public int size() {
+    return vectors.size();
+  }
+
+  @Override
+  public Iterator<ValueVector> iterator() {
+    return vectors.values().iterator();
+  }
+
+  /**
+   * Returns a list of scalar child vectors recursing the entire vector hierarchy.
+   */
+  public List<ValueVector> getPrimitiveVectors() {
+    List<ValueVector> primitiveVectors = Lists.newArrayList();
+    for (ValueVector v : vectors.values()) {
+      if (v instanceof AbstractMapVector) {
+        AbstractMapVector mapVector = (AbstractMapVector) v;
+        primitiveVectors.addAll(mapVector.getPrimitiveVectors());
+      } else {
+        primitiveVectors.add(v);
+      }
+    }
+    return primitiveVectors;
+  }
+
+  /**
+   * Returns a vector with its corresponding ordinal mapping if field exists or null.
+   */
+  public VectorWithOrdinal getChildVectorWithOrdinal(String name) {
+    final int ordinal = vectors.getOrdinal(name.toLowerCase());
+    if (ordinal < 0) {
+      return null;
+    }
+    final ValueVector vector = vectors.getByOrdinal(ordinal);
+    return new VectorWithOrdinal(vector, ordinal);
+  }
+
+  @Override
+  public DrillBuf[] getBuffers(boolean clear) {
+    List<DrillBuf> buffers = Lists.newArrayList();
+
+    for (ValueVector v : vectors.values()) {
+      for (DrillBuf buf : v.getBuffers(clear)) {
+        buffers.add(buf);
+      }
+    }
+
+    return buffers.toArray(new DrillBuf[buffers.size()]);
+  }
+
+  @Override
+  public int getBufferSize() {
+    int actualBufSize = 0 ;
+
+    for (ValueVector v : vectors.values()) {
+      for (DrillBuf buf : v.getBuffers(false)) {
+        actualBufSize += buf.writerIndex();
+      }
+    }
+    return actualBufSize;
+  }
+}

http://git-wip-us.apache.org/repos/asf/drill/blob/9d088f53/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/MapVector.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/MapVector.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/MapVector.java
index 1c3cee1..fb19afa 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/MapVector.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/MapVector.java
@@ -48,7 +48,7 @@ import org.apache.drill.exec.vector.complex.reader.FieldReader;
 
 import com.google.common.base.Preconditions;
 
-public class MapVector extends AbstractContainerVector {
+public class MapVector extends AbstractMapVector {
   static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(MapVector.class);
 
   public final static MajorType TYPE = MajorType.newBuilder().setMinorType(MinorType.MAP).setMode(DataMode.REQUIRED).build();

http://git-wip-us.apache.org/repos/asf/drill/blob/9d088f53/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedMapVector.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedMapVector.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedMapVector.java
index 7e1b353..6dce363 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedMapVector.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedMapVector.java
@@ -51,7 +51,7 @@ import org.apache.drill.exec.vector.complex.reader.FieldReader;
 import com.google.common.base.Preconditions;
 import com.google.common.collect.Maps;
 
-public class RepeatedMapVector extends AbstractContainerVector implements RepeatedFixedWidthVector
{
+public class RepeatedMapVector extends AbstractMapVector implements RepeatedFixedWidthVector
{
 
   static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(RepeatedMapVector.class);
 


Mime
View raw message