drill-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From par...@apache.org
Subject drill git commit: DRILL-2366: accurately materialize schema path for complex types + test cases
Date Thu, 26 Mar 2015 00:22:09 GMT
Repository: drill
Updated Branches:
  refs/heads/master 8493713ca -> 7bded6d3c


DRILL-2366: accurately materialize schema path for complex types + test cases


Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/7bded6d3
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/7bded6d3
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/7bded6d3

Branch: refs/heads/master
Commit: 7bded6d3c0655ffa6e1f9731bd2d26c86b593700
Parents: 8493713
Author: Hanifi Gunes <hgunes@maprtech.com>
Authored: Fri Mar 20 14:17:51 2015 -0700
Committer: Parth Chandra <pchandra@maprtech.com>
Committed: Wed Mar 25 13:34:51 2015 -0700

----------------------------------------------------------------------
 .../drill/exec/record/HyperVectorWrapper.java   | 10 ++-
 .../drill/exec/record/SimpleVectorWrapper.java  | 10 ++-
 .../vector/complex/AbstractContainerVector.java |  2 -
 .../drill/exec/vector/complex/MapVector.java    |  4 --
 .../exec/vector/complex/RepeatedMapVector.java  | 22 ++++++
 .../expr/TestSchemaPathMaterialization.java     | 70 ++++++++++++++++++++
 .../resources/complex/json/repeated_list.json   |  7 ++
 .../complex/json/repeated_list_map.json         |  7 ++
 8 files changed, 121 insertions(+), 11 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/drill/blob/7bded6d3/exec/java-exec/src/main/java/org/apache/drill/exec/record/HyperVectorWrapper.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/record/HyperVectorWrapper.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/record/HyperVectorWrapper.java
index 9dbb583..f68bed1 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/record/HyperVectorWrapper.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/record/HyperVectorWrapper.java
@@ -17,10 +17,13 @@
  */
 package org.apache.drill.exec.record;
 
+import java.util.AbstractMap;
+
 import org.apache.commons.lang3.ArrayUtils;
 import org.apache.drill.common.expression.SchemaPath;
 import org.apache.drill.exec.vector.ValueVector;
 import org.apache.drill.exec.vector.complex.AbstractContainerVector;
+import org.apache.drill.exec.vector.complex.AbstractMapVector;
 import org.apache.drill.exec.vector.complex.MapVector;
 
 import com.google.common.base.Preconditions;
@@ -92,8 +95,11 @@ public class HyperVectorWrapper<T extends ValueVector> implements
VectorWrapper<
     for (ValueVector v : this.vectors) {
       ValueVector vector = v;
       for (int i = 1; i < ids.length; i++) {
-        MapVector map = (MapVector) vector;
-        vector = map.getVectorById(ids[i]);
+        final AbstractMapVector mapLike = AbstractMapVector.class.cast(vector);
+        if (mapLike == null) {
+          return null;
+        }
+        vector = mapLike.getChildByOrdinal(ids[i]);
       }
       vectors[index] = vector;
       index++;

http://git-wip-us.apache.org/repos/asf/drill/blob/7bded6d3/exec/java-exec/src/main/java/org/apache/drill/exec/record/SimpleVectorWrapper.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/record/SimpleVectorWrapper.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/record/SimpleVectorWrapper.java
index 5bd3e41..6b1d178 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/record/SimpleVectorWrapper.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/record/SimpleVectorWrapper.java
@@ -22,6 +22,7 @@ import org.apache.drill.common.expression.SchemaPath;
 import org.apache.drill.common.types.TypeProtos.DataMode;
 import org.apache.drill.exec.vector.ValueVector;
 import org.apache.drill.exec.vector.complex.AbstractContainerVector;
+import org.apache.drill.exec.vector.complex.AbstractMapVector;
 import org.apache.drill.exec.vector.complex.MapVector;
 
 public class SimpleVectorWrapper<T extends ValueVector> implements VectorWrapper<T>{
@@ -85,11 +86,14 @@ public class SimpleVectorWrapper<T extends ValueVector> implements
VectorWrapper
 
     ValueVector vector = v;
     for (int i = 1; i < ids.length; i++) {
-      MapVector map = (MapVector) vector;
-      vector = map.getVectorById(ids[i]);
+      final AbstractMapVector mapLike = AbstractMapVector.class.cast(vector);
+      if (mapLike == null) {
+        return null;
+      }
+      vector = mapLike.getChildByOrdinal(ids[i]);
     }
 
-    return new SimpleVectorWrapper<ValueVector>(vector);
+    return new SimpleVectorWrapper<>(vector);
   }
 
   @Override

http://git-wip-us.apache.org/repos/asf/drill/blob/7bded6d3/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/AbstractContainerVector.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/AbstractContainerVector.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/AbstractContainerVector.java
index 3bc7d11..acf9ec7 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/AbstractContainerVector.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/AbstractContainerVector.java
@@ -132,8 +132,6 @@ public abstract class AbstractContainerVector implements ValueVector {
           addToBreadCrumb = false;
           builder.remainder(seg);
         }
-        // this is a complex array reference, which means it doesn't correspond directly
to a vector by itself.
-        seg = seg.getChild();
       }
     } else {
       // name segment.

http://git-wip-us.apache.org/repos/asf/drill/blob/7bded6d3/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/MapVector.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/MapVector.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/MapVector.java
index b995462..54ffeb2 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/MapVector.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/MapVector.java
@@ -320,10 +320,6 @@ public class MapVector extends AbstractMapVector {
     }
   }
 
-  public ValueVector getVectorById(int id) {
-    return getChildByOrdinal(id);
-  }
-
   public class Mutator implements ValueVector.Mutator{
 
     @Override

http://git-wip-us.apache.org/repos/asf/drill/blob/7bded6d3/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedMapVector.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedMapVector.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedMapVector.java
index c7e7cba..5afc889 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedMapVector.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedMapVector.java
@@ -25,6 +25,7 @@ import java.util.Map;
 
 import org.apache.commons.lang3.ArrayUtils;
 import org.apache.drill.common.expression.FieldReference;
+import org.apache.drill.common.expression.PathSegment;
 import org.apache.drill.common.expression.SchemaPath;
 import org.apache.drill.common.types.TypeProtos.DataMode;
 import org.apache.drill.common.types.TypeProtos.MajorType;
@@ -37,6 +38,7 @@ import org.apache.drill.exec.memory.BufferAllocator;
 import org.apache.drill.exec.proto.UserBitShared.SerializedField;
 import org.apache.drill.exec.record.MaterializedField;
 import org.apache.drill.exec.record.TransferPair;
+import org.apache.drill.exec.record.TypedFieldId;
 import org.apache.drill.exec.util.CallBack;
 import org.apache.drill.exec.util.JsonStringArrayList;
 import org.apache.drill.exec.vector.AllocationHelper;
@@ -166,6 +168,26 @@ public class RepeatedMapVector extends AbstractMapVector implements RepeatedFixe
     }
   }
 
+  @Override
+  public TypedFieldId getFieldIdIfMatches(TypedFieldId.Builder builder, boolean addToBreadCrumb,
PathSegment seg) {
+    if (seg != null && seg.isArray() && !seg.isLastPath()) {
+      if (addToBreadCrumb) {
+        addToBreadCrumb = false;
+        builder.remainder(seg);
+      }
+      // skip the first array segment as there is no corresponding child vector.
+      seg = seg.getChild();
+
+      // multi-level numbered access to a repeated map is not possible so return if the next
part is also an array
+      // segment.
+      if (seg.isArray()) {
+        return null;
+      }
+    }
+
+    return super.getFieldIdIfMatches(builder, addToBreadCrumb, seg);
+  }
+
   public TransferPair getTransferPairToSingleMap(FieldReference reference) {
     return new SingleMapTransferPair(this, reference);
   }

http://git-wip-us.apache.org/repos/asf/drill/blob/7bded6d3/exec/java-exec/src/test/java/org/apache/drill/exec/expr/TestSchemaPathMaterialization.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/expr/TestSchemaPathMaterialization.java
b/exec/java-exec/src/test/java/org/apache/drill/exec/expr/TestSchemaPathMaterialization.java
new file mode 100644
index 0000000..fb4cad9
--- /dev/null
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/expr/TestSchemaPathMaterialization.java
@@ -0,0 +1,70 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.expr;
+
+import org.apache.drill.BaseTestQuery;
+import org.apache.drill.exec.proto.UserBitShared;
+import org.junit.Ignore;
+import org.junit.Test;
+
+public class TestSchemaPathMaterialization extends BaseTestQuery {
+
+  @Test
+  public void testSingleProjectionFromMultiLevelRepeatedList() throws Exception {
+    final String query = "select t.odd[2][0][0] v1 " +
+        " from cp.`complex/json/repeated_list.json` t";
+
+    testBuilder()
+        .sqlQuery(query)
+        .ordered()
+        .baselineColumns("v1")
+        .baselineValues(5L)
+        .go();
+  }
+
+  @Test
+  @Ignore("Ignored until DRILL-2375 is fixed")
+  public void testProjectionFromMultiLevelRepeatedList() throws Exception {
+    final String query = "select t.odd[0][0][0] v1, t.odd[0][0][1] v2, t.odd[0][0][2] v3,
" +
+        " t.odd[1] v4, t.odd[2][0][0] v5, t.odd[2][1][0] v6" +
+        " from cp.`complex/json/repeated_list.json` t";
+
+    testRunAndPrint(UserBitShared.QueryType.SQL, query);
+    testBuilder()
+        .sqlQuery(query)
+        .ordered()
+        .baselineColumns("v1", "v2", "v3", "v4", "v5", "v6")
+        .baselineValues(1L, null, 3L, null, 5L, null)
+        .go();
+  }
+
+  @Test
+  @Ignore("Ignored until DRILL-2375 is fixed")
+  public void testProjectionFromMultiLevelRepeatedListMap() throws Exception {
+    final String query = "select t.odd[0][0].val[0] v1, t.odd[0][0].val[0] v2, " +
+        " from cp.`complex/json/repeated_list_map.json` t";
+
+    testRunAndPrint(UserBitShared.QueryType.SQL, query);
+    testBuilder()
+        .sqlQuery(query)
+        .ordered()
+        .baselineColumns("v1", "v2")
+        .baselineValues(1L, 3L)
+        .go();
+  }
+}

http://git-wip-us.apache.org/repos/asf/drill/blob/7bded6d3/exec/java-exec/src/test/resources/complex/json/repeated_list.json
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/resources/complex/json/repeated_list.json b/exec/java-exec/src/test/resources/complex/json/repeated_list.json
new file mode 100644
index 0000000..696d069
--- /dev/null
+++ b/exec/java-exec/src/test/resources/complex/json/repeated_list.json
@@ -0,0 +1,7 @@
+{
+  "odd": [
+    [[1],[],[3]],
+    [],
+    [[5]]
+  ]
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/drill/blob/7bded6d3/exec/java-exec/src/test/resources/complex/json/repeated_list_map.json
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/resources/complex/json/repeated_list_map.json b/exec/java-exec/src/test/resources/complex/json/repeated_list_map.json
new file mode 100644
index 0000000..ef6c0ee
--- /dev/null
+++ b/exec/java-exec/src/test/resources/complex/json/repeated_list_map.json
@@ -0,0 +1,7 @@
+{
+  "odd": [
+    [{"val": [1, 3]}, {}, {"val": [5]}],
+    [],
+    [{"val": [7]}]
+  ]
+}
\ No newline at end of file


Mime
View raw message