drill-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jacq...@apache.org
Subject [09/16] incubator-drill git commit: DRILL-1671, DRILL-1653, DRILL-1652: Fixes for flatten bugs
Date Wed, 12 Nov 2014 04:11:26 GMT
DRILL-1671, DRILL-1653, DRILL-1652: Fixes for flatten bugs


Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/60736dea
Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/60736dea
Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/60736dea

Branch: refs/heads/master
Commit: 60736dea0fc7099f3773616c1995df4789221cdb
Parents: ed96249
Author: Jacques Nadeau <jacques@apache.org>
Authored: Mon Nov 10 22:16:19 2014 -0800
Committer: Jacques Nadeau <jacques@apache.org>
Committed: Tue Nov 11 16:48:45 2014 -0800

----------------------------------------------------------------------
 .../drill/common/expression/SchemaPath.java     |   3 +-
 .../codegen/templates/RepeatedValueVectors.java |  30 +--
 .../exec/planner/physical/DrillFlattenPrel.java |  75 -------
 .../exec/planner/physical/FlattenPrel.java      |  80 ++++++++
 .../visitor/RewriteProjectToFlatten.java        |   4 +-
 .../visitor/SplitUpComplexExpressions.java      |   2 +-
 .../drill/exec/record/MaterializedField.java    |  22 +-
 .../exec/vector/complex/RepeatedMapVector.java  |  35 +++-
 .../exec/vector/complex/fn/JsonReader.java      |   3 +-
 .../exec/physical/impl/flatten/TestFlatten.java |  51 +++++
 .../src/test/resources/flatten/empty-rm.json    |   7 +
 .../test/resources/flatten/many-arrays-50.json  | 203 +++++++++++++++++++
 .../src/test/resources/flatten/missing-map.json |  11 +
 .../flatten/single-user-transactions.json       |  18 ++
 14 files changed, 445 insertions(+), 99 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/60736dea/common/src/main/java/org/apache/drill/common/expression/SchemaPath.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/expression/SchemaPath.java b/common/src/main/java/org/apache/drill/common/expression/SchemaPath.java
index 3b31a15..1d7f922 100644
--- a/common/src/main/java/org/apache/drill/common/expression/SchemaPath.java
+++ b/common/src/main/java/org/apache/drill/common/expression/SchemaPath.java
@@ -233,8 +233,7 @@ public class SchemaPath extends LogicalExpressionBase {
 
   @Override
   public String toString() {
-    String expr = ExpressionStringBuilder.toString(this);
-    return "SchemaPath ["+ expr + "]";
+    return ExpressionStringBuilder.toString(this);
   }
 
   public String toExpr() {

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/60736dea/exec/java-exec/src/main/codegen/templates/RepeatedValueVectors.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/codegen/templates/RepeatedValueVectors.java b/exec/java-exec/src/main/codegen/templates/RepeatedValueVectors.java
index 2853e83..0f86db9 100644
--- a/exec/java-exec/src/main/codegen/templates/RepeatedValueVectors.java
+++ b/exec/java-exec/src/main/codegen/templates/RepeatedValueVectors.java
@@ -18,6 +18,7 @@
 
 import java.lang.Override;
 
+import org.apache.drill.exec.record.TransferPair;
 import org.apache.drill.exec.vector.RepeatedFixedWidthVector;
 import org.apache.drill.exec.vector.UInt4Vector;
 import org.mortbay.jetty.servlet.Holder;
@@ -102,20 +103,25 @@ public final class Repeated${minor.class}Vector extends BaseValueVector
implemen
     clear();
   }
 
-  public void splitAndTransferTo(int startIndex, int length, Repeated${minor.class}Vector
target) {
-    int startPos = offsets.getAccessor().get(startIndex);
-    int endPos = offsets.getAccessor().get(startIndex+length);
-    values.splitAndTransferTo(startIndex, endPos-startPos, target.values);
-    target.offsets.clear();
-    target.offsets.allocateNew(endPos - startPos + 1);
+  public void splitAndTransferTo(final int startIndex, final int groups, Repeated${minor.class}Vector
to) {
+    final UInt4Vector.Accessor a = offsets.getAccessor();
+    final UInt4Vector.Mutator m = to.offsets.getMutator();
+    
+    final int startPos = offsets.getAccessor().get(startIndex);
+    final int endPos = offsets.getAccessor().get(startIndex + groups);
+    final int valuesToCopy = endPos - startPos;
+    
+    values.splitAndTransferTo(startPos, valuesToCopy, to.values);
+    to.offsets.clear();
+    to.offsets.allocateNew(valuesToCopy + 1);
     int normalizedPos = 0;
-    for (int i=0; i<length+1;i++) {
-      normalizedPos = offsets.getAccessor().get(startIndex+i) - startPos;
-      target.offsets.getMutator().set(i, normalizedPos);
+    for (int i=0; i < groups + 1;i++ ) {
+      normalizedPos = a.get(startIndex+i) - startPos;
+      m.set(i, normalizedPos);
     }
-    target.parentValueCount = length;
-    target.childValueCount  = offsets.getAccessor().get(startIndex+length) - startPos;
-    target.offsets.getMutator().setValueCount(length);
+    to.parentValueCount = groups;
+    to.childValueCount  = valuesToCopy;
+    m.setValueCount(groups);
   }
   
   private class TransferImpl implements TransferPair{

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/60736dea/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/DrillFlattenPrel.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/DrillFlattenPrel.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/DrillFlattenPrel.java
deleted file mode 100644
index 1409347..0000000
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/DrillFlattenPrel.java
+++ /dev/null
@@ -1,75 +0,0 @@
-/*******************************************************************************
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- ******************************************************************************/
-package org.apache.drill.exec.planner.physical;
-
-import org.apache.drill.common.expression.LogicalExpression;
-import org.apache.drill.common.expression.PathSegment;
-import org.apache.drill.common.expression.SchemaPath;
-import org.apache.drill.exec.physical.base.PhysicalOperator;
-import org.apache.drill.exec.physical.config.FlattenPOP;
-import org.apache.drill.exec.planner.common.DrillFilterRelBase;
-import org.apache.drill.exec.planner.logical.DrillOptiq;
-import org.apache.drill.exec.planner.logical.DrillParseContext;
-import org.apache.drill.exec.planner.physical.visitor.PrelVisitor;
-import org.apache.drill.exec.record.BatchSchema;
-import org.eigenbase.rel.RelNode;
-import org.eigenbase.relopt.RelOptCluster;
-import org.eigenbase.relopt.RelTraitSet;
-import org.eigenbase.rex.RexNode;
-
-import java.io.IOException;
-import java.util.Iterator;
-import java.util.List;
-
-public class DrillFlattenPrel extends SinglePrel implements Prel {
-
-  RexNode toFlatten;
-
-  public DrillFlattenPrel(RelOptCluster cluster, RelTraitSet traits, RelNode child, RexNode
toFlatten) {
-    super(cluster, traits, child);
-    this.toFlatten = toFlatten;
-  }
-
-  @Override
-  public RelNode copy(RelTraitSet traitSet, List<RelNode> inputs) {
-    return new DrillFlattenPrel(getCluster(), traitSet, sole(inputs), toFlatten);
-  }
-
-  @Override
-  public Iterator<Prel> iterator() {
-    return PrelUtil.iter(getChild());
-  }
-
-  @Override
-  public PhysicalOperator getPhysicalOperator(PhysicalPlanCreator creator) throws IOException
{
-    Prel child = (Prel) this.getChild();
-
-    PhysicalOperator childPOP = child.getPhysicalOperator(creator);
-    FlattenPOP f = new FlattenPOP(childPOP, (SchemaPath) getFlattenExpression(new DrillParseContext()));
-    return creator.addMetadata(this, f);
-  }
-
-  @Override
-  public BatchSchema.SelectionVectorMode getEncoding() {
-    return BatchSchema.SelectionVectorMode.NONE;
-  }
-
-  protected LogicalExpression getFlattenExpression(DrillParseContext context){
-    return DrillOptiq.toDrill(context, getChild(), toFlatten);
-  }
-}

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/60736dea/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/FlattenPrel.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/FlattenPrel.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/FlattenPrel.java
new file mode 100644
index 0000000..18a2cc2
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/FlattenPrel.java
@@ -0,0 +1,80 @@
+/*******************************************************************************
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ ******************************************************************************/
+package org.apache.drill.exec.planner.physical;
+
+import org.apache.drill.common.expression.LogicalExpression;
+import org.apache.drill.common.expression.PathSegment;
+import org.apache.drill.common.expression.SchemaPath;
+import org.apache.drill.exec.physical.base.PhysicalOperator;
+import org.apache.drill.exec.physical.config.FlattenPOP;
+import org.apache.drill.exec.planner.common.DrillFilterRelBase;
+import org.apache.drill.exec.planner.logical.DrillOptiq;
+import org.apache.drill.exec.planner.logical.DrillParseContext;
+import org.apache.drill.exec.planner.physical.visitor.PrelVisitor;
+import org.apache.drill.exec.record.BatchSchema;
+import org.eigenbase.rel.RelNode;
+import org.eigenbase.rel.RelWriter;
+import org.eigenbase.relopt.RelOptCluster;
+import org.eigenbase.relopt.RelTraitSet;
+import org.eigenbase.rex.RexNode;
+
+import java.io.IOException;
+import java.util.Iterator;
+import java.util.List;
+
+public class FlattenPrel extends SinglePrel implements Prel {
+
+  RexNode toFlatten;
+
+  public FlattenPrel(RelOptCluster cluster, RelTraitSet traits, RelNode child, RexNode toFlatten)
{
+    super(cluster, traits, child);
+    this.toFlatten = toFlatten;
+  }
+
+  @Override
+  public RelNode copy(RelTraitSet traitSet, List<RelNode> inputs) {
+    return new FlattenPrel(getCluster(), traitSet, sole(inputs), toFlatten);
+  }
+
+  @Override
+  public Iterator<Prel> iterator() {
+    return PrelUtil.iter(getChild());
+  }
+
+  @Override
+  public PhysicalOperator getPhysicalOperator(PhysicalPlanCreator creator) throws IOException
{
+    Prel child = (Prel) this.getChild();
+
+    PhysicalOperator childPOP = child.getPhysicalOperator(creator);
+    FlattenPOP f = new FlattenPOP(childPOP, (SchemaPath) getFlattenExpression(new DrillParseContext()));
+    return creator.addMetadata(this, f);
+  }
+
+  public RelWriter explainTerms(RelWriter pw) {
+    return super.explainTerms(pw).item("flattenField", this.toFlatten);
+  }
+
+  @Override
+  public BatchSchema.SelectionVectorMode getEncoding() {
+    return BatchSchema.SelectionVectorMode.NONE;
+  }
+
+  protected LogicalExpression getFlattenExpression(DrillParseContext context){
+    return DrillOptiq.toDrill(context, getChild(), toFlatten);
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/60736dea/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/visitor/RewriteProjectToFlatten.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/visitor/RewriteProjectToFlatten.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/visitor/RewriteProjectToFlatten.java
index d4b3573..1055411 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/visitor/RewriteProjectToFlatten.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/visitor/RewriteProjectToFlatten.java
@@ -21,7 +21,7 @@ import com.google.common.collect.Lists;
 import net.hydromatic.optiq.tools.RelConversionException;
 import org.apache.drill.common.exceptions.DrillRuntimeException;
 import org.apache.drill.exec.expr.fn.FunctionImplementationRegistry;
-import org.apache.drill.exec.planner.physical.DrillFlattenPrel;
+import org.apache.drill.exec.planner.physical.FlattenPrel;
 import org.apache.drill.exec.planner.physical.Prel;
 import org.apache.drill.exec.planner.physical.ProjectPrel;
 import org.apache.drill.exec.planner.physical.visitor.BasePrelVisitor;
@@ -103,7 +103,7 @@ public class RewriteProjectToFlatten extends BasePrelVisitor<Prel,
Object, RelCo
       // TODO - figure out what is the right setting for the traits
       Prel newChild = ((Prel)project.getInput(0)).accept(this, null);
       ProjectPrel newProject = new ProjectPrel(node.getCluster(), project.getTraitSet(),
newChild, exprList, new RelRecordType(relDataTypes));
-      DrillFlattenPrel flatten = new DrillFlattenPrel(project.getCluster(), project.getTraitSet(),
newProject, flatttenExpr);
+      FlattenPrel flatten = new FlattenPrel(project.getCluster(), project.getTraitSet(),
newProject, flatttenExpr);
       return flatten;
     }
 

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/60736dea/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/visitor/SplitUpComplexExpressions.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/visitor/SplitUpComplexExpressions.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/visitor/SplitUpComplexExpressions.java
index 0a49f3a..89c8447 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/visitor/SplitUpComplexExpressions.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/visitor/SplitUpComplexExpressions.java
@@ -26,7 +26,7 @@ import org.apache.drill.exec.expr.fn.FunctionImplementationRegistry;
 import org.apache.drill.exec.planner.logical.DrillRel;
 import org.apache.drill.exec.planner.logical.DrillScanRel;
 import org.apache.drill.exec.planner.logical.DrillTable;
-import org.apache.drill.exec.planner.physical.DrillFlattenPrel;
+import org.apache.drill.exec.planner.physical.FlattenPrel;
 import org.apache.drill.exec.planner.physical.Prel;
 import org.apache.drill.exec.planner.physical.PrelUtil;
 import org.apache.drill.exec.planner.physical.ProjectPrel;

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/60736dea/exec/java-exec/src/main/java/org/apache/drill/exec/record/MaterializedField.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/record/MaterializedField.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/record/MaterializedField.java
index b4da2b4..7f7dcfa 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/record/MaterializedField.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/record/MaterializedField.java
@@ -18,10 +18,12 @@
 package org.apache.drill.exec.record;
 
 import java.util.Collection;
+import java.util.Iterator;
 import java.util.Map;
 import java.util.Set;
 
 import com.google.common.collect.Sets;
+
 import org.apache.drill.common.expression.FieldReference;
 import org.apache.drill.common.expression.PathSegment;
 import org.apache.drill.common.expression.SchemaPath;
@@ -215,9 +217,27 @@ public class MaterializedField {
     return true;
   }
 
+
   @Override
   public String toString() {
-    return "MaterializedField [path=" + key.path + ", type=" + Types.toString(key.type) +
"]";
+    final int maxLen = 10;
+    String childStr = children != null && !children.isEmpty() ? toString(children,
maxLen) : "";
+    return key.path + "(" + key.type.getMinorType().name() + ":" + key.type.getMode().name()
+ ")" + childStr;
+  }
+
+
+  private String toString(Collection<?> collection, int maxLen) {
+    StringBuilder builder = new StringBuilder();
+    builder.append("[");
+    int i = 0;
+    for (Iterator<?> iterator = collection.iterator(); iterator.hasNext() &&
i < maxLen; i++) {
+      if (i > 0){
+        builder.append(", ");
+      }
+      builder.append(iterator.next());
+    }
+    builder.append("]");
+    return builder.toString();
   }
 
   public Key key() {

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/60736dea/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedMapVector.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedMapVector.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedMapVector.java
index 9b7011c..7ddac23 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedMapVector.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedMapVector.java
@@ -360,6 +360,9 @@ public class RepeatedMapVector extends AbstractContainerVector implements
Repeat
     public boolean copyValueSafe(int srcIndex, int destIndex) {
       RepeatedMapHolder holder = new RepeatedMapHolder();
       accessor.get(srcIndex, holder);
+      if(destIndex >= to.getValueCapacity()){
+        return false;
+      }
       to.populateEmpties(destIndex+1);
       int newIndex = to.offsets.getAccessor().get(destIndex);
       //todo: make these bulk copies
@@ -377,8 +380,30 @@ public class RepeatedMapVector extends AbstractContainerVector implements
Repeat
     }
 
     @Override
-    public void splitAndTransfer(int startIndex, int length) {
-      throw new UnsupportedOperationException();
+    public void splitAndTransfer(final int groupStart, final int groups) {
+      final UInt4Vector.Accessor a = offsets.getAccessor();
+      final UInt4Vector.Mutator m = to.offsets.getMutator();
+
+      final int startPos = a.get(groupStart);
+      final int endPos = a.get(groupStart+groups);
+      final int valuesToCopy = endPos - startPos;
+
+      to.offsets.clear();
+      to.offsets.allocateNew(groups + 1);
+      int normalizedPos = 0;
+
+      for (int i=0; i < groups+1; i++) {
+        normalizedPos = a.get(groupStart+i) - startPos;
+        m.set(i, normalizedPos);
+      }
+
+      m.setValueCount(groups + 1);
+      to.lastSet = groups;
+
+      for (TransferPair p : pairs) {
+        p.splitAndTransfer(startPos, valuesToCopy);
+      }
+
     }
 
   }
@@ -444,7 +469,9 @@ public class RepeatedMapVector extends AbstractContainerVector implements
Repeat
     SerializedField.Builder b = getField() //
         .getAsBuilder() //
         .setBufferLength(getBufferSize()) //
-        .setGroupCount(accessor.getGroupCount());
+        .setGroupCount(accessor.getGroupCount())
+        // while we don't need to actually read this on load, we need it to make sure we
don't skip deserialization of this vector
+        .setValueCount(accessor.getValueCount());
     for (ValueVector v : vectors.values()) {
       b.addChild(v.getMetadata());
     }
@@ -503,7 +530,7 @@ public class RepeatedMapVector extends AbstractContainerVector implements
Repeat
     }
 
     public void get(int index, RepeatedMapHolder holder) {
-      assert index < getValueCapacity()-1;
+      assert index < getValueCapacity() : String.format("Attempted to access index %d
when value capacity is %d", index, getValueCapacity());
       holder.start = offsets.getAccessor().get(index);
       holder.end = offsets.getAccessor().get(index+1);
     }

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/60736dea/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/fn/JsonReader.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/fn/JsonReader.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/fn/JsonReader.java
index 17e266f..0ca24e7 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/fn/JsonReader.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/fn/JsonReader.java
@@ -197,7 +197,6 @@ private boolean writeToVector(ComplexWriter writer, JsonToken t) throws
JsonPars
     map.start();
     outside: while(true) {
       if (!map.ok()) {
-        logger.warn("Error reported. Quit writing");
         break;
       }
       JsonToken t = parser.nextToken();
@@ -320,7 +319,7 @@ private boolean writeToVector(ComplexWriter writer, JsonToken t) throws
JsonPars
     list.start();
     outside: while (true) {
       if (!list.ok()) {
-        logger.warn("Error reported. Quit writing");
+//        logger.warn("Error reported. Quit writing");
         break;
       }
 

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/60736dea/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/flatten/TestFlatten.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/flatten/TestFlatten.java
b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/flatten/TestFlatten.java
index d4c19a3..960da65 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/flatten/TestFlatten.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/flatten/TestFlatten.java
@@ -17,7 +17,10 @@
  ******************************************************************************/
 package org.apache.drill.exec.physical.impl.flatten;
 
+import static org.junit.Assert.assertEquals;
+
 import org.apache.drill.BaseTestQuery;
+import org.junit.Ignore;
 import org.junit.Test;
 
 public class TestFlatten extends BaseTestQuery {
@@ -27,6 +30,7 @@ public class TestFlatten extends BaseTestQuery {
    *    - /tmp/yelp_academic_dataset_business.json
    *    - /tmp/mapkv.json
    *    - /tmp/drill1665.json
+   *    - /tmp/bigfile.json
    */
   public static boolean RUN_ADVANCED_TESTS = false;
 
@@ -38,6 +42,53 @@ public class TestFlatten extends BaseTestQuery {
   }
 
   @Test
+  public void drill1671() throws Exception{
+    int rowCount = testSql("select * from (select count(*) as cnt from (select id, flatten(evnts1),
flatten(evnts2), flatten(evnts3), flatten(evnts4), flatten(evnts5), flatten(evnts6), flatten(evnts7),
flatten(evnts8), flatten(evnts9), flatten(evnts10), flatten(evnts11) from cp.`/flatten/many-arrays-50.json`)x
)y where cnt = 2048");
+    assertEquals(rowCount, 1);
+  }
+
+  @Test
+  @Ignore("not yet fixed")
+  public void drill1660() throws Exception {
+    test("select * from cp.`/flatten/empty-rm.json`");
+  }
+
+  @Test
+  public void drill1653() throws Exception{
+    int rowCount = testSql("select * from (select sum(t.flat.`value`) as sm from (select
id, flatten(kvgen(m)) as flat from cp.`/flatten/missing-map.json`)t) where sm = 10 ");
+    assertEquals(rowCount, 1);
+  }
+
+  @Test
+  public void drill1652() throws Exception {
+    if(RUN_ADVANCED_TESTS){
+      test("select uid, flatten(transactions) from dfs.`/tmp/bigfile.json`");
+    }
+  }
+
+  @Test
+  @Ignore("Still not working.")
+  public void drill1649() throws Exception {
+    test("select event_info.uid, transaction_info.trans_id, event_info.event.evnt_id\n" +
+        "from (\n" +
+        " select userinfo.transaction.trans_id trans_id, max(userinfo.event.event_time) max_event_time\n"
+
+        " from (\n" +
+        "     select uid, flatten(events) event, flatten(transactions) transaction from cp.`/flatten/single-user-transactions.json`\n"
+
+        " ) userinfo\n" +
+        " where userinfo.transaction.trans_time >= userinfo.event.event_time\n" +
+        " group by userinfo.transaction.trans_id\n" +
+        ") transaction_info\n" +
+        "inner join\n" +
+        "(\n" +
+        " select uid, flatten(events) event\n" +
+        " from cp.`/flatten/single-user-transactions.json`\n" +
+        ") event_info\n" +
+        "on transaction_info.max_event_time = event_info.event.event_time;");
+  }
+
+
+
+  @Test
   public void testKVGenFlatten1() throws Exception {
     // works - TODO and verify results
     test("select flatten(kvgen(f1)) as monkey, x " +

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/60736dea/exec/java-exec/src/test/resources/flatten/empty-rm.json
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/resources/flatten/empty-rm.json b/exec/java-exec/src/test/resources/flatten/empty-rm.json
new file mode 100644
index 0000000..7927a3f
--- /dev/null
+++ b/exec/java-exec/src/test/resources/flatten/empty-rm.json
@@ -0,0 +1,7 @@
+{
+  "id":1,
+  "evnts":[
+    {},
+    {"key":null}
+  ]
+}

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/60736dea/exec/java-exec/src/test/resources/flatten/many-arrays-50.json
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/resources/flatten/many-arrays-50.json b/exec/java-exec/src/test/resources/flatten/many-arrays-50.json
new file mode 100644
index 0000000..c25565e
--- /dev/null
+++ b/exec/java-exec/src/test/resources/flatten/many-arrays-50.json
@@ -0,0 +1,203 @@
+{
+    "id":1,
+    "evnts1":[
+        { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000},
+        { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000}
+    ],
+    "evnts2":[
+        { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000},
+        { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000}
+    ],
+    "evnts3":[
+        { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000},
+        { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000}
+    ],
+   "evnts4":[
+        { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000},
+        { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000}
+    ],
+    "evnts5":[
+        { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000},
+        { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000}
+    ],
+    "evnts6":[
+        { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000},
+        { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000}
+    ],
+    "evnts7":[
+        { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000},
+        { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000}
+    ],
+    "evnts8":[
+        { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000},
+        { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000}
+    ],
+    "evnts9":[
+        { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000},
+        { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000}
+    ],
+    "evnts10":[
+        { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000},
+        { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000}
+    ],
+    "evnts11":[
+        { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000},
+        { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000}
+    ],
+    "evnts12":[
+        { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000},
+        { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000}
+    ],
+    "evnts13":[
+        { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000},
+        { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000}
+    ],
+    "evnts14":[
+        { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000},
+        { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000}
+    ],
+    "evnts15":[
+        { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000},
+        { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000}
+    ],
+    "evnts16":[
+        { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000},
+        { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000}
+    ],
+    "evnts17":[
+        { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000},
+        { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000}
+    ],
+    "evnts18":[
+        { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000},
+        { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000}
+    ],
+    "evnts19":[
+        { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000},
+        { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000}
+    ],
+    "evnts20":[
+        { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000},
+        { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000}
+    ],
+    "evnts21":[
+        { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000},
+        { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000}
+    ],
+    "evnts22":[
+        { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000},
+        { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000}
+    ],
+    "evnts23":[
+        { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000},
+        { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000}
+    ],
+    "evnts24":[
+        { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000},
+        { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000}
+    ],
+    "evnts25":[
+        { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000},
+        { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000}
+    ],
+    "evnts26":[
+        { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000},
+        { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000}
+    ],
+    "evnts27":[
+        { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000},
+        { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000}
+    ],
+    "evnts28":[
+        { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000},
+        { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000}
+    ],
+    "evnts29":[
+        { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000},
+        { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000}
+    ],
+    "evnts30":[
+        { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000},
+        { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000}
+    ],
+    "evnts31":[
+        { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000},
+        { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000}
+    ],
+    "evnts32":[
+        { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000},
+        { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000}
+    ],
+    "evnts33":[
+        { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000},
+        { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000}
+    ],
+    "evnts34":[
+        { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000},
+        { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000}
+    ],
+    "evnts35":[
+        { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000},
+        { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000}
+    ],
+    "evnts36":[
+        { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000},
+        { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000}
+    ],
+    "evnts37":[
+        { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000},
+        { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000}
+    ],
+    "evnts38":[
+        { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000},
+        { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000}
+    ],
+    "evnts39":[
+        { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000},
+        { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000}
+    ],
+    "evnts40":[
+        { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000},
+        { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000}
+    ],
+    "evnts41":[
+        { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000},
+        { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000}
+    ],
+    "evnts42":[
+        { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000},
+        { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000}
+    ],
+    "evnts43":[
+        { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000},
+        { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000}
+    ],
+    "evnts44":[
+        { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000},
+        { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000}
+    ],
+    "evnts45":[
+        { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000},
+        { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000}
+    ],
+    "evnts46":[
+        { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000},
+        { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000}
+    ],
+    "evnts47":[
+        { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000},
+        { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000}
+    ],
+    "evnts48":[
+        { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000},
+        { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000}
+    ],
+    "evnts49":[
+        { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000},
+        { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000}
+    ],
+    "evnts50":[
+        { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000},
+        { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000}
+    ]
+}

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/60736dea/exec/java-exec/src/test/resources/flatten/missing-map.json
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/resources/flatten/missing-map.json b/exec/java-exec/src/test/resources/flatten/missing-map.json
new file mode 100644
index 0000000..5d8f815
--- /dev/null
+++ b/exec/java-exec/src/test/resources/flatten/missing-map.json
@@ -0,0 +1,11 @@
+{
+    "id": 1,
+    "m": {"a":1,"b":2}
+}
+{
+    "id": 2
+}
+{
+    "id": 3,
+    "m": {"c":3,"d":4}
+}

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/60736dea/exec/java-exec/src/test/resources/flatten/single-user-transactions.json
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/resources/flatten/single-user-transactions.json b/exec/java-exec/src/test/resources/flatten/single-user-transactions.json
new file mode 100644
index 0000000..cff5532
--- /dev/null
+++ b/exec/java-exec/src/test/resources/flatten/single-user-transactions.json
@@ -0,0 +1,18 @@
+{
+    "uid" : 1,
+    "events" : [
+        { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000},
+        { "evnt_id":"e2", "campaign_id":"c1", "event_name":"e2_name", "event_time":2000000},
+        { "evnt_id":"e3", "campaign_id":"c1", "event_name":"e3_name", "event_time":3000000},
+        { "evnt_id":"e4", "campaign_id":"c1", "event_name":"e4_name", "event_time":4000000},
+        { "evnt_id":"e5", "campaign_id":"c2", "event_name":"e5_name", "event_time":5000000},
+        { "evnt_id":"e6", "campaign_id":"c1", "event_name":"e6_name", "event_time":6000000},
+        { "evnt_id":"e7", "campaign_id":"c1", "event_name":"e7_name", "event_time":7000000},
+        { "evnt_id":"e8", "campaign_id":"c2", "event_name":"e8_name", "event_time":8000000},
+        { "evnt_id":"e9", "campaign_id":"c2", "event_name":"e9_name", "event_time":9000000}
+    ],
+   "transactions" : [
+        { "trans_id":"t1", "amount":100, "trans_time":7777777},
+        { "trans_id":"t2", "amount":100, "trans_time":8888888}
+    ]
+}


Mime
View raw message