drill-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From meh...@apache.org
Subject incubator-drill git commit: DRILL-1673: Set the correct group count in splitAndTransferTo for repeated vectors Handle the case in FlattedRecordBatch where we return null internal vectors for repeated lists in the first schema batch
Date Wed, 19 Nov 2014 19:36:17 GMT
Repository: incubator-drill
Updated Branches:
  refs/heads/master a147bfce0 -> 90c12c8ce


DRILL-1673: Set the correct group count in splitAndTransferTo for repeated vectors
Handle the case in FlattedRecordBatch where we return null internal vectors for repeated lists
in the first schema batch


Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/90c12c8c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/90c12c8c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/90c12c8c

Branch: refs/heads/master
Commit: 90c12c8ce93fb1451d913f4af4560bd0fb6a5306
Parents: a147bfc
Author: Mehant Baid <mehantr@gmail.com>
Authored: Tue Nov 18 01:06:58 2014 -0800
Committer: Mehant Baid <mehantr@gmail.com>
Committed: Wed Nov 19 10:56:26 2014 -0800

----------------------------------------------------------------------
 .../codegen/templates/RepeatedValueVectors.java |  2 +-
 .../impl/flatten/FlattenRecordBatch.java        | 22 ++++++++++++++------
 .../complex/writer/TestComplexTypeReader.java   |  5 +++++
 3 files changed, 22 insertions(+), 7 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/90c12c8c/exec/java-exec/src/main/codegen/templates/RepeatedValueVectors.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/codegen/templates/RepeatedValueVectors.java b/exec/java-exec/src/main/codegen/templates/RepeatedValueVectors.java
index bf93d00..3d50b00 100644
--- a/exec/java-exec/src/main/codegen/templates/RepeatedValueVectors.java
+++ b/exec/java-exec/src/main/codegen/templates/RepeatedValueVectors.java
@@ -124,7 +124,7 @@ public final class Repeated${minor.class}Vector extends BaseValueVector
implemen
     }
     to.parentValueCount = groups;
     to.childValueCount  = valuesToCopy;
-    m.setValueCount(groups);
+    m.setValueCount(groups == 0 ? 0 : groups + 1);
   }
   
   private class TransferImpl implements TransferPair{

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/90c12c8c/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/flatten/FlattenRecordBatch.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/flatten/FlattenRecordBatch.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/flatten/FlattenRecordBatch.java
index 66c6168..78c1c50 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/flatten/FlattenRecordBatch.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/flatten/FlattenRecordBatch.java
@@ -29,6 +29,7 @@ import org.apache.drill.common.expression.FieldReference;
 import org.apache.drill.common.expression.LogicalExpression;
 import org.apache.drill.common.expression.PathSegment;
 import org.apache.drill.common.logical.data.NamedExpression;
+import org.apache.drill.common.types.TypeProtos;
 import org.apache.drill.exec.exception.ClassTransformationException;
 import org.apache.drill.exec.exception.SchemaChangeException;
 import org.apache.drill.exec.expr.ClassGenerator;
@@ -263,7 +264,11 @@ public class FlattenRecordBatch extends AbstractSingleRecordBatch<FlattenPOP>
{
             ValueVector vector = container.addOrGet(vw.getField());
             container.add(vector);
           } else {
-            container.add(getFlattenFieldTransferPair().getTo());
+            TransferPair pair = getFlattenFieldTransferPair();
+            if (pair == null) {
+              continue;
+            }
+            container.add(pair.getTo());
           }
         } else {
           ValueVector vector = container.addOrGet(vw.getField());
@@ -297,12 +302,15 @@ public class FlattenRecordBatch extends AbstractSingleRecordBatch<FlattenPOP>
{
                 popConfig.getColumn()).getFieldIds()[0]).getValueClass(),
         incoming.getValueVectorId(popConfig.getColumn()).getFieldIds()).getValueVector();
 
-    TransferPair tp;
+    TransferPair tp = null;
     if (flattenField instanceof RepeatedMapVector) {
       tp = ((RepeatedMapVector)flattenField).getTransferPairToSingleMap();
     } else {
       ValueVector vvIn = ((RepeatedVector)flattenField).getAccessor().getAllChildValues();
-      tp = vvIn.getTransferPair();
+      // vvIn may be null because of fast schema return for repeated list vectors
+      if (vvIn != null) {
+        tp = vvIn.getTransferPair();
+      }
     }
     return tp;
   }
@@ -326,9 +334,11 @@ public class FlattenRecordBatch extends AbstractSingleRecordBatch<FlattenPOP>
{
 
     TransferPair tp = getFlattenFieldTransferPair();
 
-    transfers.add(tp);
-    container.add(tp.getTo());
-    transferFieldIds.add(vectorRead.getFieldId().getFieldIds()[0]);
+    if (tp != null) {
+      transfers.add(tp);
+      container.add(tp.getTo());
+      transferFieldIds.add(vectorRead.getFieldId().getFieldIds()[0]);
+    }
 
     logger.debug("Added transfer for project expression.");
 

http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/90c12c8c/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/writer/TestComplexTypeReader.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/writer/TestComplexTypeReader.java
b/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/writer/TestComplexTypeReader.java
index df92dde..83e1130 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/writer/TestComplexTypeReader.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/writer/TestComplexTypeReader.java
@@ -188,4 +188,9 @@ public class TestComplexTypeReader extends BaseTestQuery{
     test("select kvgen(x) from cp.`jsoninput/input2.json`");
     test("select kvgen(bigintegercol), kvgen(float8col) from cp.`jsoninput/input3.json`");
   }
+
+  @Test
+  public void testNestedFlatten() throws Exception {
+    test("select flatten(rl) from cp.`jsoninput/input2.json`");
+  }
 }


Mime
View raw message