drill-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From j..@apache.org
Subject drill git commit: DRILL-4783: flatten operator handle empty resultset(and schema unknown due to this)
Date Fri, 22 Jul 2016 05:35:40 GMT
Repository: drill
Updated Branches:
  refs/heads/master 43776706b -> 04964bbf8


DRILL-4783: flatten operator handle empty resultset(and schema unknown due to this)

close apache/drill#546


Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/04964bbf
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/04964bbf
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/04964bbf

Branch: refs/heads/master
Commit: 04964bbf816746ffaefbe58bf9ddedbff54e0f69
Parents: 4377670
Author: chunhui-shi <cshi@maprtech.com>
Authored: Sat Jul 16 00:18:33 2016 -0700
Committer: Jinfeng Ni <jni@apache.org>
Committed: Thu Jul 21 21:27:30 2016 -0700

----------------------------------------------------------------------
 .../impl/flatten/FlattenRecordBatch.java        | 34 ++++++++++++++------
 .../exec/nested/TestFastComplexSchema.java      | 29 +++++++++++++++++
 2 files changed, 54 insertions(+), 9 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/drill/blob/04964bbf/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/flatten/FlattenRecordBatch.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/flatten/FlattenRecordBatch.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/flatten/FlattenRecordBatch.java
index 5b3f6ae..a2b170d 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/flatten/FlattenRecordBatch.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/flatten/FlattenRecordBatch.java
@@ -126,15 +126,23 @@ public class FlattenRecordBatch extends AbstractSingleRecordBatch<FlattenPOP>
{
   }
 
   private void setFlattenVector() {
-    try {
-      final TypedFieldId typedFieldId = incoming.getValueVectorId(popConfig.getColumn());
-      final MaterializedField field = incoming.getSchema().getColumn(typedFieldId.getFieldIds()[0]);
-      final RepeatedValueVector vector = RepeatedValueVector.class.cast(incoming.getValueAccessorById(
-          field.getValueClass(), typedFieldId.getFieldIds()).getValueVector());
-      flattener.setFlattenField(vector);
-    } catch (Exception ex) {
-      throw UserException.unsupportedError(ex).message("Trying to flatten a non-repeated
field.").build(logger);
+    final TypedFieldId typedFieldId = incoming.getValueVectorId(popConfig.getColumn());
+    final MaterializedField field = incoming.getSchema().getColumn(typedFieldId.getFieldIds()[0]);
+    final RepeatedValueVector vector;
+    final ValueVector inVV = incoming.getValueAccessorById(
+        field.getValueClass(), typedFieldId.getFieldIds()).getValueVector();
+
+    if (! (inVV instanceof RepeatedValueVector)) {
+      if (incoming.getRecordCount() != 0) {
+        throw UserException.unsupportedError().message("Flatten does not support inputs of
non-list values.").build(logger);
+      }
+      //when incoming recordCount is 0, don't throw exception since the type being seen here
is not solid
+      logger.error("setFlattenVector cast failed and recordcount is 0, create empty vector
anyway.");
+      vector = new RepeatedMapVector(field, oContext.getAllocator(), null);
+    } else {
+      vector = RepeatedValueVector.class.cast(inVV);
     }
+    flattener.setFlattenField(vector);
   }
 
   @Override
@@ -151,7 +159,7 @@ public class FlattenRecordBatch extends AbstractSingleRecordBatch<FlattenPOP>
{
     setFlattenVector();
 
     int childCount = incomingRecordCount == 0 ? 0 : flattener.getFlattenField().getAccessor().getInnerValueCount();
-    int outputRecords = flattener.flattenRecords(incomingRecordCount, 0, monitor);
+    int outputRecords = childCount == 0 ? 0: flattener.flattenRecords(incomingRecordCount,
0, monitor);
     // TODO - change this to be based on the repeated vector length
     if (outputRecords < childCount) {
       setValueCount(outputRecords);
@@ -266,6 +274,14 @@ public class FlattenRecordBatch extends AbstractSingleRecordBatch<FlattenPOP>
{
     TransferPair tp = null;
     if (flattenField instanceof RepeatedMapVector) {
       tp = ((RepeatedMapVector)flattenField).getTransferPairToSingleMap(reference.getAsNamePart().getName(),
oContext.getAllocator());
+    } else if ( !(flattenField instanceof RepeatedValueVector) ) {
+      if(incoming.getRecordCount() != 0) {
+        throw UserException.unsupportedError().message("Flatten does not support inputs of
non-list values.").build(logger);
+      }
+      logger.error("Cannot cast {} to RepeatedValueVector", flattenField);
+      //when incoming recordCount is 0, don't throw exception since the type being seen here
is not solid
+      final ValueVector vv = new RepeatedMapVector(flattenField.getField(), oContext.getAllocator(),
null);
+      tp = RepeatedValueVector.class.cast(vv).getTransferPair(reference.getAsNamePart().getName(),
oContext.getAllocator());
     } else {
       final ValueVector vvIn = RepeatedValueVector.class.cast(flattenField).getDataVector();
       // vvIn may be null because of fast schema return for repeated list vectors

http://git-wip-us.apache.org/repos/asf/drill/blob/04964bbf/exec/java-exec/src/test/java/org/apache/drill/exec/nested/TestFastComplexSchema.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/nested/TestFastComplexSchema.java
b/exec/java-exec/src/test/java/org/apache/drill/exec/nested/TestFastComplexSchema.java
index 955c93e..e5bd62d 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/nested/TestFastComplexSchema.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/nested/TestFastComplexSchema.java
@@ -80,4 +80,33 @@ public class TestFastComplexSchema extends BaseTestQuery {
             "                       AND r.r_regionkey = 4)) t \n" +
             "ORDER  BY t.f.name");
   }
+
+  @Test //DRILL-4783 when resultset is empty, don't throw exception.
+  public void test5() throws Exception {
+
+    //when there is no incoming record, flatten won't throw exception
+    testBuilder().sqlQuery("select flatten(j) from \n" +
+           " (select convert_from(names, 'json') j \n" +
+           " from (select concat('[\"', first_name, '\", ', '\"', last_name, '\"]') names
\n" +
+           " from cp.`employee.json` where store_id=9999))")
+        .expectsEmptyResultSet()
+        .build().run();
+
+    //result is not empty and is list type,
+    testBuilder().sqlQuery("select flatten(j) n from \n" +
+        " (select convert_from(names, 'json') j \n" +
+        " from (select concat('[\"', first_name, '\", ', '\"', last_name, '\"]') names \n"
+
+        " from cp.`employee.json` where first_name='Sheri'))")
+        .unOrdered()
+        .baselineColumns("n")
+        .baselineValues("Sheri")
+        .baselineValues("Nowmer")
+        .build().run();
+
+    //result is not empty, and flatten got incompatible (non-list) incoming records. got
exception thrown
+    errorMsgTestHelper("select flatten(first_name) from \n" +
+        "(select first_name from cp.`employee.json` where first_name='Sheri')",
+        "Flatten does not support inputs of non-list values");
+  }
+
 }


Mime
View raw message