drill-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From j..@apache.org
Subject drill git commit: DRILL-3533: Fix incorrect query result when querying parquet for fields that do not exists.
Date Fri, 24 Jul 2015 18:06:28 GMT
Repository: drill
Updated Branches:
  refs/heads/master 66a30f12b -> 1b69869d9


DRILL-3533: Fix incorrect query result when querying parquet for fields that do not exists.


Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/1b69869d
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/1b69869d
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/1b69869d

Branch: refs/heads/master
Commit: 1b69869d981eece6c23a2e52a413133f993f3125
Parents: 66a30f1
Author: Jinfeng Ni <jni@apache.org>
Authored: Tue Jul 21 22:00:40 2015 -0700
Committer: Jinfeng Ni <jni@apache.org>
Committed: Thu Jul 23 15:54:37 2015 -0700

----------------------------------------------------------------------
 .../exec/store/parquet2/DrillParquetReader.java     |  5 ++++-
 .../exec/store/parquet/TestParquetComplex.java      | 14 ++++++++++++++
 .../resources/store/parquet/complex/baseline8.json  | 16 ++++++++++++++++
 3 files changed, 34 insertions(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/drill/blob/1b69869d/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet2/DrillParquetReader.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet2/DrillParquetReader.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet2/DrillParquetReader.java
index 4e7d628..4c49def 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet2/DrillParquetReader.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet2/DrillParquetReader.java
@@ -26,6 +26,7 @@ import java.util.List;
 import java.util.Map;
 import java.util.Set;
 
+import org.apache.commons.collections.CollectionUtils;
 import org.apache.drill.common.exceptions.DrillRuntimeException;
 import org.apache.drill.common.exceptions.ExecutionSetupException;
 import org.apache.drill.common.expression.PathSegment;
@@ -259,7 +260,9 @@ public class DrillParquetReader extends AbstractRecordReader {
 
       if(!noColumnsFound) {
         writer = new VectorContainerWriter(output);
-        recordMaterializer = new DrillParquetRecordMaterializer(output, writer, projection,
getColumns(),
+        // Discard the columns not found in the schema when create DrillParquetRecordMaterializer,
since they have been added to output already.
+        final Collection<SchemaPath> columns = columnsNotFound == null || columnsNotFound.size()
== 0 ? getColumns(): CollectionUtils.subtract(getColumns(), columnsNotFound);
+        recordMaterializer = new DrillParquetRecordMaterializer(output, writer, projection,
columns,
             fragmentContext.getOptions());
         primitiveVectors = writer.getMapVector().getPrimitiveVectors();
         recordReader = columnIO.getRecordReader(pageReadStore, recordMaterializer);

http://git-wip-us.apache.org/repos/asf/drill/blob/1b69869d/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/TestParquetComplex.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/TestParquetComplex.java
b/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/TestParquetComplex.java
index baf5e82..6397ef7 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/TestParquetComplex.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/TestParquetComplex.java
@@ -177,4 +177,18 @@ public class TestParquetComplex extends BaseTestQuery {
             .build()
             .run();
   }
+
+  @Test //DRILL-3533
+  public void notxistsField() throws Exception {
+    String query = String.format("select t.`marketing_info`.notexists as notexists, t.`marketing_info`.camp_id
as id from %s t", DATAFILE);
+    String[] columns = {"notexists", "id"};
+    testBuilder()
+        .sqlQuery(query)
+        .unOrdered()
+        .jsonBaselineFile("store/parquet/complex/baseline8.json")
+        .baselineColumns(columns)
+        .build()
+        .run();
+  }
+
 }

http://git-wip-us.apache.org/repos/asf/drill/blob/1b69869d/exec/java-exec/src/test/resources/store/parquet/complex/baseline8.json
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/resources/store/parquet/complex/baseline8.json b/exec/java-exec/src/test/resources/store/parquet/complex/baseline8.json
new file mode 100644
index 0000000..290d896
--- /dev/null
+++ b/exec/java-exec/src/test/resources/store/parquet/complex/baseline8.json
@@ -0,0 +1,16 @@
+{
+  "notexists" : null,
+  "id" : 4
+} {
+  "notexists" : null,
+  "id" : 6
+} {
+  "notexists" : null,
+  "id" : 17
+} {
+  "notexists" : null,
+  "id" : 17
+} {
+  "notexists" : null,
+  "id" : 8
+}
\ No newline at end of file


Mime
View raw message