drill-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From amansi...@apache.org
Subject [6/7] drill git commit: DRILL-4733: max(dir0) reading more columns than necessary
Date Thu, 23 Jun 2016 18:10:46 GMT
DRILL-4733: max(dir0) reading more columns than necessary


Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/19cdcd95
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/19cdcd95
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/19cdcd95

Branch: refs/heads/1.7.0
Commit: 19cdcd95808ddb6f9e19bbccb753017993ad78d6
Parents: 4687a8b
Author: Arina Ielchiieva <arina.yelchiyeva@gmail.com>
Authored: Tue Jun 21 12:33:32 2016 +0000
Committer: Aman Sinha <asinha@maprtech.com>
Committed: Thu Jun 23 10:29:38 2016 -0700

----------------------------------------------------------------------
 .../drill/exec/store/ImplicitColumnExplorer.java  |   6 ------
 .../exec/store/dfs/easy/EasyFormatPlugin.java     |   7 ++++++-
 .../drill/exec/store/TestImplicitFileColumns.java |  17 +++++++++++++++++
 .../parquetWithSchemaChange/voter5/voter5.parquet | Bin 0 -> 1250 bytes
 .../voter50/voter50.parquet                       | Bin 0 -> 2960 bytes
 5 files changed, 23 insertions(+), 7 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/drill/blob/19cdcd95/exec/java-exec/src/main/java/org/apache/drill/exec/store/ImplicitColumnExplorer.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/ImplicitColumnExplorer.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/store/ImplicitColumnExplorer.java
index 94a0dca..af74eb7 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/ImplicitColumnExplorer.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/ImplicitColumnExplorer.java
@@ -135,12 +135,6 @@ public class ImplicitColumnExplorer {
           tableColumns.add(column);
         }
       }
-
-      // We must make sure to pass a table column(not to be confused with partition column)
to the underlying record
-      // reader.
-      if (tableColumns.size() == 0) {
-        tableColumns.add(AbstractRecordReader.STAR_COLUMN);
-      }
     }
   }
 

http://git-wip-us.apache.org/repos/asf/drill/blob/19cdcd95/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/easy/EasyFormatPlugin.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/easy/EasyFormatPlugin.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/easy/EasyFormatPlugin.java
index 5881d33..f56f445 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/easy/EasyFormatPlugin.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/easy/EasyFormatPlugin.java
@@ -41,6 +41,7 @@ import org.apache.drill.exec.planner.physical.PlannerSettings;
 import org.apache.drill.exec.record.CloseableRecordBatch;
 import org.apache.drill.exec.record.RecordBatch;
 import org.apache.drill.exec.server.DrillbitContext;
+import org.apache.drill.exec.store.AbstractRecordReader;
 import org.apache.drill.exec.store.ImplicitColumnExplorer;
 import org.apache.drill.exec.store.RecordReader;
 import org.apache.drill.exec.store.RecordWriter;
@@ -126,8 +127,12 @@ public abstract class EasyFormatPlugin<T extends FormatPluginConfig>
implements
     final ImplicitColumnExplorer columnExplorer = new ImplicitColumnExplorer(context, scan.getColumns());
 
     if (!columnExplorer.isSelectAllColumns()) {
+      // We must make sure to pass a table column (not to be confused with implicit column)
to the underlying record reader.
+      List<SchemaPath> tableColumns =
+          columnExplorer.getTableColumns().size() == 0 ?
+              Lists.<SchemaPath>newArrayList(AbstractRecordReader.STAR_COLUMN) : columnExplorer.getTableColumns();
       scan = new EasySubScan(scan.getUserName(), scan.getWorkUnits(), scan.getFormatPlugin(),
-          columnExplorer.getTableColumns(), scan.getSelectionRoot());
+          tableColumns, scan.getSelectionRoot());
       scan.setOperatorId(scan.getOperatorId());
     }
 

http://git-wip-us.apache.org/repos/asf/drill/blob/19cdcd95/exec/java-exec/src/test/java/org/apache/drill/exec/store/TestImplicitFileColumns.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/TestImplicitFileColumns.java
b/exec/java-exec/src/test/java/org/apache/drill/exec/store/TestImplicitFileColumns.java
index 6900da9..ce0f5e9 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/store/TestImplicitFileColumns.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/store/TestImplicitFileColumns.java
@@ -20,6 +20,7 @@ package org.apache.drill.exec.store;
 import com.google.common.base.Charsets;
 import com.google.common.io.Files;
 import org.apache.drill.BaseTestQuery;
+import org.apache.drill.common.util.TestTools;
 import org.apache.drill.exec.util.JsonStringArrayList;
 import org.apache.drill.exec.util.Text;
 import org.apache.hadoop.fs.Path;
@@ -110,4 +111,20 @@ public class TestImplicitFileColumns extends BaseTestQuery {
         .go();
   }
 
+  @Test // DRILL-4733
+  public void testMultilevelParquetWithSchemaChange() throws Exception {
+    try {
+      test("alter session set `planner.enable_decimal_data_type` = true");
+      testBuilder()
+          .sqlQuery(String.format("select max(dir0) as max_dir from dfs_test.`%s/src/test/resources/multilevel/parquetWithSchemaChange`",
+              TestTools.getWorkingPath()))
+          .unOrdered()
+          .baselineColumns("max_dir")
+          .baselineValues("voter50")
+          .go();
+    } finally {
+      test("alter session set `planner.enable_decimal_data_type` = false");
+    }
+  }
+
 }

http://git-wip-us.apache.org/repos/asf/drill/blob/19cdcd95/exec/java-exec/src/test/resources/multilevel/parquetWithSchemaChange/voter5/voter5.parquet
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/resources/multilevel/parquetWithSchemaChange/voter5/voter5.parquet
b/exec/java-exec/src/test/resources/multilevel/parquetWithSchemaChange/voter5/voter5.parquet
new file mode 100644
index 0000000..cc76280
Binary files /dev/null and b/exec/java-exec/src/test/resources/multilevel/parquetWithSchemaChange/voter5/voter5.parquet
differ

http://git-wip-us.apache.org/repos/asf/drill/blob/19cdcd95/exec/java-exec/src/test/resources/multilevel/parquetWithSchemaChange/voter50/voter50.parquet
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/resources/multilevel/parquetWithSchemaChange/voter50/voter50.parquet
b/exec/java-exec/src/test/resources/multilevel/parquetWithSchemaChange/voter50/voter50.parquet
new file mode 100644
index 0000000..ff66b42
Binary files /dev/null and b/exec/java-exec/src/test/resources/multilevel/parquetWithSchemaChange/voter50/voter50.parquet
differ


Mime
View raw message