drill-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From j..@apache.org
Subject drill git commit: DRILL-4323: Handle skipAll query when use HiveDrillNativeParquetScan
Date Fri, 05 Feb 2016 06:57:37 GMT
Repository: drill
Updated Branches:
  refs/heads/master ca53c2440 -> af98e93f0


DRILL-4323: Handle skipAll query when use HiveDrillNativeParquetScan

Do not add Project when no column is needed to be read out from Scan (e.g., select count(*)
from hive.table)


Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/af98e93f
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/af98e93f
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/af98e93f

Branch: refs/heads/master
Commit: af98e93f09f13e82881637a68f40bbb0bc92586c
Parents: ca53c24
Author: Hsuan-Yi Chu <hsuanyi@usc.edu>
Authored: Fri Jan 29 13:20:12 2016 -0800
Committer: Jinfeng Ni <jni@apache.org>
Committed: Thu Feb 4 21:11:21 2016 -0800

----------------------------------------------------------------------
 .../ConvertHiveParquetScanToDrillParquetScan.java   |  9 ++++++---
 .../org/apache/drill/exec/hive/TestHiveStorage.java |  9 ++++++++-
 .../exec/hive/TestInfoSchemaOnHiveStorage.java      |  1 +
 .../exec/store/hive/HiveTestDataGenerator.java      | 16 ++++++++++++++++
 4 files changed, 31 insertions(+), 4 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/drill/blob/af98e93f/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/planner/sql/logical/ConvertHiveParquetScanToDrillParquetScan.java
----------------------------------------------------------------------
diff --git a/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/planner/sql/logical/ConvertHiveParquetScanToDrillParquetScan.java
b/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/planner/sql/logical/ConvertHiveParquetScanToDrillParquetScan.java
index a1933be..97a5b98 100644
--- a/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/planner/sql/logical/ConvertHiveParquetScanToDrillParquetScan.java
+++ b/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/planner/sql/logical/ConvertHiveParquetScanToDrillParquetScan.java
@@ -177,9 +177,12 @@ public class ConvertHiveParquetScanToDrillParquetScan extends StoragePluginOptim
           getPartitionColMapping(hiveTable, partitionColumnLabel);
 
       final DrillScanRel nativeScanRel = createNativeScanRel(partitionColMapping, hiveScanRel);
-      final DrillProjectRel projectRel = createProjectRel(hiveScanRel, partitionColMapping,
nativeScanRel);
-
-      call.transformTo(projectRel);
+      if(hiveScanRel.getRowType().getFieldCount() == 0) {
+        call.transformTo(nativeScanRel);
+      } else {
+        final DrillProjectRel projectRel = createProjectRel(hiveScanRel, partitionColMapping,
nativeScanRel);
+        call.transformTo(projectRel);
+      }
     } catch (final Exception e) {
       logger.warn("Failed to convert HiveScan to HiveDrillNativeParquetScan", e);
     }

http://git-wip-us.apache.org/repos/asf/drill/blob/af98e93f/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/hive/TestHiveStorage.java
----------------------------------------------------------------------
diff --git a/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/hive/TestHiveStorage.java
b/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/hive/TestHiveStorage.java
index c2e367d..55de2d7 100644
--- a/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/hive/TestHiveStorage.java
+++ b/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/hive/TestHiveStorage.java
@@ -44,8 +44,15 @@ public class TestHiveStorage extends HiveTestBase {
     try {
       test(String.format("alter session set `%s` = true", ExecConstants.HIVE_OPTIMIZE_SCAN_WITH_NATIVE_READERS));
 
-      String query = "SELECT count(*) FROM hive.readtest_parquet";
+      String query = "SELECT count(*) as col FROM hive.countStar_Parquet";
       testPhysicalPlan(query, "hive-drill-native-parquet-scan");
+
+      testBuilder()
+          .sqlQuery(query)
+          .unOrdered()
+          .baselineColumns("col")
+          .baselineValues(200l)
+          .go();
     } finally {
       test(String.format("alter session set `%s` = %s",
           ExecConstants.HIVE_OPTIMIZE_SCAN_WITH_NATIVE_READERS,

http://git-wip-us.apache.org/repos/asf/drill/blob/af98e93f/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/hive/TestInfoSchemaOnHiveStorage.java
----------------------------------------------------------------------
diff --git a/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/hive/TestInfoSchemaOnHiveStorage.java
b/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/hive/TestInfoSchemaOnHiveStorage.java
index 9352ce0..f9fc0ac 100644
--- a/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/hive/TestInfoSchemaOnHiveStorage.java
+++ b/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/hive/TestInfoSchemaOnHiveStorage.java
@@ -41,6 +41,7 @@ public class TestInfoSchemaOnHiveStorage extends HiveTestBase {
         .baselineValues("hive.default", "kv")
         .baselineValues("hive.default", "kv_parquet")
         .baselineValues("hive.default", "kv_sh")
+        .baselineValues("hive.default", "countstar_parquet")
         .go();
 
     testBuilder()

http://git-wip-us.apache.org/repos/asf/drill/blob/af98e93f/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/store/hive/HiveTestDataGenerator.java
----------------------------------------------------------------------
diff --git a/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/store/hive/HiveTestDataGenerator.java
b/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/store/hive/HiveTestDataGenerator.java
index f42e8d7..b38290b 100644
--- a/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/store/hive/HiveTestDataGenerator.java
+++ b/contrib/storage-hive/core/src/test/java/org/apache/drill/exec/store/hive/HiveTestDataGenerator.java
@@ -440,6 +440,22 @@ public class HiveTestDataGenerator {
     executeQuery(hiveDriver, "INSERT INTO TABLE kv_parquet PARTITION(part1) SELECT key, value,
key FROM default.kv");
     executeQuery(hiveDriver, "ALTER TABLE kv_parquet ADD COLUMNS (newcol string)");
 
+    executeQuery(hiveDriver,
+        "CREATE TABLE countStar_Parquet (int_field INT) STORED AS parquet");
+
+    final int numOfRows = 200;
+    final StringBuffer sb = new StringBuffer();
+    sb.append("VALUES ");
+    for(int i = 0; i < numOfRows; ++i) {
+      if(i != 0) {
+        sb.append(",");
+      }
+      sb.append("(").append(i).append(")");
+    }
+
+    executeQuery(hiveDriver, "INSERT INTO TABLE countStar_Parquet \n" +
+        sb.toString());
+
     // Create a StorageHandler based table (DRILL-3739)
     executeQuery(hiveDriver, "CREATE TABLE kv_sh(key INT, value STRING) STORED BY " +
         "'org.apache.hadoop.hive.ql.metadata.DefaultStorageHandler'");


Mime
View raw message