drill-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From amansi...@apache.org
Subject drill git commit: DRILL-3710: New option for the IN LIST size to convert into join
Date Fri, 29 Jul 2016 14:37:28 GMT
Repository: drill
Updated Branches:
  refs/heads/master 69a44ed79 -> 4e1bdac08


DRILL-3710: New option for the IN LIST size to convert into join

Cosmetic changes

close apache/drill#552


Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/4e1bdac0
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/4e1bdac0
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/4e1bdac0

Branch: refs/heads/master
Commit: 4e1bdac08d12ee8b0375d4437fa0d52f9ead124d
Parents: 69a44ed
Author: Gautam Parai <gparai@maprtech.com>
Authored: Tue Jul 19 15:21:59 2016 -0700
Committer: Aman Sinha <asinha@maprtech.com>
Committed: Fri Jul 29 07:35:24 2016 -0700

----------------------------------------------------------------------
 .../exec/planner/physical/PlannerSettings.java  |  6 +++
 .../drill/exec/planner/sql/SqlConverter.java    | 51 ++++++++++++++++++--
 .../server/options/SystemOptionManager.java     |  1 +
 .../org/apache/drill/TestPartitionFilter.java   | 10 ++++
 pom.xml                                         |  2 +-
 5 files changed, 65 insertions(+), 5 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/drill/blob/4e1bdac0/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/PlannerSettings.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/PlannerSettings.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/PlannerSettings.java
index 2a23d8b..51d3708 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/PlannerSettings.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/PlannerSettings.java
@@ -94,6 +94,8 @@ public class PlannerSettings implements Context{
 
   public static final String TYPE_INFERENCE_KEY = "planner.enable_type_inference";
   public static final BooleanValidator TYPE_INFERENCE = new BooleanValidator(TYPE_INFERENCE_KEY,
true);
+  public static final LongValidator IN_SUBQUERY_THRESHOLD =
+      new PositiveLongValidator("planner.in_subquery_threshold", Integer.MAX_VALUE, 20);
/* Same as Calcite's default IN List subquery size */
 
   public OptionManager options = null;
   public FunctionImplementationRegistry functionImplementationRegistry = null;
@@ -235,6 +237,10 @@ public class PlannerSettings implements Context{
     return options.getOption(TYPE_INFERENCE);
   }
 
+  public long getInSubqueryThreshold() {
+    return options.getOption(IN_SUBQUERY_THRESHOLD);
+  }
+
   @Override
   public <T> T unwrap(Class<T> clazz) {
     if(clazz == PlannerSettings.class){

http://git-wip-us.apache.org/repos/asf/drill/blob/4e1bdac0/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/SqlConverter.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/SqlConverter.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/SqlConverter.java
index a7fcf99..3d0d538 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/SqlConverter.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/SqlConverter.java
@@ -65,6 +65,7 @@ import org.apache.drill.exec.planner.cost.DrillCostBase;
 import org.apache.drill.exec.planner.logical.DrillConstExecutor;
 import org.apache.drill.exec.planner.physical.DrillDistributionTraitDef;
 import org.apache.drill.exec.planner.physical.PlannerSettings;
+import org.apache.drill.exec.planner.physical.PrelUtil;
 import org.apache.drill.exec.planner.sql.parser.impl.DrillParserWithCompoundIdConverter;
 
 import com.google.common.base.Joiner;
@@ -79,6 +80,8 @@ public class SqlConverter {
 
   private final JavaTypeFactory typeFactory;
   private final SqlParser.Config parserConfig;
+  // Allow the default config to be modified using immutable configs
+  private SqlToRelConverter.Config sqlToRelConverterConfig;
   private final CalciteCatalogReader catalog;
   private final PlannerSettings settings;
   private final SchemaPlus rootSchema;
@@ -100,6 +103,7 @@ public class SqlConverter {
     this.util = util;
     this.functions = functions;
     this.parserConfig = new ParserConfig();
+    this.sqlToRelConverterConfig = new SqlToRelConverterConfig();
     this.isInnerQuery = false;
     this.typeFactory = new JavaTypeFactoryImpl(DRILL_TYPE_SYSTEM);
     this.defaultSchema = defaultSchema;
@@ -118,6 +122,7 @@ public class SqlConverter {
   private SqlConverter(SqlConverter parent, SchemaPlus defaultSchema, SchemaPlus rootSchema,
       CalciteCatalogReader catalog) {
     this.parserConfig = parent.parserConfig;
+    this.sqlToRelConverterConfig = parent.sqlToRelConverterConfig;
     this.defaultSchema = defaultSchema;
     this.functions = parent.functions;
     this.util = parent.util;
@@ -244,10 +249,8 @@ public class SqlConverter {
 
     final RelOptCluster cluster = RelOptCluster.create(planner, rexBuilder);
     final SqlToRelConverter sqlToRelConverter =
-        new SqlToRelConverter(new Expander(), validator, catalog, cluster, DrillConvertletTable.INSTANCE);
-
-    sqlToRelConverter.setTrimUnusedFields(false);
-    sqlToRelConverter.enableTableAccessConversion(false);
+        new SqlToRelConverter(new Expander(), validator, catalog, cluster, DrillConvertletTable.INSTANCE,
+            sqlToRelConverterConfig);
     final RelNode rel = sqlToRelConverter.convertQuery(validatedNode, false, !isInnerQuery);
     final RelNode rel2 = sqlToRelConverter.flattenTypes(rel, true);
     final RelNode rel3 = RelDecorrelator.decorrelateQuery(rel2);
@@ -345,6 +348,46 @@ public class SqlConverter {
 
   }
 
+  private class SqlToRelConverterConfig implements SqlToRelConverter.Config {
+
+    final int inSubqueryThreshold = (int)settings.getInSubqueryThreshold();
+
+    @Override
+    public boolean isConvertTableAccess() {
+      return false;
+    }
+
+    @Override
+    public boolean isDecorrelationEnabled() {
+      return SqlToRelConverterConfig.DEFAULT.isDecorrelationEnabled();
+    }
+
+    @Override
+    public boolean isTrimUnusedFields() {
+      return false;
+    }
+
+    @Override
+    public boolean isCreateValuesRel() {
+      return SqlToRelConverterConfig.DEFAULT.isCreateValuesRel();
+    }
+
+    @Override
+    public boolean isExplain() {
+      return SqlToRelConverterConfig.DEFAULT.isExplain();
+    }
+
+    @Override
+    public boolean isExpand() {
+      return SqlToRelConverterConfig.DEFAULT.isExpand();
+    }
+
+    @Override
+    public int getInSubqueryThreshold() {
+      return inSubqueryThreshold;
+    }
+  }
+
   /**
    *
    * @param sql

http://git-wip-us.apache.org/repos/asf/drill/blob/4e1bdac0/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SystemOptionManager.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SystemOptionManager.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SystemOptionManager.java
index cfa3c40..3053c85 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SystemOptionManager.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SystemOptionManager.java
@@ -86,6 +86,7 @@ public class SystemOptionManager extends BaseOptionManager implements AutoClosea
       PlannerSettings.FILTER_MIN_SELECTIVITY_ESTIMATE_FACTOR,
       PlannerSettings.FILTER_MAX_SELECTIVITY_ESTIMATE_FACTOR,
       PlannerSettings.TYPE_INFERENCE,
+      PlannerSettings.IN_SUBQUERY_THRESHOLD,
       ExecConstants.CAST_TO_NULLABLE_NUMERIC_OPTION,
       ExecConstants.OUTPUT_FORMAT_VALIDATOR,
       ExecConstants.PARQUET_BLOCK_SIZE_VALIDATOR,

http://git-wip-us.apache.org/repos/asf/drill/blob/4e1bdac0/exec/java-exec/src/test/java/org/apache/drill/TestPartitionFilter.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/TestPartitionFilter.java b/exec/java-exec/src/test/java/org/apache/drill/TestPartitionFilter.java
index be2bfaf..48ecf21 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/TestPartitionFilter.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/TestPartitionFilter.java
@@ -376,4 +376,14 @@ public class TestPartitionFilter extends PlanTestBase {
     testIncludeFilter(query4, 4, "Filter", 16);
   }
 
+  @Test //DRILL-3710 Partition pruning should occur with varying IN-LIST size
+  public void testPartitionFilterWithInSubquery() throws Exception {
+    String query = String.format("select * from dfs_test.`%s/multilevel/parquet` where cast
(dir0 as int) IN (1994, 1994, 1994, 1994, 1994, 1994)", TEST_RES_PATH);
+    /* In list size exceeds threshold - no partition pruning since predicate converted to
join */
+    test("alter session set `planner.in_subquery_threshold` = 2");
+    testExcludeFilter(query, 12, "Filter", 40);
+    /* In list size does not exceed threshold - partition pruning */
+    test("alter session set `planner.in_subquery_threshold` = 10");
+    testExcludeFilter(query, 4, "Filter", 40);
+  }
 }

http://git-wip-us.apache.org/repos/asf/drill/blob/4e1bdac0/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 0f9d4c0..c953f88 100644
--- a/pom.xml
+++ b/pom.xml
@@ -1287,7 +1287,7 @@
           <dependency>
             <groupId>org.apache.calcite</groupId>
             <artifactId>calcite-core</artifactId>
-            <version>1.4.0-drill-r14</version>
+            <version>1.4.0-drill-r15</version>
             <exclusions>
               <exclusion>
                 <groupId>org.jgrapht</groupId>


Mime
View raw message