drill-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From j..@apache.org
Subject drill git commit: DRILL-3765: Move partitioning pruning to HepPlanner to avoid the performance overhead for redundant rule execution.
Date Tue, 17 Nov 2015 04:27:11 GMT
Repository: drill
Updated Branches:
  refs/heads/master ea8e17d7b -> e0c9b841d


DRILL-3765: Move partitioning pruning to HepPlanner to avoid the performance overhead for
redundant rule execution.

Add fall back option in planner.

close apache/drill#255


Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/e0c9b841
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/e0c9b841
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/e0c9b841

Branch: refs/heads/master
Commit: e0c9b841d4001c2574cf015a91974c536528d345
Parents: ea8e17d
Author: Jinfeng Ni <jni@apache.org>
Authored: Tue Nov 3 15:51:10 2015 -0800
Committer: Jinfeng Ni <jni@apache.org>
Committed: Mon Nov 16 16:26:43 2015 -0800

----------------------------------------------------------------------
 .../HivePushPartitionFilterIntoScan.java        | 13 ++-
 .../exec/planner/common/DrillJoinRelBase.java   |  2 +-
 .../exec/planner/logical/DrillRuleSets.java     | 27 +++++-
 .../exec/planner/logical/DrillScanRel.java      | 32 ++++++-
 .../logical/partition/ParquetPruneScanRule.java | 13 ++-
 .../logical/partition/PruneScanRule.java        | 15 +++-
 .../exec/planner/physical/PlannerSettings.java  |  7 +-
 .../drill/exec/planner/sql/DrillSqlWorker.java  | 27 ++++--
 .../planner/sql/handlers/DefaultSqlHandler.java | 88 ++++++++++----------
 .../server/options/SystemOptionManager.java     |  3 +-
 10 files changed, 159 insertions(+), 68 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/drill/blob/e0c9b841/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/planner/sql/logical/HivePushPartitionFilterIntoScan.java
----------------------------------------------------------------------
diff --git a/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/planner/sql/logical/HivePushPartitionFilterIntoScan.java
b/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/planner/sql/logical/HivePushPartitionFilterIntoScan.java
index 105a07e..fc2007e 100644
--- a/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/planner/sql/logical/HivePushPartitionFilterIntoScan.java
+++ b/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/planner/sql/logical/HivePushPartitionFilterIntoScan.java
@@ -27,6 +27,7 @@ import org.apache.drill.exec.planner.logical.DrillScanRel;
 import org.apache.drill.exec.planner.logical.RelOptHelper;
 import org.apache.drill.exec.planner.logical.partition.PruneScanRule;
 import org.apache.drill.exec.planner.physical.PlannerSettings;
+import org.apache.drill.exec.planner.physical.PrelUtil;
 import org.apache.drill.exec.planner.sql.HivePartitionDescriptor;
 import org.apache.drill.exec.store.StoragePluginOptimizerRule;
 import org.apache.drill.exec.store.hive.HiveScan;
@@ -52,7 +53,11 @@ public abstract class HivePushPartitionFilterIntoScan {
         final DrillScanRel scan = (DrillScanRel) call.rel(2);
         GroupScan groupScan = scan.getGroupScan();
         // this rule is applicable only for Hive based partition pruning
-        return groupScan instanceof HiveScan && groupScan.supportsPartitionFilterPushdown();
+        if (PrelUtil.getPlannerSettings(scan.getCluster().getPlanner()).isHepPartitionPruningEnabled())
{
+          return groupScan instanceof HiveScan && groupScan.supportsPartitionFilterPushdown()
&& !scan.partitionFilterPushdown();
+        } else {
+          return groupScan instanceof HiveScan && groupScan.supportsPartitionFilterPushdown();
+        }
       }
 
       @Override
@@ -82,7 +87,11 @@ public abstract class HivePushPartitionFilterIntoScan {
         final DrillScanRel scan = (DrillScanRel) call.rel(1);
         GroupScan groupScan = scan.getGroupScan();
         // this rule is applicable only for Hive based partition pruning
-        return groupScan instanceof HiveScan && groupScan.supportsPartitionFilterPushdown();
+        if (PrelUtil.getPlannerSettings(scan.getCluster().getPlanner()).isHepPartitionPruningEnabled())
{
+          return groupScan instanceof HiveScan && groupScan.supportsPartitionFilterPushdown()
&& !scan.partitionFilterPushdown();
+        } else {
+          return groupScan instanceof HiveScan && groupScan.supportsPartitionFilterPushdown();
+        }
       }
 
       @Override

http://git-wip-us.apache.org/repos/asf/drill/blob/e0c9b841/exec/java-exec/src/main/java/org/apache/drill/exec/planner/common/DrillJoinRelBase.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/common/DrillJoinRelBase.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/common/DrillJoinRelBase.java
index 7d3ad0a..2d6f7d6 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/common/DrillJoinRelBase.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/common/DrillJoinRelBase.java
@@ -74,7 +74,7 @@ public abstract class DrillJoinRelBase extends Join implements DrillRelNode
{
              *   - Such filter on top of join might be pushed into JOIN, when LOPT planner
is called.
              *   - Return non-infinite cost will give LOPT planner a chance to try to push
the filters.
              */
-            if (PrelUtil.getPlannerSettings(planner).isHepJoinOptEnabled()) {
+            if (PrelUtil.getPlannerSettings(planner).isHepOptEnabled()) {
              return computeCartesianJoinCost(planner);
             } else {
               return ((DrillCostFactory)planner.getCostFactory()).makeInfiniteCost();

http://git-wip-us.apache.org/repos/asf/drill/blob/e0c9b841/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillRuleSets.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillRuleSets.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillRuleSets.java
index 16ec559..b7974f3 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillRuleSets.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillRuleSets.java
@@ -31,12 +31,15 @@ import org.apache.calcite.rel.rules.FilterMergeRule;
 import org.apache.calcite.rel.rules.FilterSetOpTransposeRule;
 import org.apache.calcite.rel.rules.JoinPushExpressionsRule;
 import org.apache.calcite.rel.rules.JoinPushThroughJoinRule;
+import org.apache.calcite.rel.rules.JoinToMultiJoinRule;
+import org.apache.calcite.rel.rules.LoptOptimizeJoinRule;
 import org.apache.calcite.rel.rules.ProjectRemoveRule;
 import org.apache.calcite.rel.rules.ProjectWindowTransposeRule;
 import org.apache.calcite.rel.rules.ReduceExpressionsRule;
 import org.apache.calcite.rel.rules.SortRemoveRule;
 import org.apache.calcite.rel.rules.UnionToDistinctRule;
 import org.apache.calcite.tools.RuleSet;
+import org.apache.commons.digester.Rules;
 import org.apache.drill.exec.ops.OptimizerRulesContext;
 import org.apache.drill.exec.planner.logical.partition.ParquetPruneScanRule;
 import org.apache.drill.exec.planner.logical.partition.PruneScanRule;
@@ -67,6 +70,12 @@ import com.google.common.collect.ImmutableSet.Builder;
 public class DrillRuleSets {
   //private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(DrillRuleSets.class);
 
+  public static final RelOptRule DRILL_JOIN_TO_MULTIJOIN_RULE = new JoinToMultiJoinRule(DrillJoinRel.class);
+  public static final RelOptRule DRILL_LOPT_OPTIMIZE_JOIN_RULE = new LoptOptimizeJoinRule(
+      DrillRelFactories.DRILL_LOGICAL_JOIN_FACTORY,
+      DrillRelFactories.DRILL_LOGICAL_PROJECT_FACTORY,
+      DrillRelFactories.DRILL_LOGICAL_FILTER_FACTORY);
+
   /**
    * Get a list of logical rules that can be turned on or off by session/system options.
    *
@@ -90,7 +99,6 @@ public class DrillRuleSets {
     if (ps.isConstantFoldingEnabled()) {
       // TODO - DRILL-2218
       userConfigurableRules.add(ReduceExpressionsRule.PROJECT_INSTANCE);
-
       userConfigurableRules.add(DrillReduceExpressionsRule.FILTER_INSTANCE_DRILL);
       userConfigurableRules.add(DrillReduceExpressionsRule.CALC_INSTANCE_DRILL);
     }
@@ -183,16 +191,27 @@ public class DrillRuleSets {
         .addAll(staticRuleSet)
         .add(
             DrillMergeProjectRule.getInstance(true, RelFactories.DEFAULT_PROJECT_FACTORY,
-                optimizerRulesContext.getFunctionRegistry()),
+                optimizerRulesContext.getFunctionRegistry())
+            )
+        .build();
+
+    return new DrillRuleSet(basicRules);
+  }
 
+  /**
+   *   Get an immutable list of partition pruning rules that will be used in logical planning.
+   */
+  public static RuleSet getPruneScanRules(OptimizerRulesContext optimizerRulesContext) {
+    final ImmutableSet<RelOptRule> pruneRules = ImmutableSet.<RelOptRule>builder()
+        .add(
             PruneScanRule.getFilterOnProject(optimizerRulesContext),
             PruneScanRule.getFilterOnScan(optimizerRulesContext),
             ParquetPruneScanRule.getFilterOnProjectParquet(optimizerRulesContext),
             ParquetPruneScanRule.getFilterOnScanParquet(optimizerRulesContext)
-            )
+        )
         .build();
 
-    return new DrillRuleSet(basicRules);
+    return new DrillRuleSet(pruneRules);
   }
 
   // Ruleset for join permutation, used only in VolcanoPlanner.

http://git-wip-us.apache.org/repos/asf/drill/blob/e0c9b841/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillScanRel.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillScanRel.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillScanRel.java
index 9a009ce..0d087aa 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillScanRel.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillScanRel.java
@@ -54,19 +54,31 @@ public class DrillScanRel extends DrillScanRelBase implements DrillRel
{
   private GroupScan groupScan;
   private List<SchemaPath> columns;
   private PlannerSettings settings;
+  private final boolean partitionFilterPushdown;
 
   /** Creates a DrillScan. */
   public DrillScanRel(final RelOptCluster cluster, final RelTraitSet traits,
-      final RelOptTable table) {
+                      final RelOptTable table) {
+    this(cluster, traits, table, false);
+  }
+    /** Creates a DrillScan. */
+  public DrillScanRel(final RelOptCluster cluster, final RelTraitSet traits,
+      final RelOptTable table, boolean partitionFilterPushdown) {
     // By default, scan does not support project pushdown.
     // Decision whether push projects into scan will be made solely in DrillPushProjIntoScanRule.
-    this(cluster, traits, table, table.getRowType(), GroupScan.ALL_COLUMNS);
+    this(cluster, traits, table, table.getRowType(), GroupScan.ALL_COLUMNS, partitionFilterPushdown);
     this.settings = PrelUtil.getPlannerSettings(cluster.getPlanner());
   }
 
   /** Creates a DrillScan. */
   public DrillScanRel(final RelOptCluster cluster, final RelTraitSet traits,
-      final RelOptTable table, final RelDataType rowType, final List<SchemaPath> columns)
{
+                      final RelOptTable table, final RelDataType rowType, final List<SchemaPath>
columns) {
+    this(cluster, traits, table, rowType, columns, false);
+  }
+
+  /** Creates a DrillScan. */
+  public DrillScanRel(final RelOptCluster cluster, final RelTraitSet traits,
+      final RelOptTable table, final RelDataType rowType, final List<SchemaPath> columns,
boolean partitionFilterPushdown) {
     super(DRILL_LOGICAL, cluster, traits, table);
     this.settings = PrelUtil.getPlannerSettings(cluster.getPlanner());
     this.rowType = rowType;
@@ -77,6 +89,7 @@ public class DrillScanRel extends DrillScanRelBase implements DrillRel {
     } else { // planner asks to scan some columns
       this.columns  = ColumnList.some(columns);
     }
+    this.partitionFilterPushdown = partitionFilterPushdown;
     try {
       this.groupScan = drillTable.getGroupScan().clone(this.columns);
     } catch (final IOException e) {
@@ -86,12 +99,19 @@ public class DrillScanRel extends DrillScanRelBase implements DrillRel
{
 
   /** Creates a DrillScanRel for a particular GroupScan */
   public DrillScanRel(final RelOptCluster cluster, final RelTraitSet traits,
-      final RelOptTable table, final GroupScan groupScan, final RelDataType rowType, final
List<SchemaPath> columns) {
+                      final RelOptTable table, final GroupScan groupScan, final RelDataType
rowType, final List<SchemaPath> columns) {
+    this(cluster, traits, table, groupScan, rowType, columns, false);
+  }
+
+  /** Creates a DrillScanRel for a particular GroupScan */
+  public DrillScanRel(final RelOptCluster cluster, final RelTraitSet traits,
+      final RelOptTable table, final GroupScan groupScan, final RelDataType rowType, final
List<SchemaPath> columns, boolean partitionFilterPushdown) {
     super(DRILL_LOGICAL, cluster, traits, table);
     this.rowType = rowType;
     this.columns = columns;
     this.groupScan = groupScan;
     this.settings = PrelUtil.getPlannerSettings(cluster.getPlanner());
+    this.partitionFilterPushdown = partitionFilterPushdown;
   }
 
 //
@@ -180,4 +200,8 @@ public class DrillScanRel extends DrillScanRelBase implements DrillRel
{
     return groupScan;
   }
 
+  public boolean partitionFilterPushdown() {
+    return this.partitionFilterPushdown;
+  }
+
 }

http://git-wip-us.apache.org/repos/asf/drill/blob/e0c9b841/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/partition/ParquetPruneScanRule.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/partition/ParquetPruneScanRule.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/partition/ParquetPruneScanRule.java
index a951854..b4f4e95 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/partition/ParquetPruneScanRule.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/partition/ParquetPruneScanRule.java
@@ -29,6 +29,7 @@ import org.apache.drill.exec.planner.logical.DrillProjectRel;
 import org.apache.drill.exec.planner.logical.DrillScanRel;
 import org.apache.drill.exec.planner.logical.RelOptHelper;
 import org.apache.drill.exec.planner.physical.PlannerSettings;
+import org.apache.drill.exec.planner.physical.PrelUtil;
 import org.apache.drill.exec.store.parquet.ParquetGroupScan;
 
 public class ParquetPruneScanRule {
@@ -49,7 +50,11 @@ public class ParquetPruneScanRule {
         final DrillScanRel scan = (DrillScanRel) call.rel(2);
         GroupScan groupScan = scan.getGroupScan();
         // this rule is applicable only for parquet based partition pruning
-        return groupScan instanceof ParquetGroupScan && groupScan.supportsPartitionFilterPushdown();
+        if (PrelUtil.getPlannerSettings(scan.getCluster().getPlanner()).isHepPartitionPruningEnabled())
{
+          return groupScan instanceof ParquetGroupScan && groupScan.supportsPartitionFilterPushdown()
&& !scan.partitionFilterPushdown();
+        } else {
+          return groupScan instanceof ParquetGroupScan && groupScan.supportsPartitionFilterPushdown();
+        }
       }
 
       @Override
@@ -77,7 +82,11 @@ public class ParquetPruneScanRule {
         final DrillScanRel scan = (DrillScanRel) call.rel(1);
         GroupScan groupScan = scan.getGroupScan();
         // this rule is applicable only for parquet based partition pruning
-        return groupScan instanceof ParquetGroupScan && groupScan.supportsPartitionFilterPushdown();
+        if (PrelUtil.getPlannerSettings(scan.getCluster().getPlanner()).isHepPartitionPruningEnabled())
{
+          return groupScan instanceof ParquetGroupScan && groupScan.supportsPartitionFilterPushdown()
&& !scan.partitionFilterPushdown();
+        } else {
+          return groupScan instanceof ParquetGroupScan && groupScan.supportsPartitionFilterPushdown();
+        }
       }
 
       @Override

http://git-wip-us.apache.org/repos/asf/drill/blob/e0c9b841/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/partition/PruneScanRule.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/partition/PruneScanRule.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/partition/PruneScanRule.java
index b7b58ec..ab6c5a8 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/partition/PruneScanRule.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/partition/PruneScanRule.java
@@ -94,7 +94,11 @@ public abstract class PruneScanRule extends StoragePluginOptimizerRule
{
         final DrillScanRel scan = (DrillScanRel) call.rel(2);
         GroupScan groupScan = scan.getGroupScan();
         // this rule is applicable only for dfs based partition pruning
-        return groupScan instanceof FileGroupScan && groupScan.supportsPartitionFilterPushdown();
+        if (PrelUtil.getPlannerSettings(scan.getCluster().getPlanner()).isHepPartitionPruningEnabled())
{
+          return groupScan instanceof FileGroupScan && groupScan.supportsPartitionFilterPushdown()
&& !scan.partitionFilterPushdown();
+        } else {
+          return groupScan instanceof FileGroupScan && groupScan.supportsPartitionFilterPushdown();
+        }
       }
 
       @Override
@@ -122,7 +126,11 @@ public abstract class PruneScanRule extends StoragePluginOptimizerRule
{
         final DrillScanRel scan = (DrillScanRel) call.rel(1);
         GroupScan groupScan = scan.getGroupScan();
         // this rule is applicable only for dfs based partition pruning
-        return groupScan instanceof FileGroupScan && groupScan.supportsPartitionFilterPushdown();
+        if (PrelUtil.getPlannerSettings(scan.getCluster().getPlanner()).isHepPartitionPruningEnabled())
{
+          return groupScan instanceof FileGroupScan && groupScan.supportsPartitionFilterPushdown()
&& !scan.partitionFilterPushdown();
+        } else {
+          return groupScan instanceof FileGroupScan && groupScan.supportsPartitionFilterPushdown();
+        }
       }
 
       @Override
@@ -319,7 +327,8 @@ public abstract class PruneScanRule extends StoragePluginOptimizerRule
{
               scanRel.getTable(),
               descriptor.createNewGroupScan(newFiles),
               scanRel.getRowType(),
-              scanRel.getColumns());
+              scanRel.getColumns(),
+              true /*filter pushdown*/);
 
       RelNode inputRel = newScanRel;
 

http://git-wip-us.apache.org/repos/asf/drill/blob/e0c9b841/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/PlannerSettings.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/PlannerSettings.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/PlannerSettings.java
index 0a05d08..3eb5038 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/PlannerSettings.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/PlannerSettings.java
@@ -72,7 +72,8 @@ public class PlannerSettings implements Context{
   public static final OptionValidator HASH_JOIN_SWAP_MARGIN_FACTOR = new RangeDoubleValidator("planner.join.hash_join_swap_margin_factor",
0, 100, 10d);
   public static final String ENABLE_DECIMAL_DATA_TYPE_KEY = "planner.enable_decimal_data_type";
   public static final OptionValidator ENABLE_DECIMAL_DATA_TYPE = new BooleanValidator(ENABLE_DECIMAL_DATA_TYPE_KEY,
false);
-  public static final OptionValidator HEP_JOIN_OPT = new BooleanValidator("planner.enable_hep_join_opt",
true);
+  public static final OptionValidator HEP_OPT = new BooleanValidator("planner.enable_hep_opt",
true);
+  public static final OptionValidator HEP_PARTITION_PRUNING = new BooleanValidator("planner.enable_hep_partition_pruning",
true);
   public static final OptionValidator PLANNER_MEMORY_LIMIT = new RangeLongValidator("planner.memory_limit",
       INITIAL_OFF_HEAP_ALLOCATION_IN_BYTES, MAX_OFF_HEAP_ALLOCATION_IN_BYTES, DEFAULT_MAX_OFF_HEAP_ALLOCATION_IN_BYTES);
 
@@ -172,7 +173,9 @@ public class PlannerSettings implements Context{
     return options.getOption(HASH_JOIN_SWAP.getOptionName()).bool_val;
   }
 
-  public boolean isHepJoinOptEnabled() { return options.getOption(HEP_JOIN_OPT.getOptionName()).bool_val;}
+  public boolean isHepPartitionPruningEnabled() { return options.getOption(HEP_PARTITION_PRUNING.getOptionName()).bool_val;}
+
+  public boolean isHepOptEnabled() { return options.getOption(HEP_OPT.getOptionName()).bool_val;}
 
   public double getHashJoinSwapMarginFactor() {
     return options.getOption(HASH_JOIN_SWAP_MARGIN_FACTOR.getOptionName()).float_val / 100d;

http://git-wip-us.apache.org/repos/asf/drill/blob/e0c9b841/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/DrillSqlWorker.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/DrillSqlWorker.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/DrillSqlWorker.java
index 37f0ecd..92e59ec 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/DrillSqlWorker.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/DrillSqlWorker.java
@@ -74,7 +74,8 @@ public class DrillSqlWorker {
   private final HepPlanner hepPlanner;
   public final static int LOGICAL_RULES = 0;
   public final static int PHYSICAL_MEM_RULES = 1;
-  public final static int LOGICAL_CONVERT_RULES = 2;
+  public final static int LOGICAL_HEP_JOIN_RULES = 2;
+  public final static int LOGICAL_HEP_JOIN__PP_RULES = 3;
 
   private final QueryContext context;
 
@@ -116,20 +117,32 @@ public class DrillSqlWorker {
 
   private RuleSet[] getRules(QueryContext context) {
     StoragePluginRegistry storagePluginRegistry = context.getStorage();
-    RuleSet drillLogicalRules = DrillRuleSets.mergedRuleSets(
+
+    // Ruleset for the case where VolcanoPlanner is used for everything : join, filter/project
pushdown, partition pruning.
+    RuleSet drillLogicalVolOnlyRules = DrillRuleSets.mergedRuleSets(
         DrillRuleSets.getDrillBasicRules(context),
+        DrillRuleSets.getPruneScanRules(context),
         DrillRuleSets.getJoinPermRules(context),
         DrillRuleSets.getDrillUserConfigurableLogicalRules(context));
+
+    // Ruleset for the case where join planning is done in Hep-LOPT, filter/project pushdown
and parttion pruning are done in VolcanoPlanner
+    RuleSet drillLogicalHepJoinRules = DrillRuleSets.mergedRuleSets(
+        DrillRuleSets.getDrillBasicRules(context),
+        DrillRuleSets.getPruneScanRules(context),
+        DrillRuleSets.getDrillUserConfigurableLogicalRules(context));
+
+    // Ruleset for the case where join planning and partition pruning is done in Hep, filter/project
pushdown are done in VolcanoPlanner
+    RuleSet drillLogicalHepJoinPPRules = DrillRuleSets.mergedRuleSets(
+        DrillRuleSets.getDrillBasicRules(context),
+        DrillRuleSets.getDrillUserConfigurableLogicalRules(context));
+
+    // Ruleset for physical planning rules
     RuleSet drillPhysicalMem = DrillRuleSets.mergedRuleSets(
         DrillRuleSets.getPhysicalRules(context),
         storagePluginRegistry.getStoragePluginRuleSet(context));
 
-    // Following is used in LOPT join OPT.
-    RuleSet logicalConvertRules = DrillRuleSets.mergedRuleSets(
-        DrillRuleSets.getDrillBasicRules(context),
-        DrillRuleSets.getDrillUserConfigurableLogicalRules(context));
 
-    RuleSet[] allRules = new RuleSet[] {drillLogicalRules, drillPhysicalMem, logicalConvertRules};
+    RuleSet[] allRules = new RuleSet[] {drillLogicalVolOnlyRules, drillPhysicalMem, drillLogicalHepJoinRules,
drillLogicalHepJoinPPRules};
 
     return allRules;
   }

http://git-wip-us.apache.org/repos/asf/drill/blob/e0c9b841/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/DefaultSqlHandler.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/DefaultSqlHandler.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/DefaultSqlHandler.java
index 9d11b96..686f7d7 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/DefaultSqlHandler.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/DefaultSqlHandler.java
@@ -21,8 +21,11 @@ import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.List;
+import java.util.Set;
 
+import com.google.common.collect.ImmutableSet;
 import org.apache.calcite.plan.RelOptPlanner;
+import org.apache.calcite.plan.RelOptRule;
 import org.apache.calcite.plan.RelOptUtil;
 import org.apache.calcite.plan.RelTraitSet;
 import org.apache.calcite.plan.hep.HepMatchOrder;
@@ -68,12 +71,17 @@ import org.apache.drill.exec.physical.impl.join.JoinUtils;
 import org.apache.drill.exec.planner.common.DrillRelOptUtil;
 import org.apache.drill.exec.planner.cost.DrillDefaultRelMetadataProvider;
 import org.apache.drill.exec.planner.logical.DrillJoinRel;
+import org.apache.drill.exec.planner.logical.DrillMergeProjectRule;
 import org.apache.drill.exec.planner.logical.DrillProjectRel;
+import org.apache.drill.exec.planner.logical.DrillPushProjectPastFilterRule;
 import org.apache.drill.exec.planner.logical.DrillRel;
 import org.apache.drill.exec.planner.logical.DrillRelFactories;
+import org.apache.drill.exec.planner.logical.DrillRuleSets;
 import org.apache.drill.exec.planner.logical.DrillScreenRel;
 import org.apache.drill.exec.planner.logical.DrillStoreRel;
 import org.apache.drill.exec.planner.logical.PreProcessLogicalRel;
+import org.apache.drill.exec.planner.logical.partition.ParquetPruneScanRule;
+import org.apache.drill.exec.planner.logical.partition.PruneScanRule;
 import org.apache.drill.exec.planner.physical.DrillDistributionTrait;
 import org.apache.drill.exec.planner.physical.PhysicalPlanCreator;
 import org.apache.drill.exec.planner.physical.PlannerSettings;
@@ -207,11 +215,7 @@ public class DefaultSqlHandler extends AbstractSqlHandler {
     try {
       final DrillRel convertedRelNode;
 
-      if (! context.getPlannerSettings().isHepJoinOptEnabled()) {
-        convertedRelNode = (DrillRel) logicalPlanningVolcano(relNode);
-      } else {
-        convertedRelNode = (DrillRel) logicalPlanningVolcanoAndLopt(relNode);
-      }
+      convertedRelNode = (DrillRel) doLogicalPlanning(relNode);
 
       if (convertedRelNode instanceof DrillStoreRel) {
         throw new UnsupportedOperationException();
@@ -529,55 +533,56 @@ public class DefaultSqlHandler extends AbstractSqlHandler {
   }
 
 
-  private RelNode logicalPlanningVolcano(RelNode relNode) throws RelConversionException,
SqlUnsupportedException {
-    return planner.transform(DrillSqlWorker.LOGICAL_RULES, relNode.getTraitSet().plus(DrillRel.DRILL_LOGICAL),
relNode);
-  }
+  private RelNode doLogicalPlanning(RelNode relNode) throws RelConversionException, SqlUnsupportedException
{
+    if (! context.getPlannerSettings().isHepOptEnabled()) {
+      return planner.transform(DrillSqlWorker.LOGICAL_RULES, relNode.getTraitSet().plus(DrillRel.DRILL_LOGICAL),
relNode);
+    } else {
+      RelNode convertedRelNode = null;
+      if (context.getPlannerSettings().isHepPartitionPruningEnabled()) {
+        convertedRelNode = planner.transform(DrillSqlWorker.LOGICAL_HEP_JOIN__PP_RULES, relNode.getTraitSet().plus(DrillRel.DRILL_LOGICAL),
relNode);
+        log("VolcanoRel", convertedRelNode, logger);
 
-  /**
-   * Do logical planning using both VolcanoPlanner and LOPT HepPlanner.
-   * @param relNode
-   * @return
-   * @throws RelConversionException
-   * @throws SqlUnsupportedException
-   */
-  private RelNode logicalPlanningVolcanoAndLopt(RelNode relNode) throws RelConversionException,
SqlUnsupportedException {
+        // Partition pruning .
+        ImmutableSet<RelOptRule> pruneScanRules = ImmutableSet.<RelOptRule>builder()
+            .addAll(DrillRuleSets.getPruneScanRules(context))
+            .build();
 
-    final RelNode convertedRelNode = planner.transform(DrillSqlWorker.LOGICAL_CONVERT_RULES,
relNode.getTraitSet().plus(DrillRel.DRILL_LOGICAL), relNode);
-    log("VolCalciteRel", convertedRelNode, logger);
+        convertedRelNode = doHepPlan(convertedRelNode, pruneScanRules, HepMatchOrder.BOTTOM_UP);
+      } else {
+        convertedRelNode = planner.transform(DrillSqlWorker.LOGICAL_HEP_JOIN_RULES, relNode.getTraitSet().plus(DrillRel.DRILL_LOGICAL),
relNode);
+        log("VolcanoRel", convertedRelNode, logger);
+      }
 
-    final RelNode loptNode = getLoptJoinOrderTree(
-        convertedRelNode,
-        DrillJoinRel.class,
-        DrillRelFactories.DRILL_LOGICAL_JOIN_FACTORY,
-        DrillRelFactories.DRILL_LOGICAL_FILTER_FACTORY,
-        DrillRelFactories.DRILL_LOGICAL_PROJECT_FACTORY);
+      // Join order planning with LOPT rule
+      ImmutableSet<RelOptRule> joinOrderRules = ImmutableSet.<RelOptRule>builder()
+          .add(
+              DrillRuleSets.DRILL_JOIN_TO_MULTIJOIN_RULE,
+              DrillRuleSets.DRILL_LOPT_OPTIMIZE_JOIN_RULE,
+              ProjectRemoveRule.INSTANCE)
+          .build();
 
-    log("HepCalciteRel", loptNode, logger);
+      final RelNode loptNode = doHepPlan(convertedRelNode, joinOrderRules, HepMatchOrder.BOTTOM_UP);
 
-    return loptNode;
-  }
+      log("HepRel", loptNode, logger);
 
+      return loptNode;
+    }
+  }
 
   /**
-   * Appy Join Order Optimizations using Hep Planner.
+   * Use HepPlanner to apply optimization rules to RelNode tree.
+   * @return : the root node for optimized plan.
    */
-  private RelNode getLoptJoinOrderTree(RelNode root,
-                                      Class<? extends Join> joinClass,
-                                             RelFactories.JoinFactory joinFactory,
-                                             RelFactories.FilterFactory filterFactory,
-                                             RelFactories.ProjectFactory projectFactory)
{
+  private RelNode doHepPlan(RelNode root, Set<RelOptRule> rules, HepMatchOrder matchOrder)
{
     final HepProgramBuilder hepPgmBldr = new HepProgramBuilder()
-        .addMatchOrder(HepMatchOrder.BOTTOM_UP)
-        .addRuleInstance(new JoinToMultiJoinRule(joinClass))
-        .addRuleInstance(new LoptOptimizeJoinRule(joinFactory, projectFactory, filterFactory))
-        .addRuleInstance(ProjectRemoveRule.INSTANCE);
-        // .addRuleInstance(new ProjectMergeRule(true, projectFactory));
-
-        // .addRuleInstance(DrillMergeProjectRule.getInstance(true, projectFactory, this.context.getFunctionRegistry()));
+        .addMatchOrder(matchOrder);
 
+    for (final RelOptRule rule : rules) {
+      hepPgmBldr.addRuleInstance(rule);
+    }
 
     final HepProgram hepPgm = hepPgmBldr.build();
-    final HepPlanner hepPlanner = new HepPlanner(hepPgm);
+    final HepPlanner hepPlanner = new HepPlanner(hepPgm, context.getPlannerSettings());
 
     final List<RelMetadataProvider> list = Lists.newArrayList();
     list.add(DrillDefaultRelMetadataProvider.INSTANCE);
@@ -594,7 +599,6 @@ public class DefaultSqlHandler extends AbstractSqlHandler {
     return calciteOptimizedPlan;
   }
 
-
   public static class MetaDataProviderModifier extends RelShuttleImpl {
     private final RelMetadataProvider metadataProvider;
 

http://git-wip-us.apache.org/repos/asf/drill/blob/e0c9b841/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SystemOptionManager.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SystemOptionManager.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SystemOptionManager.java
index 5e16483..72fcccb 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SystemOptionManager.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SystemOptionManager.java
@@ -79,8 +79,9 @@ public class SystemOptionManager extends BaseOptionManager {
       PlannerSettings.PARTITION_SENDER_MAX_THREADS,
       PlannerSettings.PARTITION_SENDER_SET_THREADS,
       PlannerSettings.ENABLE_DECIMAL_DATA_TYPE,
-      PlannerSettings.HEP_JOIN_OPT,
+      PlannerSettings.HEP_OPT,
       PlannerSettings.PLANNER_MEMORY_LIMIT,
+      PlannerSettings.HEP_PARTITION_PRUNING,
       ExecConstants.CAST_TO_NULLABLE_NUMERIC_OPTION,
       ExecConstants.OUTPUT_FORMAT_VALIDATOR,
       ExecConstants.PARQUET_BLOCK_SIZE_VALIDATOR,


Mime
View raw message