drill-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From j..@apache.org
Subject [2/2] drill git commit: Address review comments. Modify costing for Cartesian Logical Join, to fix CanNotPlan in some TPCDS queries with view on text format.
Date Sun, 10 May 2015 02:41:03 GMT
Address review comments. Modify costing for Cartesian Logical Join, to fix CanNotPlan in some
TPCDS queries with view on text format.

Code clean up.


Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/a2963836
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/a2963836
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/a2963836

Branch: refs/heads/master
Commit: a296383632946a1f45a9b66d4638dab00a026d30
Parents: cd05500
Author: Jinfeng Ni <jni@apache.org>
Authored: Sat May 9 15:29:00 2015 -0700
Committer: Jinfeng Ni <jni@apache.org>
Committed: Sat May 9 16:42:43 2015 -0700

----------------------------------------------------------------------
 .../exec/planner/common/DrillJoinRelBase.java   | 63 ++++++++++----------
 .../exec/planner/logical/DrillRuleSets.java     | 13 ++--
 2 files changed, 35 insertions(+), 41 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/drill/blob/a2963836/exec/java-exec/src/main/java/org/apache/drill/exec/planner/common/DrillJoinRelBase.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/common/DrillJoinRelBase.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/common/DrillJoinRelBase.java
index 63fd497..7d3ad0a 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/common/DrillJoinRelBase.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/common/DrillJoinRelBase.java
@@ -28,6 +28,7 @@ import org.apache.drill.exec.planner.cost.DrillCostBase;
 import org.apache.drill.exec.physical.impl.join.JoinUtils;
 import org.apache.drill.exec.physical.impl.join.JoinUtils.JoinCategory;
 import org.apache.drill.exec.planner.cost.DrillCostBase.DrillCostFactory;
+import org.apache.drill.exec.planner.cost.DrillRelOptCost;
 import org.apache.drill.exec.planner.physical.PrelUtil;
 import org.apache.calcite.rel.InvalidRelException;
 import org.apache.calcite.rel.core.Join;
@@ -122,38 +123,26 @@ public abstract class DrillJoinRelBase extends Join implements DrillRelNode
{
   }
 
   protected  RelOptCost computeCartesianJoinCost(RelOptPlanner planner) {
-    double probeRowCount = RelMetadataQuery.getRowCount(this.getLeft());
-    double buildRowCount = RelMetadataQuery.getRowCount(this.getRight());
-
-    // cpu cost of hashing the join keys for the build side
-    double cpuCostBuild = DrillCostBase.HASH_CPU_COST * getRightKeys().size() * buildRowCount;
-    // cpu cost of hashing the join keys for the probe side
-    double cpuCostProbe = DrillCostBase.HASH_CPU_COST * getLeftKeys().size() * probeRowCount;
-
-    // cpu cost of evaluating each leftkey=rightkey join condition
-    double joinConditionCost = DrillCostBase.COMPARE_CPU_COST * this.getLeftKeys().size();
+    final double probeRowCount = RelMetadataQuery.getRowCount(this.getLeft());
+    final double buildRowCount = RelMetadataQuery.getRowCount(this.getRight());
 
-    double factor = PrelUtil.getPlannerSettings(planner).getOptions()
-        .getOption(ExecConstants.HASH_JOIN_TABLE_FACTOR_KEY).float_val;
-    long fieldWidth = PrelUtil.getPlannerSettings(planner).getOptions()
-        .getOption(ExecConstants.AVERAGE_FIELD_WIDTH_KEY).num_val;
-
-    // table + hashValues + links
-    double memCost =
-        (
-            (fieldWidth * this.getRightKeys().size()) +
-                IntHolder.WIDTH +
-                IntHolder.WIDTH
-        ) * buildRowCount * factor;
+    final DrillCostFactory costFactory = (DrillCostFactory) planner.getCostFactory();
 
-    double cpuCost = joinConditionCost * (probeRowCount * buildRowCount) // probe size determine
the join condition comparison cost
-        + cpuCostBuild + cpuCostProbe ;
+    final double mulFactor = 10000; // This is a magic number,
+                                    // just to make sure Cartesian Join is more expensive
+                                    // than Non-Cartesian Join.
 
-    DrillCostFactory costFactory = (DrillCostFactory) planner.getCostFactory();
+    final int keySize = 1 ;  // assume having 1 join key, when estimate join cost.
+    final DrillCostBase cost = (DrillCostBase) computeHashJoinCostWithKeySize(planner, keySize).multiplyBy(mulFactor);
 
-    final double mulFactor = 100000; // This is a magic number, just to make sure CartesianJoin
is more expensive than Non-CartesianJoin.
+    // Cartesian join row count will be product of two inputs. The other factors come from
the above estimated DrillCost.
+    return costFactory.makeCost(
+        buildRowCount * probeRowCount,
+        cost.getCpu(),
+        cost.getIo(),
+        cost.getNetwork(),
+        cost.getMemory() );
 
-    return costFactory.makeCost(buildRowCount * probeRowCount, cpuCost * mulFactor, 0, 0,
memCost * mulFactor);
   }
 
   protected RelOptCost computeLogicalJoinCost(RelOptPlanner planner) {
@@ -168,16 +157,26 @@ public abstract class DrillJoinRelBase extends Join implements DrillRelNode
{
   }
 
   protected RelOptCost computeHashJoinCost(RelOptPlanner planner) {
+      return computeHashJoinCostWithKeySize(planner, this.getLeftKeys().size());
+  }
+
+  /**
+   *
+   * @param planner  : Optimization Planner.
+   * @param keySize  : the # of join keys in join condition. Left key size should be equal
to right key size.
+   * @return         : RelOptCost
+   */
+  private RelOptCost computeHashJoinCostWithKeySize(RelOptPlanner planner, int keySize) {
     double probeRowCount = RelMetadataQuery.getRowCount(this.getLeft());
     double buildRowCount = RelMetadataQuery.getRowCount(this.getRight());
 
     // cpu cost of hashing the join keys for the build side
-    double cpuCostBuild = DrillCostBase.HASH_CPU_COST * getRightKeys().size() * buildRowCount;
+    double cpuCostBuild = DrillCostBase.HASH_CPU_COST * keySize * buildRowCount;
     // cpu cost of hashing the join keys for the probe side
-    double cpuCostProbe = DrillCostBase.HASH_CPU_COST * getLeftKeys().size() * probeRowCount;
+    double cpuCostProbe = DrillCostBase.HASH_CPU_COST * keySize * probeRowCount;
 
     // cpu cost of evaluating each leftkey=rightkey join condition
-    double joinConditionCost = DrillCostBase.COMPARE_CPU_COST * this.getLeftKeys().size();
+    double joinConditionCost = DrillCostBase.COMPARE_CPU_COST * keySize;
 
     double factor = PrelUtil.getPlannerSettings(planner).getOptions()
         .getOption(ExecConstants.HASH_JOIN_TABLE_FACTOR_KEY).float_val;
@@ -187,7 +186,7 @@ public abstract class DrillJoinRelBase extends Join implements DrillRelNode
{
     // table + hashValues + links
     double memCost =
         (
-            (fieldWidth * this.getRightKeys().size()) +
+            (fieldWidth * keySize) +
                 IntHolder.WIDTH +
                 IntHolder.WIDTH
         ) * buildRowCount * factor;
@@ -198,8 +197,8 @@ public abstract class DrillJoinRelBase extends Join implements DrillRelNode
{
     DrillCostFactory costFactory = (DrillCostFactory) planner.getCostFactory();
 
     return costFactory.makeCost(buildRowCount + probeRowCount, cpuCost, 0, 0, memCost);
-
   }
+
   private boolean hasScalarSubqueryInput() {
     if (JoinUtils.isScalarSubquery(this.getLeft())
         || JoinUtils.isScalarSubquery(this.getRight())) {

http://git-wip-us.apache.org/repos/asf/drill/blob/a2963836/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillRuleSets.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillRuleSets.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillRuleSets.java
index 212cffa..f7cfbf4 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillRuleSets.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/logical/DrillRuleSets.java
@@ -124,20 +124,15 @@ public class DrillRuleSets {
       DrillPushFilterPastProjectRule.INSTANCE,
       DrillFilterJoinRules.DRILL_FILTER_ON_JOIN,
       DrillFilterJoinRules.DRILL_JOIN,
-//      JoinPushThroughJoinRule.RIGHT,
-//      JoinPushThroughJoinRule.LEFT,
       // End support for WHERE style joins.
 
       FilterMergeRule.INSTANCE,
-//      SwapJoinRule.INSTANCE,
-      AggregateRemoveRule.INSTANCE,   // RemoveDistinctRule
-//      UnionToDistinctRule.INSTANCE,
-      ProjectRemoveRule.NAME_CALC_INSTANCE,     // RemoveTrivialProjectRule
-//      RemoveTrivialCalcRule.INSTANCE,
-      SortRemoveRule.INSTANCE,      //RemoveSortRule.INSTANCE,
+      AggregateRemoveRule.INSTANCE,
+      ProjectRemoveRule.NAME_CALC_INSTANCE,
+      SortRemoveRule.INSTANCE,
 
       DrillMergeProjectRule.getInstance(true, RelFactories.DEFAULT_PROJECT_FACTORY, context.getFunctionRegistry()),
-      AggregateExpandDistinctAggregatesRule.INSTANCE, //RemoveDistinctAggregateRule.INSTANCE,
//
+      AggregateExpandDistinctAggregatesRule.INSTANCE,
       DrillReduceAggregatesRule.INSTANCE,
 
       /*


Mime
View raw message