drill-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From j..@apache.org
Subject [2/3] drill git commit: DRILL-2400: Part 1: Change cpu cost estimation formula for DrillFilterRelBase. Add testcase for MergeFilter rule.
Date Sat, 07 Mar 2015 06:46:46 GMT
DRILL-2400: Part 1: Change cpu cost estimation formula for DrillFilterRelBase. Add testcase
for MergeFilter rule.

Modify costing of Filter.

Cost change to Filter.

Change to filter costing.

Move one test utility method to PlanTestBase.


Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/6e6f789a
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/6e6f789a
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/6e6f789a

Branch: refs/heads/master
Commit: 6e6f789a46c4237a53bc963eeab30eb6e34291c2
Parents: ed39786
Author: Jinfeng Ni <jni@maprtech.com>
Authored: Thu Feb 19 18:51:01 2015 -0800
Committer: Jinfeng Ni <jni@maprtech.com>
Committed: Fri Mar 6 22:19:21 2015 -0800

----------------------------------------------------------------------
 .../exec/planner/common/DrillFilterRelBase.java | 30 +++++++-
 .../java/org/apache/drill/PlanTestBase.java     | 36 ++++++++++
 .../org/apache/drill/TestMergeFilterPlan.java   | 73 ++++++++++++++++++++
 3 files changed, 136 insertions(+), 3 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/drill/blob/6e6f789a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/common/DrillFilterRelBase.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/common/DrillFilterRelBase.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/common/DrillFilterRelBase.java
index f9b1e42..7813d15 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/common/DrillFilterRelBase.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/common/DrillFilterRelBase.java
@@ -33,12 +33,16 @@ import org.eigenbase.relopt.RelOptPlanner;
 import org.eigenbase.relopt.RelOptUtil;
 import org.eigenbase.relopt.RelTraitSet;
 import org.eigenbase.rex.RexNode;
+import org.eigenbase.rex.RexUtil;
+
+import java.util.List;
 
 /**
  * Base class for logical and physical Filters implemented in Drill
  */
 public abstract class DrillFilterRelBase extends FilterRelBase implements DrillRelNode {
-  int numConjuncts = 0;
+  private final int numConjuncts;
+  private final List<RexNode> conjunctions;
 
   protected DrillFilterRelBase(Convention convention, RelOptCluster cluster, RelTraitSet
traits, RelNode child, RexNode condition) {
     super(cluster, traits, child, condition);
@@ -46,8 +50,10 @@ public abstract class DrillFilterRelBase extends FilterRelBase implements
DrillR
 
     // save the number of conjuncts that make up the filter condition such
     // that repeated calls to the costing function can use the saved copy
-    numConjuncts = RelOptUtil.conjunctions(condition).size();
+    conjunctions = RelOptUtil.conjunctions(condition);
+    numConjuncts = conjunctions.size();
     assert numConjuncts >= 1;
+
   }
 
   @Override
@@ -57,7 +63,7 @@ public abstract class DrillFilterRelBase extends FilterRelBase implements
DrillR
     }
     RelNode child = this.getChild();
     double inputRows = RelMetadataQuery.getRowCount(child);
-    double cpuCost = DrillCostBase.COMPARE_CPU_COST * numConjuncts * inputRows;
+    double cpuCost = estimateCpuCost();
     DrillCostFactory costFactory = (DrillCostFactory)planner.getCostFactory();
     return costFactory.makeCost(inputRows, cpuCost, 0, 0);
   }
@@ -65,4 +71,22 @@ public abstract class DrillFilterRelBase extends FilterRelBase implements
DrillR
   protected LogicalExpression getFilterExpression(DrillParseContext context){
     return DrillOptiq.toDrill(context, getChild(), getCondition());
   }
+
+  /* Given the condition (C1 and C2 and C3 and ... C_n), here is how to estimate cpu cost
of FILTER :
+  *  Let's say child's rowcount is n. We assume short circuit evaluation will be applied
to the boolean expression evaluation.
+  *  #_of_comparison = n + n * Selectivity(C1) + n * Selectivity(C1 and C2) + ... + n * Selecitivity(C1
and C2 ... and C_n)
+  *  cpu_cost = #_of_comparison * DrillCostBase_COMPARE_CPU_COST;
+  */
+  private double estimateCpuCost() {
+    RelNode child = this.getChild();
+    double compNum = RelMetadataQuery.getRowCount(child);
+
+    for (int i = 0; i< numConjuncts; i++) {
+      RexNode conjFilter = RexUtil.composeConjunction(this.getCluster().getRexBuilder(),
conjunctions.subList(0, i + 1), false);
+      compNum += FilterRelBase.estimateFilteredRows(child, conjFilter);
+    }
+
+    return compNum * DrillCostBase.COMPARE_CPU_COST;
+  }
+
 }

http://git-wip-us.apache.org/repos/asf/drill/blob/6e6f789a/exec/java-exec/src/test/java/org/apache/drill/PlanTestBase.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/PlanTestBase.java b/exec/java-exec/src/test/java/org/apache/drill/PlanTestBase.java
index 4ac8b9b..c95d2b2 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/PlanTestBase.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/PlanTestBase.java
@@ -103,6 +103,42 @@ public class PlanTestBase extends BaseTestQuery {
     }
   }
 
+  /**
+   * Runs an explain plan query and check for expected substring patterns (in optiq
+   * text format), also ensure excluded patterns are not found. Either list can
+   * be empty or null to skip that part of the check.
+   *
+   * This is different from testPlanMatchingPatterns in that this one use substring contains,
+   * in stead of regex pattern matching. This one is useful when the pattern contains
+   * many regex reserved chars, and you do not want to put the escape char.
+   *
+   * See the convenience methods for passing a single string in either the
+   * excluded list, included list or both.
+   *
+   * @param query - an explain query, this method does not add it for you
+   * @param expectedPatterns - list of patterns that should appear in the plan
+   * @param excludedPatterns - list of patterns that should not appear in the plan
+   * @throws Exception - if an inclusion or exclusion check fails, or the
+   *                     planning process throws an exception
+   */
+  public void testPlanSubstrPatterns(String query, String[] expectedPatterns, String[] excludedPatterns)
throws Exception {
+    String plan = getPlanInString("EXPLAIN PLAN for " + normalizeQuery(query), OPTIQ_FORMAT);
+
+    // Check and make sure all expected patterns are in the plan
+    if (expectedPatterns != null) {
+      for (String s : expectedPatterns) {
+        assert plan.contains(s) : EXPECTED_NOT_FOUND + s;
+      }
+    }
+
+    // Check and make sure all excluded patterns are not in the plan
+    if (excludedPatterns != null) {
+      for (String s : excludedPatterns) {
+        assert ! plan.contains(s) : UNEXPECTED_FOUND + s;
+      }
+    }
+  }
+
   public void testPlanOneExpectedPatternOneExcluded(String query, String expectedPattern,
String excludedPattern) throws Exception {
     testPlanMatchingPatterns(query, new String[]{expectedPattern}, new String[]{excludedPattern});
   }

http://git-wip-us.apache.org/repos/asf/drill/blob/6e6f789a/exec/java-exec/src/test/java/org/apache/drill/TestMergeFilterPlan.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/TestMergeFilterPlan.java b/exec/java-exec/src/test/java/org/apache/drill/TestMergeFilterPlan.java
new file mode 100644
index 0000000..65559b1
--- /dev/null
+++ b/exec/java-exec/src/test/java/org/apache/drill/TestMergeFilterPlan.java
@@ -0,0 +1,73 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.drill;
+
+import org.junit.Test;
+
+public class TestMergeFilterPlan extends PlanTestBase {
+
+  @Test
+  public void testDRILL_FilterMerge() throws Exception {
+    String viewDDL = "create or replace view MyViewWithFilter as " +
+        " SELECT  first_name, " +
+        "         last_name, " +
+        "         full_name, " +
+        "         salary, " +
+        "         employee_id, " +
+        "         store_id, " +
+        "         position_id, " +
+        "         position_title, " +
+        "         education_level " +
+        " FROM cp.`employee.json` " +
+        " WHERE position_id in (1, 2, 3 ) " ;
+
+    String querySQL =  " select dat.store_id\n" +
+        "      , sum(dat.store_cost) as total_cost\n" +
+        " from (\n" +
+        "  select store_id, position_id\n" +
+        "            , sum( salary) as store_cost\n" +
+        "       from MyViewWithFilter \n" +
+        " where full_name in ( select n_name\n" +
+        "                      from cp.`tpch/nation.parquet`)\n" +
+        "  and  education_level = 'GRADUATE DEGREE' " +
+        "  and position_id in ( select position_id \n" +
+        "                       from MyViewWithFilter\n" +
+        "                        where position_title like '%VP%'\n" +
+        "                      )\n" +
+        "  group by store_id, position_id\n" +
+        ") dat\n" +
+        "group by dat.store_id\n" +
+        "order by dat.store_id";
+
+    String expectedPattern1 = "Filter(condition=[AND(OR(=($0, 1), =($0, 2), =($0, 3)), =(CAST($4):ANY
NOT NULL, 'GRADUATE DEGREE'))])";
+    String expectedPattern2 = "Filter(condition=[AND(OR(=($0, 1), =($0, 2), =($0, 3)), LIKE(CAST($1):ANY
NOT NULL, '%VP%'))])";
+    String excludedPattern = "Filter(condition=[OR(=($0, 1), =($0, 2), =($0, 3))])";
+
+    test("use dfs.tmp");
+
+    test(viewDDL);
+
+    testPlanSubstrPatterns(querySQL,
+        new String[]{expectedPattern1, expectedPattern2}, new String[]{excludedPattern});
+
+    test("drop view MyViewWithFilter ");
+  }
+
+
+}


Mime
View raw message