calcite-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jh...@apache.org
Subject incubator-calcite git commit: [CALCITE-890] Register all combinations of materialization substitutions (Maryann Xue)
Date Fri, 23 Oct 2015 23:27:26 GMT
Repository: incubator-calcite
Updated Branches:
  refs/heads/master 2c339be18 -> 9d0fef317


 [CALCITE-890] Register all combinations of materialization substitutions (Maryann Xue)


Project: http://git-wip-us.apache.org/repos/asf/incubator-calcite/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-calcite/commit/9d0fef31
Tree: http://git-wip-us.apache.org/repos/asf/incubator-calcite/tree/9d0fef31
Diff: http://git-wip-us.apache.org/repos/asf/incubator-calcite/diff/9d0fef31

Branch: refs/heads/master
Commit: 9d0fef31725e73912b37fff79037a3258bca7fa6
Parents: 2c339be
Author: Julian Hyde <jhyde@apache.org>
Authored: Fri Oct 23 13:08:42 2015 -0700
Committer: Julian Hyde <jhyde@apache.org>
Committed: Fri Oct 23 14:36:42 2015 -0700

----------------------------------------------------------------------
 core/pom.xml                                    |  5 ++
 .../calcite/plan/volcano/VolcanoPlanner.java    | 61 +++++++++++++++-----
 .../calcite/test/MaterializationTest.java       | 56 ++++++++++++++++++
 3 files changed, 107 insertions(+), 15 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-calcite/blob/9d0fef31/core/pom.xml
----------------------------------------------------------------------
diff --git a/core/pom.xml b/core/pom.xml
index 65adb3e..6a62d68 100644
--- a/core/pom.xml
+++ b/core/pom.xml
@@ -56,6 +56,11 @@ limitations under the License.
       <artifactId>commons-dbcp</artifactId>
     </dependency>
     <dependency>
+      <groupId>org.apache.commons</groupId>
+      <artifactId>commons-lang3</artifactId>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
       <groupId>com.google.code.findbugs</groupId>
       <artifactId>jsr305</artifactId>
     </dependency>

http://git-wip-us.apache.org/repos/asf/incubator-calcite/blob/9d0fef31/core/src/main/java/org/apache/calcite/plan/volcano/VolcanoPlanner.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/calcite/plan/volcano/VolcanoPlanner.java b/core/src/main/java/org/apache/calcite/plan/volcano/VolcanoPlanner.java
index e57a760..4cc513a 100644
--- a/core/src/main/java/org/apache/calcite/plan/volcano/VolcanoPlanner.java
+++ b/core/src/main/java/org/apache/calcite/plan/volcano/VolcanoPlanner.java
@@ -360,7 +360,8 @@ public class VolcanoPlanner extends AbstractRelOptPlanner {
     registerImpl(rel, root.set);
   }
 
-  private void useMaterialization(RelOptMaterialization materialization) {
+  private RelNode useMaterialization(RelNode root,
+      RelOptMaterialization materialization, boolean firstRun) {
     // Try to rewrite the original root query in terms of the materialized
     // query. If that is possible, register the remnant query as equivalent
     // to the root.
@@ -368,22 +369,23 @@ public class VolcanoPlanner extends AbstractRelOptPlanner {
 
     // This call modifies originalRoot. Doesn't look like originalRoot should be mutable
though.
     // Need to check.
-    RelNode sub = substitute(originalRoot, materialization);
+    RelNode sub = substitute(root, materialization);
     if (sub != null) {
-      // TODO: try to substitute other materializations in the remnant.
-      // Useful for big queries, e.g.
-      //   (t1 group by c1) join (t2 group by c2).
       Hook.SUB.run(sub);
-      registerImpl(sub, root.set);
-      return;
+      registerImpl(sub, this.root.set);
+      return sub;
+    }
+
+    if (firstRun) {
+      RelSubset subset = registerImpl(materialization.queryRel, null);
+      RelNode tableRel2 =
+          RelOptUtil.createCastRel(
+              materialization.tableRel,
+              materialization.queryRel.getRowType(),
+              true);
+      registerImpl(tableRel2, subset.set);
     }
-    RelSubset subset = registerImpl(materialization.queryRel, null);
-    RelNode tableRel2 =
-        RelOptUtil.createCastRel(
-            materialization.tableRel,
-            materialization.queryRel.getRowType(),
-            true);
-    registerImpl(tableRel2, subset.set);
+    return null;
   }
 
   private RelNode substitute(
@@ -417,6 +419,33 @@ public class VolcanoPlanner extends AbstractRelOptPlanner {
             .go(materialization.tableRel);
   }
 
+  // Register all possible combinations of materialization substitution.
+  // Useful for big queries, e.g.
+  //   (t1 group by c1) join (t2 group by c2).
+  private void useMaterializations(RelNode root,
+      List<RelOptMaterialization> materializations, boolean firstRun) {
+    for (RelOptMaterialization m : materializations) {
+      RelNode sub = useMaterialization(root, m, firstRun);
+      if (sub != null) {
+        useMaterializations(sub, materializations, false);
+      } else {
+        // Based on the assumption that a substitution itself won't trigger another
+        // substitution, if a materialization is not matched here it won't be useful
+        // in any subsequent matching for the current level of recursion or this level
+        // down. So we can safely remove the unmatched materialization from the remnant
+        // list for the current root.
+        List<RelOptMaterialization> newList =
+            Lists.newArrayListWithExpectedSize(materializations.size() - 1);
+        for (RelOptMaterialization elem : materializations) {
+          if (elem != m) {
+            newList.add(elem);
+          }
+        }
+        materializations = newList;
+      }
+    }
+  }
+
   private void useApplicableMaterializations() {
     // Avoid using materializations while populating materializations!
     final CalciteConnectionConfig config =
@@ -454,6 +483,7 @@ public class VolcanoPlanner extends AbstractRelOptPlanner {
     final Graphs.FrozenGraph<List<String>, DefaultEdge> frozenGraph =
         Graphs.makeImmutable(usesGraph);
     final Set<RelOptTable> queryTables = findTables(originalRoot);
+    final List<RelOptMaterialization> applicableMaterializations = Lists.newArrayList();
     for (RelOptMaterialization materialization : materializations) {
       if (materialization.starTable != null) {
         // Materialization is a tile in a lattice. We will deal with it shortly.
@@ -461,10 +491,11 @@ public class VolcanoPlanner extends AbstractRelOptPlanner {
       }
       if (materialization.table != null) {
         if (usesTable(materialization.table, queryTables, frozenGraph)) {
-          useMaterialization(materialization);
+          applicableMaterializations.add(materialization);
         }
       }
     }
+    useMaterializations(originalRoot, applicableMaterializations, true);
 
     // Use a lattice if the query uses at least the central (fact) table of the
     // lattice.

http://git-wip-us.apache.org/repos/asf/incubator-calcite/blob/9d0fef31/core/src/test/java/org/apache/calcite/test/MaterializationTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/calcite/test/MaterializationTest.java b/core/src/test/java/org/apache/calcite/test/MaterializationTest.java
index 61d49dd..07d5a44 100644
--- a/core/src/test/java/org/apache/calcite/test/MaterializationTest.java
+++ b/core/src/test/java/org/apache/calcite/test/MaterializationTest.java
@@ -29,6 +29,9 @@ import org.apache.calcite.rex.RexNode;
 import org.apache.calcite.rex.RexUtil;
 import org.apache.calcite.sql.fun.SqlStdOperatorTable;
 import org.apache.calcite.util.JsonBuilder;
+import org.apache.calcite.util.Util;
+
+import org.apache.commons.lang3.StringUtils;
 
 import com.google.common.base.Function;
 import com.google.common.collect.ImmutableList;
@@ -38,6 +41,7 @@ import org.junit.Test;
 
 import java.math.BigDecimal;
 import java.sql.ResultSet;
+import java.sql.SQLException;
 import java.util.List;
 import java.util.Map;
 
@@ -815,6 +819,58 @@ public class MaterializationTest {
       Prepare.THREAD_TRIM.set(false);
     }
   }
+
+  @Test public void testSingleMaterializationMultiUsage() {
+    String q = "select *\n"
+        + "from (select * from \"emps\" where \"empid\" < 300)\n"
+        + "join (select * from \"emps\" where \"empid\" < 200) using (\"empid\")";
+    try {
+      Prepare.THREAD_TRIM.set(true);
+      MaterializationService.setThreadLocal();
+      CalciteAssert.that()
+          .withMaterializations(JdbcTest.HR_MODEL,
+              "m0", "select * from \"emps\" where \"empid\" < 500")
+          .query(q)
+          .enableMaterializations(true)
+          .explainMatches("", new Function<ResultSet, Void>() {
+            public Void apply(ResultSet s) {
+              try {
+                final String actual = Util.toLinux(CalciteAssert.toString(s));
+                final String scan = "EnumerableTableScan(table=[[hr, m0]])";
+                assertTrue(actual + " should have had two occurrences of " + scan,
+                    StringUtils.countMatches(actual, scan) == 2);
+                return null;
+              } catch (SQLException e) {
+                throw new RuntimeException(e);
+              }
+            }
+          })
+          .sameResultWithMaterializationsDisabled();
+    } finally {
+      Prepare.THREAD_TRIM.set(false);
+    }
+  }
+
+  @Test public void testMultiMaterializationMultiUsage() {
+    String q = "select *\n"
+        + "from (select * from \"emps\" where \"empid\" < 300)\n"
+        + "join (select \"deptno\", count(*) as c from \"emps\" group by \"deptno\") using
(\"deptno\")";
+    try {
+      Prepare.THREAD_TRIM.set(true);
+      MaterializationService.setThreadLocal();
+      CalciteAssert.that()
+          .withMaterializations(JdbcTest.HR_MODEL,
+              "m0", "select \"deptno\", count(*) as c, sum(\"empid\") as s from \"emps\"
group by \"deptno\"",
+              "m1", "select * from \"emps\" where \"empid\" < 500")
+          .query(q)
+          .enableMaterializations(true)
+          .explainContains("EnumerableTableScan(table=[[hr, m0]])")
+          .explainContains("EnumerableTableScan(table=[[hr, m1]])")
+          .sameResultWithMaterializationsDisabled();
+    } finally {
+      Prepare.THREAD_TRIM.set(false);
+    }
+  }
 }
 
 // End MaterializationTest.java


Mime
View raw message