calcite-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jh...@apache.org
Subject [2/2] incubator-calcite git commit: [CALCITE-761] Pre-populated materializations (Maryann Xue)
Date Wed, 22 Jul 2015 00:23:17 GMT
[CALCITE-761] Pre-populated materializations (Maryann Xue)


Project: http://git-wip-us.apache.org/repos/asf/incubator-calcite/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-calcite/commit/4b60b9bf
Tree: http://git-wip-us.apache.org/repos/asf/incubator-calcite/tree/4b60b9bf
Diff: http://git-wip-us.apache.org/repos/asf/incubator-calcite/diff/4b60b9bf

Branch: refs/heads/master
Commit: 4b60b9bf46b88b4dc6af8279013cbb731d98a8d3
Parents: 629a56e
Author: maryannxue <wei.xue@intel.com>
Authored: Fri Jun 26 15:32:02 2015 -0700
Committer: Julian Hyde <jhyde@apache.org>
Committed: Tue Jul 21 16:51:15 2015 -0700

----------------------------------------------------------------------
 .../materialize/MaterializationService.java     | 19 +++++--
 .../org/apache/calcite/model/ModelHandler.java  | 17 +++++-
 .../calcite/plan/RelOptMaterialization.java     |  3 +-
 .../schema/impl/MaterializedViewTable.java      | 13 +++--
 .../org/apache/calcite/test/CalciteAssert.java  | 58 ++++++++++----------
 .../calcite/test/MaterializationTest.java       | 39 +++++++++++++
 site/_docs/model.md                             |  7 ++-
 7 files changed, 111 insertions(+), 45 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-calcite/blob/4b60b9bf/core/src/main/java/org/apache/calcite/materialize/MaterializationService.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/calcite/materialize/MaterializationService.java
b/core/src/main/java/org/apache/calcite/materialize/MaterializationService.java
index b594cca..615d185 100644
--- a/core/src/main/java/org/apache/calcite/materialize/MaterializationService.java
+++ b/core/src/main/java/org/apache/calcite/materialize/MaterializationService.java
@@ -95,15 +95,16 @@ public class MaterializationService {
   /** Defines a new materialization. Returns its key. */
   public MaterializationKey defineMaterialization(final CalciteSchema schema,
       TileKey tileKey, String viewSql, List<String> viewSchemaPath,
-      final String suggestedTableName, boolean create) {
+      final String suggestedTableName, boolean create, boolean existing) {
     return defineMaterialization(schema, tileKey, viewSql, viewSchemaPath,
-        suggestedTableName, tableFactory, create);
+        suggestedTableName, tableFactory, create, existing);
   }
 
   /** Defines a new materialization. Returns its key. */
   public MaterializationKey defineMaterialization(final CalciteSchema schema,
       TileKey tileKey, String viewSql, List<String> viewSchemaPath,
-      String suggestedTableName, TableFactory tableFactory, boolean create) {
+      String suggestedTableName, TableFactory tableFactory, boolean create,
+      boolean existing) {
     final MaterializationActor.QueryKey queryKey =
         new MaterializationActor.QueryKey(viewSql, schema, viewSchemaPath);
     final MaterializationKey existingKey = actor.keyBySql.get(queryKey);
@@ -116,7 +117,15 @@ public class MaterializationService {
 
     final CalciteConnection connection =
         CalciteMetaImpl.connect(schema.root(), null);
-    CalciteSchema.TableEntry tableEntry = schema.getTableBySql(viewSql);
+    CalciteSchema.TableEntry tableEntry;
+    if (existing) {
+      tableEntry = schema.getTable(suggestedTableName, true);
+    } else {
+      tableEntry = null;
+    }
+    if (tableEntry == null) {
+      tableEntry = schema.getTableBySql(viewSql);
+    }
     RelDataType rowType = null;
     if (tableEntry == null) {
       Table table = tableFactory.createTable(schema, viewSql, viewSchemaPath);
@@ -265,7 +274,7 @@ public class MaterializationService {
     final String sql = lattice.sql(groupSet, newTileKey.measures);
     materializationKey =
         defineMaterialization(schema, newTileKey, sql, schema.path(null),
-            suggestedTableName, tableFactory, true);
+            suggestedTableName, tableFactory, true, false);
     if (materializationKey != null) {
       final CalciteSchema.TableEntry tableEntry =
           checkValid(materializationKey);

http://git-wip-us.apache.org/repos/asf/incubator-calcite/blob/4b60b9bf/core/src/main/java/org/apache/calcite/model/ModelHandler.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/calcite/model/ModelHandler.java b/core/src/main/java/org/apache/calcite/model/ModelHandler.java
index 1afaf41..3faf446 100644
--- a/core/src/main/java/org/apache/calcite/model/ModelHandler.java
+++ b/core/src/main/java/org/apache/calcite/model/ModelHandler.java
@@ -265,9 +265,22 @@ public class ModelHandler {
                 + "' is not a SemiMutableSchema");
       }
       CalciteSchema calciteSchema = CalciteSchema.from(schema);
-      schema.add(jsonMaterialization.view,
+
+      final String viewName;
+      final boolean existing;
+      if (jsonMaterialization.view == null) {
+        // If the user did not supply a view name, that means the materialized
+        // view is pre-populated. Generate a synthetic view name.
+        viewName = "$" + schema.getTableNames().size();
+        existing = true;
+      } else {
+        viewName = jsonMaterialization.view;
+        existing = false;
+      }
+      schema.add(viewName,
           MaterializedViewTable.create(calciteSchema,
-              jsonMaterialization.getSql(), null, jsonMaterialization.table));
+              jsonMaterialization.getSql(), null, jsonMaterialization.table,
+              existing));
     } catch (Exception e) {
       throw new RuntimeException("Error instantiating " + jsonMaterialization,
           e);

http://git-wip-us.apache.org/repos/asf/incubator-calcite/blob/4b60b9bf/core/src/main/java/org/apache/calcite/plan/RelOptMaterialization.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/calcite/plan/RelOptMaterialization.java b/core/src/main/java/org/apache/calcite/plan/RelOptMaterialization.java
index f945e65..4108cff 100644
--- a/core/src/main/java/org/apache/calcite/plan/RelOptMaterialization.java
+++ b/core/src/main/java/org/apache/calcite/plan/RelOptMaterialization.java
@@ -62,7 +62,8 @@ public class RelOptMaterialization {
    */
   public RelOptMaterialization(RelNode tableRel, RelNode queryRel,
       RelOptTable starRelOptTable) {
-    this.tableRel = tableRel;
+    this.tableRel =
+        RelOptUtil.createCastRel(tableRel, queryRel.getRowType(), false);
     this.starRelOptTable = starRelOptTable;
     if (starRelOptTable == null) {
       this.starTable = null;

http://git-wip-us.apache.org/repos/asf/incubator-calcite/blob/4b60b9bf/core/src/main/java/org/apache/calcite/schema/impl/MaterializedViewTable.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/calcite/schema/impl/MaterializedViewTable.java
b/core/src/main/java/org/apache/calcite/schema/impl/MaterializedViewTable.java
index 27e873a..8d80465 100644
--- a/core/src/main/java/org/apache/calcite/schema/impl/MaterializedViewTable.java
+++ b/core/src/main/java/org/apache/calcite/schema/impl/MaterializedViewTable.java
@@ -75,11 +75,10 @@ public class MaterializedViewTable extends ViewTable {
 
   /** Table macro that returns a materialized view. */
   public static MaterializedViewTableMacro create(final CalciteSchema schema,
-      final String viewSql,
-      final List<String> viewSchemaPath,
-      final String tableName) {
+      final String viewSql, final List<String> viewSchemaPath,
+      final String suggestedTableName, boolean existing) {
     return new MaterializedViewTableMacro(schema, viewSql, viewSchemaPath,
-        tableName);
+        suggestedTableName, existing);
   }
 
   @Override public RelNode toRel(RelOptTable.ToRelContext context,
@@ -102,11 +101,13 @@ public class MaterializedViewTable extends ViewTable {
     private final MaterializationKey key;
 
     private MaterializedViewTableMacro(CalciteSchema schema, String viewSql,
-        List<String> viewSchemaPath, String suggestedTableName) {
+        List<String> viewSchemaPath, String suggestedTableName,
+        boolean existing) {
       super(schema, viewSql, viewSchemaPath, Boolean.TRUE);
       this.key = Preconditions.checkNotNull(
           MaterializationService.instance().defineMaterialization(
-              schema, null, viewSql, schemaPath, suggestedTableName, true));
+              schema, null, viewSql, schemaPath, suggestedTableName, true,
+              existing));
     }
 
     @Override public TranslatableTable apply(List<Object> arguments) {

http://git-wip-us.apache.org/repos/asf/incubator-calcite/blob/4b60b9bf/core/src/test/java/org/apache/calcite/test/CalciteAssert.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/calcite/test/CalciteAssert.java b/core/src/test/java/org/apache/calcite/test/CalciteAssert.java
index 39ed982..3d0a0dc 100644
--- a/core/src/test/java/org/apache/calcite/test/CalciteAssert.java
+++ b/core/src/test/java/org/apache/calcite/test/CalciteAssert.java
@@ -405,7 +405,7 @@ public class CalciteAssert {
 
   private static String typeString(ResultSetMetaData metaData)
       throws SQLException {
-    final List<String> list = new ArrayList<String>();
+    final List<String> list = new ArrayList<>();
     for (int i = 0; i < metaData.getColumnCount(); i++) {
       list.add(
           metaData.getColumnName(i + 1)
@@ -452,13 +452,7 @@ public class CalciteAssert {
           exceptionChecker.apply(null);
           return;
         }
-      } catch (Exception e) {
-        if (exceptionChecker != null) {
-          exceptionChecker.apply(e);
-          return;
-        }
-        throw e;
-      } catch (Error e) {
+      } catch (Exception | Error e) {
         if (exceptionChecker != null) {
           exceptionChecker.apply(e);
           return;
@@ -471,13 +465,11 @@ public class CalciteAssert {
       resultSet.close();
       statement.close();
       connection.close();
-    } catch (Error e) {
+    } catch (Error | RuntimeException e) {
       // We ignore extended message for non-runtime exception, however
       // it does not matter much since it is better to have AssertionError
       // at the very top level of the exception stack.
       throw e;
-    } catch (RuntimeException e) {
-      throw e;
     } catch (Throwable e) {
       throw new RuntimeException(message, e);
     } finally {
@@ -840,21 +832,32 @@ public class CalciteAssert {
 
     /** Adds materializations to the schema. */
     public final AssertThat withMaterializations(String model,
-        String... materializations) {
-      assert materializations.length % 2 == 0;
+        final String... materializations) {
+      return withMaterializations(model,
+          new Function<JsonBuilder, List<Object>>() {
+            public List<Object> apply(JsonBuilder builder) {
+              assert materializations.length % 2 == 0;
+              final List<Object> list = builder.list();
+              for (int i = 0; i < materializations.length; i++) {
+                String table = materializations[i++];
+                final Map<String, Object> map = builder.map();
+                map.put("table", table);
+                map.put("view", table + "v");
+                String sql = materializations[i];
+                final String sql2 = sql.replaceAll("`", "\"");
+                map.put("sql", sql2);
+                list.add(map);
+              }
+              return list;
+            }
+          });
+    }
+
+    /** Adds materializations to the schema. */
+    public final AssertThat withMaterializations(String model,
+        Function<JsonBuilder, List<Object>> materializations) {
       final JsonBuilder builder = new JsonBuilder();
-      final List<Object> list = builder.list();
-      for (int i = 0; i < materializations.length; i++) {
-        String table = materializations[i++];
-        final Map<String, Object> map = builder.map();
-        map.put("table", table);
-        map.put("view", table + "v");
-        String sql = materializations[i];
-        final String sql2 = sql
-            .replaceAll("`", "\"");
-        map.put("sql", sql2);
-        list.add(map);
-      }
+      final List<Object> list = materializations.apply(builder);
       final String buf =
           "materializations: " + builder.toJsonString(list);
       final String model2;
@@ -908,13 +911,10 @@ public class CalciteAssert {
      * and executes a callback. */
     public <T> AssertThat doWithConnection(Function<CalciteConnection, T> fn)
         throws Exception {
-      Connection connection = connectionFactory.createConnection();
-      try {
+      try (Connection connection = connectionFactory.createConnection()) {
         T t = fn.apply((CalciteConnection) connection);
         Util.discard(t);
         return AssertThat.this;
-      } finally {
-        connection.close();
       }
     }
 

http://git-wip-us.apache.org/repos/asf/incubator-calcite/blob/4b60b9bf/core/src/test/java/org/apache/calcite/test/MaterializationTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/calcite/test/MaterializationTest.java b/core/src/test/java/org/apache/calcite/test/MaterializationTest.java
index ae235ab..f132915 100644
--- a/core/src/test/java/org/apache/calcite/test/MaterializationTest.java
+++ b/core/src/test/java/org/apache/calcite/test/MaterializationTest.java
@@ -28,14 +28,18 @@ import org.apache.calcite.rex.RexLiteral;
 import org.apache.calcite.rex.RexNode;
 import org.apache.calcite.rex.RexUtil;
 import org.apache.calcite.sql.fun.SqlStdOperatorTable;
+import org.apache.calcite.util.JsonBuilder;
 
 import com.google.common.base.Function;
+import com.google.common.collect.ImmutableList;
 
 import org.junit.Ignore;
 import org.junit.Test;
 
 import java.math.BigDecimal;
 import java.sql.ResultSet;
+import java.util.List;
+import java.util.Map;
 
 import static org.hamcrest.CoreMatchers.equalTo;
 import static org.junit.Assert.assertEquals;
@@ -53,6 +57,10 @@ public class MaterializationTest {
       CalciteAssert.checkResultContains(
           "EnumerableTableScan(table=[[hr, m0]])");
 
+  private static final Function<ResultSet, Void> CONTAINS_LOCATIONS =
+      CalciteAssert.checkResultContains(
+          "EnumerableTableScan(table=[[hr, locations]])");
+
   final JavaTypeFactoryImpl typeFactory =
       new JavaTypeFactoryImpl(RelDataTypeSystem.DEFAULT);
   final RexBuilder rexBuilder = new RexBuilder(typeFactory);
@@ -749,6 +757,37 @@ public class MaterializationTest {
             + "join \"depts\" using (\"deptno\")";
     checkMaterialize("select * from \"emps\" where \"empid\" < 500", q);
   }
+
+  /** Test case for
+   * <a href="https://issues.apache.org/jira/browse/CALCITE-761">[CALCITE-761]
+   * Pre-populated materializations</a>. */
+  @Test public void testPrePopulated() {
+    String q = "select \"deptno\" from \"emps\"";
+    try {
+      Prepare.THREAD_TRIM.set(true);
+      MaterializationService.setThreadLocal();
+      CalciteAssert.that()
+          .withMaterializations(
+              JdbcTest.HR_MODEL,
+              new Function<JsonBuilder, List<Object>>() {
+                public List<Object> apply(JsonBuilder builder) {
+                  final Map<String, Object> map = builder.map();
+                  map.put("table", "locations");
+                  String sql = "select `deptno` as `empid`, '' as `name`\n"
+                       + "from `emps`";
+                  final String sql2 = sql.replaceAll("`", "\"");
+                  map.put("sql", sql2);
+                  return ImmutableList.<Object>of(map);
+                }
+              })
+          .query(q)
+          .enableMaterializations(true)
+          .explainMatches("", CONTAINS_LOCATIONS)
+          .sameResultWithMaterializationsDisabled();
+    } finally {
+      Prepare.THREAD_TRIM.set(false);
+    }
+  }
 }
 
 // End MaterializationTest.java

http://git-wip-us.apache.org/repos/asf/incubator-calcite/blob/4b60b9bf/site/_docs/model.md
----------------------------------------------------------------------
diff --git a/site/_docs/model.md b/site/_docs/model.md
index 0f9a548..cb8cc9c 100644
--- a/site/_docs/model.md
+++ b/site/_docs/model.md
@@ -204,9 +204,12 @@ Occurs within `root.schemas.materializations`.
 }
 {% endhighlight %}
 
-`view` (optional string) TODO
+`view` (optional string) is the name of the view; null means that the table
+already exists and is populated with the correct data.
 
-`table` (optional string) TODO
+`table` (required string) is the name of the table that materializes the data in
+the query. If `view` is not null, the table might not exist, and if it does not,
+Calcite will create and populate an in-memory table.
 
 `sql` (optional string, or list of strings that will be concatenated as a
  multi-line string) is the SQL definition of the materialization.


Mime
View raw message