calcite-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jh...@apache.org
Subject [2/7] incubator-calcite git commit: [CALCITE-429] Cardinality provider for use by lattice algorithm
Date Sat, 13 Jun 2015 00:38:06 GMT
[CALCITE-429] Cardinality provider for use by lattice algorithm


Project: http://git-wip-us.apache.org/repos/asf/incubator-calcite/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-calcite/commit/4cc539fc
Tree: http://git-wip-us.apache.org/repos/asf/incubator-calcite/tree/4cc539fc
Diff: http://git-wip-us.apache.org/repos/asf/incubator-calcite/diff/4cc539fc

Branch: refs/heads/master
Commit: 4cc539fc28fac0c5897ada1cc5e57729807d260d
Parents: 6ece12a
Author: Julian Hyde <jhyde@apache.org>
Authored: Mon Jun 8 14:07:06 2015 -0700
Committer: Julian Hyde <jhyde@apache.org>
Committed: Thu Jun 11 16:12:26 2015 -0700

----------------------------------------------------------------------
 .../calcite/avatica/test/AvaticaSuite.java      |  1 +
 avatica/pom.xml                                 |  5 ++
 .../apache/calcite/avatica/AvaticaUtils.java    | 48 ++++++++++
 .../calcite/avatica/ConnectionConfigImpl.java   | 20 +----
 .../calcite/avatica/test/AvaticaUtilsTest.java  | 64 +++++++++++++
 .../CachingLatticeStatisticProvider.java        | 57 ++++++++++++
 .../DelegatingLatticeStatisticProvider.java     | 41 +++++++++
 .../org/apache/calcite/materialize/Lattice.java | 95 ++++++++------------
 .../materialize/LatticeStatisticProvider.java   | 27 ++++++
 .../apache/calcite/materialize/Lattices.java    | 40 +++++++++
 .../SqlLatticeStatisticProvider.java            | 48 ++++++++++
 .../org/apache/calcite/model/JsonLattice.java   | 13 +++
 .../org/apache/calcite/model/ModelHandler.java  |  3 +
 .../test/FoodMartLatticeStatisticProvider.java  | 92 +++++++++++++++++++
 .../org/apache/calcite/test/LatticeTest.java    | 28 ++++--
 site/_docs/model.md                             | 17 +++-
 16 files changed, 517 insertions(+), 82 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-calcite/blob/4cc539fc/avatica-server/src/test/java/org/apache/calcite/avatica/test/AvaticaSuite.java
----------------------------------------------------------------------
diff --git a/avatica-server/src/test/java/org/apache/calcite/avatica/test/AvaticaSuite.java
b/avatica-server/src/test/java/org/apache/calcite/avatica/test/AvaticaSuite.java
index 5afba20..4a0c26c 100644
--- a/avatica-server/src/test/java/org/apache/calcite/avatica/test/AvaticaSuite.java
+++ b/avatica-server/src/test/java/org/apache/calcite/avatica/test/AvaticaSuite.java
@@ -27,6 +27,7 @@ import org.junit.runners.Suite;
  */
 @RunWith(Suite.class)
 @Suite.SuiteClasses({
+    AvaticaUtilsTest.class,
     ConnectStringParserTest.class,
     RemoteDriverTest.class
 })

http://git-wip-us.apache.org/repos/asf/incubator-calcite/blob/4cc539fc/avatica/pom.xml
----------------------------------------------------------------------
diff --git a/avatica/pom.xml b/avatica/pom.xml
index 3e241db..50b80c7 100644
--- a/avatica/pom.xml
+++ b/avatica/pom.xml
@@ -53,6 +53,11 @@ limitations under the License.
       <artifactId>junit</artifactId>
       <scope>test</scope>
     </dependency>
+    <dependency>
+      <groupId>org.hamcrest</groupId>
+      <artifactId>hamcrest-core</artifactId>
+      <scope>test</scope>
+    </dependency>
   </dependencies>
 
   <build>

http://git-wip-us.apache.org/repos/asf/incubator-calcite/blob/4cc539fc/avatica/src/main/java/org/apache/calcite/avatica/AvaticaUtils.java
----------------------------------------------------------------------
diff --git a/avatica/src/main/java/org/apache/calcite/avatica/AvaticaUtils.java b/avatica/src/main/java/org/apache/calcite/avatica/AvaticaUtils.java
index 77fafc4..5ca5245 100644
--- a/avatica/src/main/java/org/apache/calcite/avatica/AvaticaUtils.java
+++ b/avatica/src/main/java/org/apache/calcite/avatica/AvaticaUtils.java
@@ -16,6 +16,7 @@
  */
 package org.apache.calcite.avatica;
 
+import java.lang.reflect.Field;
 import java.util.AbstractList;
 import java.util.HashMap;
 import java.util.List;
@@ -117,6 +118,53 @@ public class AvaticaUtils {
     }
     return clazz;
   }
+
+  /** Creates an instance of a plugin class. First looks for a static
+   * member called INSTANCE, then calls a public default constructor.
+   *
+   * <p>If className contains a "#" instead looks for a static field.
+   *
+   * @param pluginClass Class (or interface) to instantiate
+   * @param className Name of implementing class
+   * @param <T> Class
+   * @return Plugin instance
+   */
+  public static <T> T instantiatePlugin(Class<T> pluginClass,
+      String className) {
+    try {
+      // Given a static field, say "com.example.MyClass#FOO_INSTANCE", return
+      // the value of that static field.
+      if (className.contains("#")) {
+        try {
+          int i = className.indexOf('#');
+          String left = className.substring(0, i);
+          String right = className.substring(i + 1);
+          //noinspection unchecked
+          final Class<T> clazz = (Class) Class.forName(left);
+          final Field field;
+          field = clazz.getField(right);
+          return pluginClass.cast(field.get(null));
+        } catch (NoSuchFieldException e) {
+          // ignore
+        }
+      }
+      //noinspection unchecked
+      final Class<T> clazz = (Class) Class.forName(className);
+      assert pluginClass.isAssignableFrom(clazz);
+      try {
+        // We assume that if there is an INSTANCE field it is static and
+        // has the right type.
+        final Field field = clazz.getField("INSTANCE");
+        return pluginClass.cast(field.get(null));
+      } catch (NoSuchFieldException e) {
+        // ignore
+      }
+      return clazz.newInstance();
+    } catch (Exception e) {
+      throw new RuntimeException("Property '" + className
+          + "' not valid for plugin type " + pluginClass.getName(), e);
+    }
+  }
 }
 
 // End AvaticaUtils.java

http://git-wip-us.apache.org/repos/asf/incubator-calcite/blob/4cc539fc/avatica/src/main/java/org/apache/calcite/avatica/ConnectionConfigImpl.java
----------------------------------------------------------------------
diff --git a/avatica/src/main/java/org/apache/calcite/avatica/ConnectionConfigImpl.java b/avatica/src/main/java/org/apache/calcite/avatica/ConnectionConfigImpl.java
index 300063f..3200bee 100644
--- a/avatica/src/main/java/org/apache/calcite/avatica/ConnectionConfigImpl.java
+++ b/avatica/src/main/java/org/apache/calcite/avatica/ConnectionConfigImpl.java
@@ -18,7 +18,6 @@ package org.apache.calcite.avatica;
 
 import org.apache.calcite.avatica.remote.Service;
 
-import java.lang.reflect.Field;
 import java.util.LinkedHashMap;
 import java.util.Map;
 import java.util.Properties;
@@ -217,24 +216,7 @@ public class ConnectionConfigImpl implements ConnectionConfig {
           throw new RuntimeException("Required property '"
               + connectionProperty.camelName() + "' not specified");
         }
-        // First look for a C.INSTANCE field, then do new C().
-        try {
-          //noinspection unchecked
-          final Class<T> clazz = (Class) Class.forName(s);
-          assert pluginClass.isAssignableFrom(clazz);
-          try {
-            // We assume that if there is an INSTANCE field it is static and
-            // has the right type.
-            final Field field = clazz.getField("INSTANCE");
-            return pluginClass.cast(field.get(null));
-          } catch (NoSuchFieldException e) {
-            // ignore
-          }
-          return clazz.newInstance();
-        } catch (Exception e) {
-          throw new RuntimeException("Property '" + s
-              + "' not valid for plugin type " + pluginClass.getName(), e);
-        }
+        return AvaticaUtils.instantiatePlugin(pluginClass, s);
       }
     };
   }

http://git-wip-us.apache.org/repos/asf/incubator-calcite/blob/4cc539fc/avatica/src/test/java/org/apache/calcite/avatica/test/AvaticaUtilsTest.java
----------------------------------------------------------------------
diff --git a/avatica/src/test/java/org/apache/calcite/avatica/test/AvaticaUtilsTest.java b/avatica/src/test/java/org/apache/calcite/avatica/test/AvaticaUtilsTest.java
new file mode 100644
index 0000000..ca548a6
--- /dev/null
+++ b/avatica/src/test/java/org/apache/calcite/avatica/test/AvaticaUtilsTest.java
@@ -0,0 +1,64 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.calcite.avatica.test;
+
+import org.apache.calcite.avatica.AvaticaUtils;
+
+import org.junit.Test;
+
+import java.math.BigInteger;
+
+import static org.hamcrest.CoreMatchers.is;
+import static org.junit.Assert.assertThat;
+import static org.junit.Assert.fail;
+
+/**
+ * Unit test for Avatica utilities.
+ */
+public class AvaticaUtilsTest {
+  @Test public void testInstantiatePlugin() {
+    final String s =
+        AvaticaUtils.instantiatePlugin(String.class, "java.lang.String");
+    assertThat(s, is(""));
+
+    // No default constructor or INSTANCE member
+    try {
+      final Integer i =
+          AvaticaUtils.instantiatePlugin(Integer.class, "java.lang.Integer");
+      fail("expected error, got " + i);
+    } catch (Throwable e) {
+      assertThat(e.getMessage(),
+          is("Property 'java.lang.Integer' not valid for plugin type java.lang.Integer"));
+    }
+
+    final BigInteger b =
+        AvaticaUtils.instantiatePlugin(BigInteger.class, "java.math.BigInteger#ONE");
+    assertThat(b, is(BigInteger.ONE));
+
+    try {
+      final BigInteger b2 =
+          AvaticaUtils.instantiatePlugin(BigInteger.class,
+              "java.math.BigInteger.ONE");
+      fail("expected error, got " + b2);
+    } catch (Throwable e) {
+      assertThat(e.getMessage(),
+          is("Property 'java.math.BigInteger.ONE' not valid for plugin type java.math.BigInteger"));
+    }
+  }
+}
+
+// End AvaticaUtilsTest.java

http://git-wip-us.apache.org/repos/asf/incubator-calcite/blob/4cc539fc/core/src/main/java/org/apache/calcite/materialize/CachingLatticeStatisticProvider.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/calcite/materialize/CachingLatticeStatisticProvider.java
b/core/src/main/java/org/apache/calcite/materialize/CachingLatticeStatisticProvider.java
new file mode 100644
index 0000000..bf70e5d
--- /dev/null
+++ b/core/src/main/java/org/apache/calcite/materialize/CachingLatticeStatisticProvider.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.calcite.materialize;
+
+import org.apache.calcite.util.Pair;
+
+import com.google.common.base.Throwables;
+import com.google.common.cache.CacheBuilder;
+import com.google.common.cache.CacheLoader;
+import com.google.common.cache.LoadingCache;
+
+import java.util.concurrent.ExecutionException;
+
+/**
+ * Implementation of {@link LatticeStatisticProvider} that gets statistics by
+ * executing "SELECT COUNT(DISTINCT ...) ..." SQL queries.
+ */
+class CachingLatticeStatisticProvider implements LatticeStatisticProvider {
+  private final LoadingCache<Pair<Lattice, Lattice.Column>, Integer> cache;
+
+  /** Creates a CachingStatisticProvider. */
+  public CachingLatticeStatisticProvider(
+      final LatticeStatisticProvider provider) {
+    cache = CacheBuilder.<Pair<Lattice, Lattice.Column>>newBuilder()
+        .build(
+            new CacheLoader<Pair<Lattice, Lattice.Column>, Integer>() {
+              public Integer load(Pair<Lattice, Lattice.Column> key)
+                  throws Exception {
+                return provider.cardinality(key.left, key.right);
+              }
+            });
+  }
+
+  public int cardinality(Lattice lattice, Lattice.Column column) {
+    try {
+      return cache.get(Pair.of(lattice, column));
+    } catch (ExecutionException e) {
+      throw Throwables.propagate(e);
+    }
+  }
+}
+
+// End CachingLatticeStatisticProvider.java

http://git-wip-us.apache.org/repos/asf/incubator-calcite/blob/4cc539fc/core/src/main/java/org/apache/calcite/materialize/DelegatingLatticeStatisticProvider.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/calcite/materialize/DelegatingLatticeStatisticProvider.java
b/core/src/main/java/org/apache/calcite/materialize/DelegatingLatticeStatisticProvider.java
new file mode 100644
index 0000000..e1ec6c5
--- /dev/null
+++ b/core/src/main/java/org/apache/calcite/materialize/DelegatingLatticeStatisticProvider.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.calcite.materialize;
+
+/**
+ * Implementation of {@link LatticeStatisticProvider} that delegates
+ * to an underlying provider.
+ */
+public class DelegatingLatticeStatisticProvider
+    implements LatticeStatisticProvider {
+  protected final LatticeStatisticProvider provider;
+
+  /** Creates a DelegatingLatticeStatisticProvider.
+   *
+   * @param provider Provider to which to delegate otherwise unhandled requests
+   */
+  protected DelegatingLatticeStatisticProvider(
+      LatticeStatisticProvider provider) {
+    this.provider = provider;
+  }
+
+  public int cardinality(Lattice lattice, Lattice.Column column) {
+    return provider.cardinality(lattice, column);
+  }
+}
+
+// End DelegatingLatticeStatisticProvider.java

http://git-wip-us.apache.org/repos/asf/incubator-calcite/blob/4cc539fc/core/src/main/java/org/apache/calcite/materialize/Lattice.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/calcite/materialize/Lattice.java b/core/src/main/java/org/apache/calcite/materialize/Lattice.java
index 38b4eac..c2b07c3 100644
--- a/core/src/main/java/org/apache/calcite/materialize/Lattice.java
+++ b/core/src/main/java/org/apache/calcite/materialize/Lattice.java
@@ -16,7 +16,9 @@
  */
 package org.apache.calcite.materialize;
 
+import org.apache.calcite.avatica.AvaticaUtils;
 import org.apache.calcite.jdbc.CalcitePrepare;
+import org.apache.calcite.jdbc.CalciteRootSchema;
 import org.apache.calcite.jdbc.CalciteSchema;
 import org.apache.calcite.plan.RelOptUtil;
 import org.apache.calcite.prepare.CalcitePrepareImpl;
@@ -55,7 +57,6 @@ import com.google.common.base.Function;
 import com.google.common.base.Preconditions;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableListMultimap;
-import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.Lists;
 import com.google.common.collect.Maps;
 import com.google.common.collect.Ordering;
@@ -85,6 +86,7 @@ public class Lattice {
         }
       };
 
+  public final CalciteRootSchema rootSchema;
   public final ImmutableList<Node> nodes;
   public final ImmutableList<Column> columns;
   public final boolean auto;
@@ -94,6 +96,7 @@ public class Lattice {
   public final ImmutableList<Measure> defaultMeasures;
   public final ImmutableList<Tile> tiles;
   public final ImmutableList<String> uniqueColumnNames;
+  public final LatticeStatisticProvider statisticProvider;
 
   private final Function<Integer, Column> toColumnFunction =
       new Function<Integer, Column>() {
@@ -109,15 +112,18 @@ public class Lattice {
         }
       };
 
-  private Lattice(ImmutableList<Node> nodes, boolean auto, boolean algorithm,
-      long algorithmMaxMillis,
-      Double rowCountEstimate, ImmutableList<Column> columns,
-      ImmutableList<Measure> defaultMeasures, ImmutableList<Tile> tiles) {
+  private Lattice(CalciteRootSchema rootSchema, ImmutableList<Node> nodes,
+      boolean auto, boolean algorithm, long algorithmMaxMillis,
+      LatticeStatisticProvider statisticProvider, Double rowCountEstimate,
+      ImmutableList<Column> columns, ImmutableList<Measure> defaultMeasures,
+      ImmutableList<Tile> tiles) {
+    this.rootSchema = rootSchema;
     this.nodes = Preconditions.checkNotNull(nodes);
     this.columns = Preconditions.checkNotNull(columns);
     this.auto = auto;
     this.algorithm = algorithm;
     this.algorithmMaxMillis = algorithmMaxMillis;
+    this.statisticProvider = Preconditions.checkNotNull(statisticProvider);
     this.defaultMeasures = Preconditions.checkNotNull(defaultMeasures);
     this.tiles = Preconditions.checkNotNull(tiles);
 
@@ -328,6 +334,14 @@ public class Lattice {
     return buf.toString();
   }
 
+  /** Returns a SQL query that counts the number of distinct values of the
+   * attributes given in {@code groupSet}. */
+  public String countSql(ImmutableBitSet groupSet) {
+    return "select count(*) as c from ("
+        + sql(groupSet, ImmutableList.<Measure>of())
+        + ")";
+  }
+
   private static void use(List<Node> usedNodes, Node node) {
     if (!usedNodes.contains(node)) {
       if (node.parent != null) {
@@ -376,7 +390,7 @@ public class Lattice {
     // distributed attribute with N1 * ... * Nm distinct values.
     BigInteger n = BigInteger.ONE;
     for (Column column : columns) {
-      final int cardinality = cardinality(column);
+      final int cardinality = statisticProvider.cardinality(this, column);
       if (cardinality > 1) {
         n = n.multiply(BigInteger.valueOf(cardinality));
       }
@@ -394,53 +408,6 @@ public class Lattice {
     return Math.min(v, f);
   }
 
-  public static final Map<String, Integer> CARDINALITY_MAP =
-      ImmutableMap.<String, Integer>builder()
-          .put("brand_name", 111)
-          .put("cases_per_pallet", 10)
-          .put("customer_id", 5581)
-          .put("day_of_month", 30)
-          .put("fiscal_period", 0)
-          .put("gross_weight", 376)
-          .put("low_fat", 2)
-          .put("month_of_year", 12)
-          .put("net_weight", 332)
-          .put("product_category", 45)
-          .put("product_class_id", 102)
-          .put("product_department", 22)
-          .put("product_family", 3)
-          .put("product_id", 1559)
-          .put("product_name", 1559)
-          .put("product_subcategory", 102)
-          .put("promotion_id", 149)
-          .put("quarter", 4)
-          .put("recyclable_package", 2)
-          .put("shelf_depth", 488)
-          .put("shelf_height", 524)
-          .put("shelf_width", 534)
-          .put("SKU", 1559)
-          .put("SRP", 315)
-          .put("store_cost", 10777)
-          .put("store_id", 13)
-          .put("store_sales", 1049)
-          .put("the_date", 323)
-          .put("the_day", 7)
-          .put("the_month", 12)
-          .put("the_year", 1)
-          .put("time_id", 323)
-          .put("units_per_case", 36)
-          .put("unit_sales", 6)
-          .put("week_of_year", 52)
-          .build();
-
-  private int cardinality(Column column) {
-    final Integer integer = CARDINALITY_MAP.get(column.alias);
-    if (integer != null && integer > 0) {
-      return integer;
-    }
-    return column.alias.length();
-  }
-
   /** Source relation of a lattice.
    *
    * <p>Relations form a tree; all relations except the root relation
@@ -601,12 +568,15 @@ public class Lattice {
         ImmutableList.builder();
     private final ImmutableList.Builder<Tile> tileListBuilder =
         ImmutableList.builder();
+    private final CalciteRootSchema rootSchema;
     private boolean algorithm = false;
     private long algorithmMaxMillis = -1;
     private boolean auto = true;
     private Double rowCountEstimate;
+    private String statisticProvider;
 
     public Builder(CalciteSchema schema, String sql) {
+      this.rootSchema = schema.root();
       CalcitePrepare.ConvertResult parsed =
           Schemas.convert(MaterializedViewTable.MATERIALIZATION_CONNECTION,
               schema, schema.path(null), sql);
@@ -710,11 +680,24 @@ public class Lattice {
       return this;
     }
 
+    /** Sets the "statisticProvider" attribute.
+     *
+     * <p>If not set, the lattice will use {@link Lattices#CACHED_SQL}. */
+    public Builder statisticProvider(String statisticProvider) {
+      this.statisticProvider = statisticProvider;
+      return this;
+    }
+
     /** Builds a lattice. */
     public Lattice build() {
-      return new Lattice(ImmutableList.copyOf(nodes), auto, algorithm,
-          algorithmMaxMillis, rowCountEstimate, columns,
-          defaultMeasureListBuilder.build(), tileListBuilder.build());
+      LatticeStatisticProvider statisticProvider =
+          this.statisticProvider != null
+              ? AvaticaUtils.instantiatePlugin(LatticeStatisticProvider.class,
+                  this.statisticProvider)
+              : Lattices.CACHED_SQL;
+      return new Lattice(rootSchema, ImmutableList.copyOf(nodes), auto,
+          algorithm, algorithmMaxMillis, statisticProvider, rowCountEstimate,
+          columns, defaultMeasureListBuilder.build(), tileListBuilder.build());
     }
 
     /** Resolves the arguments of a

http://git-wip-us.apache.org/repos/asf/incubator-calcite/blob/4cc539fc/core/src/main/java/org/apache/calcite/materialize/LatticeStatisticProvider.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/calcite/materialize/LatticeStatisticProvider.java
b/core/src/main/java/org/apache/calcite/materialize/LatticeStatisticProvider.java
new file mode 100644
index 0000000..46668cc
--- /dev/null
+++ b/core/src/main/java/org/apache/calcite/materialize/LatticeStatisticProvider.java
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.calcite.materialize;
+
+/**
+ * Estimates row counts for a lattice and its attributes.
+ */
+public interface LatticeStatisticProvider {
+  /** Returns an estimate of the number of distinct values in a column. */
+  int cardinality(Lattice lattice, Lattice.Column column);
+}
+
+// End LatticeStatisticProvider.java

http://git-wip-us.apache.org/repos/asf/incubator-calcite/blob/4cc539fc/core/src/main/java/org/apache/calcite/materialize/Lattices.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/calcite/materialize/Lattices.java b/core/src/main/java/org/apache/calcite/materialize/Lattices.java
new file mode 100644
index 0000000..cf70719
--- /dev/null
+++ b/core/src/main/java/org/apache/calcite/materialize/Lattices.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.calcite.materialize;
+
+/**
+ * Utilities for {@link Lattice}, {@link LatticeStatisticProvider}.
+ */
+public class Lattices {
+  private Lattices() {}
+
+  /** Statistics provider that uses SQL. */
+  public static final LatticeStatisticProvider SQL =
+      SqlLatticeStatisticProvider.INSTANCE;
+
+  /** Statistics provider that uses SQL then stores the results in a cache. */
+  public static final LatticeStatisticProvider CACHED_SQL =
+      cache(SqlLatticeStatisticProvider.INSTANCE);
+
+  /** Wraps a statistic provider in a cache. */
+  public static LatticeStatisticProvider cache(
+      LatticeStatisticProvider provider) {
+    return new CachingLatticeStatisticProvider(provider);
+  }
+}
+
+// End Lattices.java

http://git-wip-us.apache.org/repos/asf/incubator-calcite/blob/4cc539fc/core/src/main/java/org/apache/calcite/materialize/SqlLatticeStatisticProvider.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/calcite/materialize/SqlLatticeStatisticProvider.java
b/core/src/main/java/org/apache/calcite/materialize/SqlLatticeStatisticProvider.java
new file mode 100644
index 0000000..4a94847
--- /dev/null
+++ b/core/src/main/java/org/apache/calcite/materialize/SqlLatticeStatisticProvider.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.calcite.materialize;
+
+import org.apache.calcite.schema.ScannableTable;
+import org.apache.calcite.schema.Table;
+import org.apache.calcite.util.ImmutableBitSet;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Iterables;
+
+/**
+ * Implementation of {@link LatticeStatisticProvider} that gets statistics by
+ * executing "SELECT COUNT(DISTINCT ...) ..." SQL queries.
+ */
+class SqlLatticeStatisticProvider implements LatticeStatisticProvider {
+  static final SqlLatticeStatisticProvider INSTANCE =
+      new SqlLatticeStatisticProvider();
+
+  /** Creates an SqlLatticeStatisticProvider. */
+  private SqlLatticeStatisticProvider() {}
+
+  @Override public int cardinality(Lattice lattice, Lattice.Column column) {
+    final String sql = lattice.countSql(ImmutableBitSet.of(column.ordinal));
+    final Table table =
+        new MaterializationService.DefaultTableFactory()
+            .createTable(lattice.rootSchema, sql, ImmutableList.<String>of());
+    final Object[] values =
+        Iterables.getOnlyElement(((ScannableTable) table).scan(null));
+    return ((Number) values[0]).intValue();
+  }
+}
+
+// End SqlLatticeStatisticProvider.java

http://git-wip-us.apache.org/repos/asf/incubator-calcite/blob/4cc539fc/core/src/main/java/org/apache/calcite/model/JsonLattice.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/calcite/model/JsonLattice.java b/core/src/main/java/org/apache/calcite/model/JsonLattice.java
index 80974a3..d96eb06 100644
--- a/core/src/main/java/org/apache/calcite/model/JsonLattice.java
+++ b/core/src/main/java/org/apache/calcite/model/JsonLattice.java
@@ -55,6 +55,19 @@ public class JsonLattice {
    * <p>If null, Calcite will a query to find the real value. */
   public Double rowCountEstimate;
 
+  /** Name of a class that provides estimates of the number of distinct values
+   * in each column.
+   *
+   * <p>The class must implement the
+   * {@link org.apache.calcite.materialize.LatticeStatisticProvider} interface.
+   *
+   * <p>Or, you can use a class name plus a static field, for example
+   * "org.apache.calcite.materialize.Lattices#CACHING_SQL_STATISTIC_PROVIDER".
+   *
+   * <p>If not set, Calcite will generate and execute a SQL query to find the
+   * real value, and cache the results. */
+  public String statisticProvider;
+
   /** List of materialized aggregates to create up front. */
   public final List<JsonTile> tiles = Lists.newArrayList();
 

http://git-wip-us.apache.org/repos/asf/incubator-calcite/blob/4cc539fc/core/src/main/java/org/apache/calcite/model/ModelHandler.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/calcite/model/ModelHandler.java b/core/src/main/java/org/apache/calcite/model/ModelHandler.java
index dfdb901..0c0efa8 100644
--- a/core/src/main/java/org/apache/calcite/model/ModelHandler.java
+++ b/core/src/main/java/org/apache/calcite/model/ModelHandler.java
@@ -291,6 +291,9 @@ public class ModelHandler {
       if (jsonLattice.rowCountEstimate != null) {
         latticeBuilder.rowCountEstimate(jsonLattice.rowCountEstimate);
       }
+      if (jsonLattice.statisticProvider != null) {
+        latticeBuilder.statisticProvider(jsonLattice.statisticProvider);
+      }
       populateLattice(jsonLattice, latticeBuilder);
       schema.add(jsonLattice.name, latticeBuilder.build());
     } catch (Exception e) {

http://git-wip-us.apache.org/repos/asf/incubator-calcite/blob/4cc539fc/core/src/test/java/org/apache/calcite/test/FoodMartLatticeStatisticProvider.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/calcite/test/FoodMartLatticeStatisticProvider.java
b/core/src/test/java/org/apache/calcite/test/FoodMartLatticeStatisticProvider.java
new file mode 100644
index 0000000..3f36bf9
--- /dev/null
+++ b/core/src/test/java/org/apache/calcite/test/FoodMartLatticeStatisticProvider.java
@@ -0,0 +1,92 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.calcite.test;
+
+import org.apache.calcite.materialize.DelegatingLatticeStatisticProvider;
+import org.apache.calcite.materialize.Lattice;
+import org.apache.calcite.materialize.LatticeStatisticProvider;
+import org.apache.calcite.materialize.Lattices;
+
+import com.google.common.collect.ImmutableMap;
+
+import java.util.Map;
+
+/**
+ * Implementation of {@link LatticeStatisticProvider}
+ * that has hard-coded values for various attributes in the FoodMart lattice.
+ *
+ * <p>This makes testing faster.
+ */
+public class FoodMartLatticeStatisticProvider
+    extends DelegatingLatticeStatisticProvider {
+  public static final FoodMartLatticeStatisticProvider INSTANCE =
+      new FoodMartLatticeStatisticProvider(Lattices.CACHED_SQL);
+
+  public static final Map<String, Integer> CARDINALITY_MAP =
+      ImmutableMap.<String, Integer>builder()
+          .put("brand_name", 111)
+          .put("cases_per_pallet", 10)
+          .put("customer_id", 5581)
+          .put("day_of_month", 30)
+          .put("fiscal_period", 0)
+          .put("gross_weight", 376)
+          .put("low_fat", 2)
+          .put("month_of_year", 12)
+          .put("net_weight", 332)
+          .put("product_category", 45)
+          .put("product_class_id", 102)
+          .put("product_department", 22)
+          .put("product_family", 3)
+          .put("product_id", 1559)
+          .put("product_name", 1559)
+          .put("product_subcategory", 102)
+          .put("promotion_id", 149)
+          .put("quarter", 4)
+          .put("recyclable_package", 2)
+          .put("shelf_depth", 488)
+          .put("shelf_height", 524)
+          .put("shelf_width", 534)
+          .put("SKU", 1559)
+          .put("SRP", 315)
+          .put("store_cost", 10777)
+          .put("store_id", 13)
+          .put("store_sales", 1049)
+          .put("the_date", 323)
+          .put("the_day", 7)
+          .put("the_month", 12)
+          .put("the_year", 1)
+          .put("time_id", 323)
+          .put("units_per_case", 36)
+          .put("unit_sales", 6)
+          .put("week_of_year", 52)
+          .build();
+
+  private FoodMartLatticeStatisticProvider(LatticeStatisticProvider provider) {
+    super(provider);
+  }
+
+  /** Returns an estimate of the number of distinct values in a column. */
+  public int cardinality(Lattice lattice, Lattice.Column column) {
+    final Integer integer = CARDINALITY_MAP.get(column.alias);
+    if (integer != null && integer > 0) {
+      return integer;
+    }
+    return column.alias.length();
+  }
+}
+
+// End FoodMartLatticeStatisticProvider.java

http://git-wip-us.apache.org/repos/asf/incubator-calcite/blob/4cc539fc/core/src/test/java/org/apache/calcite/test/LatticeTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/calcite/test/LatticeTest.java b/core/src/test/java/org/apache/calcite/test/LatticeTest.java
index 492b377..b4a79af 100644
--- a/core/src/test/java/org/apache/calcite/test/LatticeTest.java
+++ b/core/src/test/java/org/apache/calcite/test/LatticeTest.java
@@ -16,6 +16,7 @@
  */
 package org.apache.calcite.test;
 
+import org.apache.calcite.materialize.Lattices;
 import org.apache.calcite.materialize.MaterializationService;
 import org.apache.calcite.plan.RelOptUtil;
 import org.apache.calcite.rel.RelNode;
@@ -309,10 +310,25 @@ public class LatticeTest {
    * tiles.
    *
    * <p>Test case for
-   * <a href="https://issues.apache.org/jira/browse/CALCITE-428">CALCITE-428,
-   * "Use optimization algorithm to suggest which tiles of a lattice to
-   * materialize"</a>. */
+   * <a href="https://issues.apache.org/jira/browse/CALCITE-428">[CALCITE-428]
+   * Use optimization algorithm to suggest which tiles of a lattice to
+   * materialize</a>. */
   @Test public void testTileAlgorithm() {
+    checkTileAlgorithm(FoodMartLatticeStatisticProvider.class.getCanonicalName(),
+        "EnumerableAggregate(group=[{2, 3}])\n"
+            + "  EnumerableTableScan(table=[[adhoc, m{16, 17, 27, 31}]])");
+  }
+
+  @Test public void testTileAlgorithm2() {
+    // Different explain than above, but note that it still selects columns
+    // (27, 31).
+    checkTileAlgorithm(Lattices.class.getCanonicalName() + "#CACHED_SQL",
+        "EnumerableAggregate(group=[{0, 1}])\n"
+            + "  EnumerableTableScan(table=[[adhoc, m{27, 31, 32, 36, 37}]");
+  }
+
+  private void checkTileAlgorithm(String statisticProvider,
+      String expectedExplain) {
     MaterializationService.setThreadLocal();
     MaterializationService.instance().clear();
     foodmartModel(
@@ -329,6 +345,9 @@ public class LatticeTest {
         + "    }, {\n"
         + "      agg: 'count'\n"
         + "  } ],\n"
+        + "  statisticProvider: '"
+        + statisticProvider
+        + "',\n"
         + "  tiles: [ {\n"
         + "    dimensions: [ 'the_year', ['t', 'quarter'] ],\n"
         + "    measures: [ ]\n"
@@ -337,8 +356,7 @@ public class LatticeTest {
             + "from \"foodmart\".\"sales_fact_1997\" as s\n"
             + "join \"foodmart\".\"time_by_day\" as t using (\"time_id\")\n")
         .enableMaterializations(true)
-        .explainContains("EnumerableAggregate(group=[{2, 3}])\n"
-            + "  EnumerableTableScan(table=[[adhoc, m{16, 17, 27, 31}]])")
+        .explainContains(expectedExplain)
         .returnsUnordered("the_year=1997; quarter=Q1",
             "the_year=1997; quarter=Q2",
             "the_year=1997; quarter=Q3",

http://git-wip-us.apache.org/repos/asf/incubator-calcite/blob/4cc539fc/site/_docs/model.md
----------------------------------------------------------------------
diff --git a/site/_docs/model.md b/site/_docs/model.md
index 78061b0..a6514a3 100644
--- a/site/_docs/model.md
+++ b/site/_docs/model.md
@@ -21,6 +21,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 {% endcomment %}
 -->
+{% assign apiRoot = "/apidocs" %}
 
 Calcite models can be represented as JSON files.
 This page describes the structure of those files.
@@ -148,7 +149,8 @@ Like base class <a href="#schema">Schema</a>, occurs within
`root.schemas`.
 <a href="#schema">Schema</a>.
 
 `factory` (required string) is the name of the factory class for this
-schema. Must implement interface `org.apache.calcite.schema.SchemaFactory`
+schema. Must implement interface
+[org.apache.calcite.schema.SchemaFactory]({{ apiRoot }}/org/apache/calcite/schema/SchemaFactory.html)
 and have a public default constructor.
 
 `operand` (optional map) contains attributes to be passed to the
@@ -293,7 +295,8 @@ Like base class <a href="#table">Table</a>, occurs within
`root.schemas.tables`.
 `name`, `type`, `columns` inherited from <a href="#table">Table</a>.
 
 `factory` (required string) is the name of the factory class for this
-table. Must implement interface `org.apache.calcite.schema.TableFactory`
+table. Must implement interface
+[org.apache.calcite.schema.TableFactory]({{ apiRoot }}/org/apache/calcite/schema/TableFactory.html)
 and have a public default constructor.
 
 `operand` (optional map) contains attributes to be passed to the
@@ -401,6 +404,16 @@ just 'count(*)':
 [ { name: 'count' } ]
 {% endhighlight %}
 
+`statisticProvider` (optional name of a class that implements 
+[org.apache.calcite.materialize.LatticeStatisticProvider]({{ apiRoot }}/org/apache/calcite/materialize/LatticeStatisticProvider.html))
+is provides estimates of the number of distinct values * in each column.
+
+You can use a class name, or a class plus a static field, for example
+`org.apache.calcite.materialize.Lattices#CACHING_SQL_STATISTIC_PROVIDER`.
+
+If not set, Calcite will generate and execute a SQL query to find the real
+value, and cache the results.
+
 See also: <a href="lattice.md">Lattices</a>.
 
 ### Tile



Mime
View raw message