calcite-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jh...@apache.org
Subject [2/2] incubator-calcite git commit: [CALCITE-510] Support for grouping sets in AggregateExpandDistinctAggregatesRule (Jesus Camacho Rodriguez)
Date Thu, 11 Dec 2014 19:24:49 GMT
[CALCITE-510] Support for grouping sets in AggregateExpandDistinctAggregatesRule (Jesus Camacho
Rodriguez)

Close apache/incubator-calcite#27


Project: http://git-wip-us.apache.org/repos/asf/incubator-calcite/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-calcite/commit/24258852
Tree: http://git-wip-us.apache.org/repos/asf/incubator-calcite/tree/24258852
Diff: http://git-wip-us.apache.org/repos/asf/incubator-calcite/diff/24258852

Branch: refs/heads/master
Commit: 24258852fe70e22dd94dd2e625e7fd781da1b236
Parents: 478ca9d
Author: Jesus Camacho Rodriguez <jcamachorodriguez@hortonworks.com>
Authored: Tue Dec 9 13:16:16 2014 +0000
Committer: Julian Hyde <jhyde@apache.org>
Committed: Thu Dec 11 10:26:07 2014 -0800

----------------------------------------------------------------------
 .../adapter/enumerable/EnumerableAggregate.java | 10 +--
 .../org/apache/calcite/rel/core/Aggregate.java  | 15 ++++
 .../AggregateExpandDistinctAggregatesRule.java  | 32 +++++----
 .../rel/rules/AggregateReduceFunctionsRule.java |  2 +-
 .../apache/calcite/sql2rel/RelFieldTrimmer.java |  6 +-
 .../apache/calcite/test/RelOptRulesTest.java    | 32 ++++++++-
 .../org/apache/calcite/test/RelOptRulesTest.xml | 76 +++++++++++++++++++-
 7 files changed, 147 insertions(+), 26 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-calcite/blob/24258852/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableAggregate.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableAggregate.java
b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableAggregate.java
index fa945eb..39aaec9 100644
--- a/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableAggregate.java
+++ b/core/src/main/java/org/apache/calcite/adapter/enumerable/EnumerableAggregate.java
@@ -184,8 +184,8 @@ public class EnumerableAggregate extends Aggregate
     final PhysType keyPhysType =
         inputPhysType.project(groupSet.asList(), getGroupType() != Group.SIMPLE,
             JavaRowFormat.LIST);
-    final int keyArity = groupSet.cardinality();
-    final int indicatorArity = indicator ? keyArity : 0;
+    final int groupCount = getGroupCount();
+    final int indicatorCount = getIndicatorCount();
 
     final List<AggImpState> aggs =
         new ArrayList<AggImpState>(aggCalls.size());
@@ -345,12 +345,12 @@ public class EnumerableAggregate extends Aggregate
     final BlockBuilder resultBlock = new BlockBuilder();
     final List<Expression> results = Expressions.list();
     final ParameterExpression key_;
-    if (keyArity == 0) {
+    if (groupCount == 0) {
       key_ = null;
     } else {
       final Type keyType = keyPhysType.getJavaRowType();
       key_ = Expressions.parameter(keyType, "key");
-      for (int j = 0; j < keyArity + indicatorArity; j++) {
+      for (int j = 0; j < groupCount + indicatorCount; j++) {
         results.add(
             keyPhysType.fieldReference(key_, j));
       }
@@ -388,7 +388,7 @@ public class EnumerableAggregate extends Aggregate
                       accumulatorAdder,
                       resultSelector)
                       .appendIfNotNull(keyPhysType.comparer()))));
-    } else if (keyArity == 0) {
+    } else if (groupCount == 0) {
       final Expression resultSelector =
           builder.append(
               "resultSelector",

http://git-wip-us.apache.org/repos/asf/incubator-calcite/blob/24258852/core/src/main/java/org/apache/calcite/rel/core/Aggregate.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/calcite/rel/core/Aggregate.java b/core/src/main/java/org/apache/calcite/rel/core/Aggregate.java
index f6d7c8f..0072d4f 100644
--- a/core/src/main/java/org/apache/calcite/rel/core/Aggregate.java
+++ b/core/src/main/java/org/apache/calcite/rel/core/Aggregate.java
@@ -198,6 +198,21 @@ public abstract class Aggregate extends SingleRel {
   }
 
   /**
+   * Returns the number of indicator fields.
+   *
+   * <p>This is the same as {@link #getGroupCount()} if {@link #indicator} is
+   * true, zero if {@code indicator} is false.
+   *
+   * <p>The offset of the first aggregate call in the output record is always
+   * <i>groupCount + indicatorCount</i>.
+   *
+   * @return number of indicator fields
+   */
+  public int getIndicatorCount() {
+    return indicator ? getGroupCount() : 0;
+  }
+
+  /**
    * Returns a bit set of the grouping fields.
    *
    * @return bit set of ordinals of grouping fields

http://git-wip-us.apache.org/repos/asf/incubator-calcite/blob/24258852/core/src/main/java/org/apache/calcite/rel/rules/AggregateExpandDistinctAggregatesRule.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/calcite/rel/rules/AggregateExpandDistinctAggregatesRule.java
b/core/src/main/java/org/apache/calcite/rel/rules/AggregateExpandDistinctAggregatesRule.java
index 4c017e0..d661727 100644
--- a/core/src/main/java/org/apache/calcite/rel/rules/AggregateExpandDistinctAggregatesRule.java
+++ b/core/src/main/java/org/apache/calcite/rel/rules/AggregateExpandDistinctAggregatesRule.java
@@ -77,7 +77,7 @@ public final class AggregateExpandDistinctAggregatesRule extends RelOptRule
{
   public AggregateExpandDistinctAggregatesRule(
       Class<? extends LogicalAggregate> clazz,
       RelFactories.JoinFactory joinFactory) {
-    super(operand(clazz, null, Aggregate.IS_SIMPLE, any()));
+    super(operand(clazz, any()));
     this.joinFactory = joinFactory;
   }
 
@@ -124,14 +124,14 @@ public final class AggregateExpandDistinctAggregatesRule extends RelOptRule
{
     final List<RexInputRef> refs = new ArrayList<RexInputRef>();
     final List<String> fieldNames = aggregate.getRowType().getFieldNames();
     final ImmutableBitSet groupSet = aggregate.getGroupSet();
-    for (int i : Util.range(groupSet.cardinality())) {
+    final int groupAndIndicatorCount =
+        aggregate.getGroupCount() + aggregate.getIndicatorCount();
+    for (int i : Util.range(groupAndIndicatorCount)) {
       refs.add(RexInputRef.of(i, aggFields));
     }
 
     // Aggregate the original relation, including any non-distinct aggregates.
-
     List<AggregateCall> newAggCallList = new ArrayList<AggregateCall>();
-    final int groupCount = groupSet.cardinality();
     int i = -1;
     for (AggregateCall aggCall : aggregate.getAggCallList()) {
       ++i;
@@ -141,8 +141,8 @@ public final class AggregateExpandDistinctAggregatesRule extends RelOptRule
{
       }
       refs.add(
           new RexInputRef(
-              groupCount + newAggCallList.size(),
-              aggFields.get(groupCount + i).getType()));
+              groupAndIndicatorCount + newAggCallList.size(),
+              aggFields.get(groupAndIndicatorCount + i).getType()));
       newAggCallList.add(aggCall);
     }
 
@@ -155,7 +155,8 @@ public final class AggregateExpandDistinctAggregatesRule extends RelOptRule
{
     } else {
       rel =
           new LogicalAggregate(aggregate.getCluster(), aggregate.getInput(),
-              false, groupSet, null, newAggCallList);
+              aggregate.indicator, groupSet, aggregate.getGroupSets(),
+              newAggCallList);
     }
 
     // For each set of operands, find and rewrite all calls which have that
@@ -294,8 +295,9 @@ public final class AggregateExpandDistinctAggregatesRule extends RelOptRule
{
     List<AggregateCall> aggCallList = new ArrayList<AggregateCall>();
     final List<AggregateCall> aggCalls = aggregate.getAggCallList();
 
-    final int groupCount = aggregate.getGroupSet().cardinality();
-    int i = groupCount - 1;
+    final int groupAndIndicatorCount =
+        aggregate.getGroupCount() + aggregate.getIndicatorCount();
+    int i = groupAndIndicatorCount - 1;
     for (AggregateCall aggCall : aggCalls) {
       ++i;
 
@@ -327,20 +329,20 @@ public final class AggregateExpandDistinctAggregatesRule extends RelOptRule
{
       assert refs.get(i) == null;
       if (left == null) {
         refs.set(i,
-            new RexInputRef(groupCount + aggCallList.size(),
+            new RexInputRef(groupAndIndicatorCount + aggCallList.size(),
                 newAggCall.getType()));
       } else {
         refs.set(i,
-            new RexInputRef(leftFields.size() + groupCount + aggCallList.size(),
-                newAggCall.getType()));
+            new RexInputRef(leftFields.size() + groupAndIndicatorCount
+                + aggCallList.size(), newAggCall.getType()));
       }
       aggCallList.add(newAggCall);
     }
 
     Aggregate distinctAgg =
-        aggregate.copy(aggregate.getTraitSet(), distinct, false,
+        aggregate.copy(aggregate.getTraitSet(), distinct, aggregate.indicator,
             ImmutableBitSet.range(aggregate.getGroupSet().cardinality()),
-            null, aggCallList);
+            aggregate.getGroupSets(), aggCallList);
 
     // If there's no left child yet, no need to create the join
     if (left == null) {
@@ -353,7 +355,7 @@ public final class AggregateExpandDistinctAggregatesRule extends RelOptRule
{
     final List<RelDataTypeField> distinctFields =
         distinctAgg.getRowType().getFieldList();
     final List<RexNode> conditions = Lists.newArrayList();
-    for (i = 0; i < groupCount; ++i) {
+    for (i = 0; i < groupAndIndicatorCount; ++i) {
       // null values form its own group
       // use "is not distinct from" so that the join condition
       // allows null values to match.

http://git-wip-us.apache.org/repos/asf/incubator-calcite/blob/24258852/core/src/main/java/org/apache/calcite/rel/rules/AggregateReduceFunctionsRule.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/calcite/rel/rules/AggregateReduceFunctionsRule.java
b/core/src/main/java/org/apache/calcite/rel/rules/AggregateReduceFunctionsRule.java
index 4d2df21..93c9f09 100644
--- a/core/src/main/java/org/apache/calcite/rel/rules/AggregateReduceFunctionsRule.java
+++ b/core/src/main/java/org/apache/calcite/rel/rules/AggregateReduceFunctionsRule.java
@@ -134,7 +134,7 @@ public class AggregateReduceFunctionsRule extends RelOptRule {
 
     List<AggregateCall> oldCalls = oldAggRel.getAggCallList();
     final int groupCount = oldAggRel.getGroupCount();
-    final int indicatorCount = oldAggRel.indicator ? groupCount : 0;
+    final int indicatorCount = oldAggRel.getIndicatorCount();
 
     final List<AggregateCall> newCalls = Lists.newArrayList();
     final Map<AggregateCall, RexNode> aggCallMapping = Maps.newHashMap();

http://git-wip-us.apache.org/repos/asf/incubator-calcite/blob/24258852/core/src/main/java/org/apache/calcite/sql2rel/RelFieldTrimmer.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/calcite/sql2rel/RelFieldTrimmer.java b/core/src/main/java/org/apache/calcite/sql2rel/RelFieldTrimmer.java
index f2409ff..dbbfa3f 100644
--- a/core/src/main/java/org/apache/calcite/sql2rel/RelFieldTrimmer.java
+++ b/core/src/main/java/org/apache/calcite/sql2rel/RelFieldTrimmer.java
@@ -773,7 +773,7 @@ public class RelFieldTrimmer implements ReflectiveVisitor {
     // We have to return group keys and (if present) indicators.
     // So, pretend that the consumer asked for them.
     final int groupCount = aggregate.getGroupSet().cardinality();
-    final int indicatorCount = aggregate.indicator ? groupCount : 0;
+    final int indicatorCount = aggregate.getIndicatorCount();
     fieldsUsed =
         fieldsUsed.union(ImmutableBitSet.range(groupCount + indicatorCount));
 
@@ -814,8 +814,8 @@ public class RelFieldTrimmer implements ReflectiveVisitor {
                   }
                 }));
 
-    // Populate mapping of where to find the fields. System and grouping
-    // fields first.
+    // Populate mapping of where to find the fields. System, group key and
+    // indicator fields first.
     for (IntPair pair : inputMapping) {
       if (pair.source < groupCount) {
         mapping.set(pair.source, pair.target);

http://git-wip-us.apache.org/repos/asf/incubator-calcite/blob/24258852/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java b/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java
index 7c78e36..9459a67 100644
--- a/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java
+++ b/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java
@@ -316,7 +316,7 @@ public class RelOptRulesTest extends RelOptTestBase {
             + " from sales.dept group by name");
   }
 
-  @Test public void testDistinctCount() {
+  @Test public void testDistinctCount1() {
     final HepProgram program = HepProgram.builder()
         .addRuleInstance(AggregateExpandDistinctAggregatesRule.INSTANCE)
         .addRuleInstance(AggregateProjectMergeRule.INSTANCE)
@@ -326,6 +326,36 @@ public class RelOptRulesTest extends RelOptTestBase {
             + " from sales.emp group by deptno");
   }
 
+  @Test public void testDistinctCount2() {
+    final HepProgram program = HepProgram.builder()
+        .addRuleInstance(AggregateExpandDistinctAggregatesRule.INSTANCE)
+        .addRuleInstance(AggregateProjectMergeRule.INSTANCE)
+        .build();
+    checkPlanning(program,
+        "select deptno, count(distinct ename), sum(sal)"
+            + " from sales.emp group by deptno");
+  }
+
+  @Test public void testDistinctCountGroupingSets1() {
+    final HepProgram program = HepProgram.builder()
+        .addRuleInstance(AggregateExpandDistinctAggregatesRule.INSTANCE)
+        .addRuleInstance(ProjectMergeRule.INSTANCE)
+        .build();
+    checkPlanning(program,
+        "select deptno, job, count(distinct ename)"
+            + " from sales.emp group by rollup(deptno,job)");
+  }
+
+  @Test public void testDistinctCountGroupingSets2() {
+    final HepProgram program = HepProgram.builder()
+        .addRuleInstance(AggregateExpandDistinctAggregatesRule.INSTANCE)
+        .addRuleInstance(ProjectMergeRule.INSTANCE)
+        .build();
+    checkPlanning(program,
+        "select deptno, job, count(distinct ename), sum(sal)"
+            + " from sales.emp group by rollup(deptno,job)");
+  }
+
   @Test public void testPushProjectPastFilter() {
     checkPlanning(ProjectFilterTransposeRule.INSTANCE,
         "select empno + deptno from emp where sal = 10 * comm "

http://git-wip-us.apache.org/repos/asf/incubator-calcite/blob/24258852/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml
----------------------------------------------------------------------
diff --git a/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml b/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml
index c725828..f80de19 100644
--- a/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml
+++ b/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml
@@ -2734,7 +2734,7 @@ LogicalProject(ENAME=[$0], R=[$1])
 ]]>
         </Resource>
     </TestCase>
-    <TestCase name="testDistinctCount">
+    <TestCase name="testDistinctCount1">
         <Resource name="sql">
             <![CDATA[select deptno, count(distinct ename) from sales.emp group by deptno]]>
         </Resource>
@@ -2753,6 +2753,80 @@ LogicalAggregate(group=[{1}], EXPR$1=[COUNT($0)])
 ]]>
         </Resource>
     </TestCase>
+    <TestCase name="testDistinctCount2">
+        <Resource name="sql">
+            <![CDATA[select deptno, count(distinct ename), sum(sal) from sales.emp group
by deptno]]>
+        </Resource>
+        <Resource name="planBefore">
+            <![CDATA[
+LogicalAggregate(group=[{0}], EXPR$1=[COUNT(DISTINCT $1)], EXPR$2=[SUM($2)])
+  LogicalProject(DEPTNO=[$7], ENAME=[$1], SAL=[$5])
+    LogicalTableScan(table=[[CATALOG, SALES, EMP]])
+]]>
+        </Resource>
+        <Resource name="planAfter">
+            <![CDATA[
+LogicalProject(DEPTNO=[$0], EXPR$1=[$3], EXPR$2=[$1])
+  LogicalJoin(condition=[IS NOT DISTINCT FROM($0, $2)], joinType=[inner])
+    LogicalAggregate(group=[{7}], EXPR$2=[SUM($5)])
+      LogicalTableScan(table=[[CATALOG, SALES, EMP]])
+    LogicalAggregate(group=[{1}], EXPR$1=[COUNT($0)])
+      LogicalAggregate(group=[{1, 7}])
+        LogicalTableScan(table=[[CATALOG, SALES, EMP]])
+]]>
+        </Resource>
+    </TestCase>
+    <TestCase name="testDistinctCountGroupingSets1">
+        <Resource name="sql">
+            <![CDATA[select deptno, job, count(distinct ename) from sales.emp group by
rollup(deptno,job)]]>
+        </Resource>
+        <Resource name="planBefore">
+            <![CDATA[
+LogicalProject(DEPTNO=[$0], JOB=[$1], EXPR$2=[$4])
+  LogicalProject(DEPTNO=[CASE($2, null, $0)], JOB=[CASE($3, null, $1)], i$DEPTNO=[$2], i$JOB=[$3],
EXPR$2=[$4])
+    LogicalAggregate(group=[{0, 1}], groups=[[{0, 1}, {0}, {}]], indicator=[true], EXPR$2=[COUNT(DISTINCT
$2)])
+      LogicalProject(DEPTNO=[$7], JOB=[$2], ENAME=[$1])
+        LogicalTableScan(table=[[CATALOG, SALES, EMP]])
+]]>
+        </Resource>
+        <Resource name="planAfter">
+            <![CDATA[
+LogicalProject(DEPTNO=[CASE($2, null, $0)], JOB=[CASE($3, null, $1)], EXPR$2=[$4])
+  LogicalAggregate(group=[{0, 1}], indicator=[true], EXPR$2=[COUNT($2)])
+    LogicalAggregate(group=[{0, 1, 2}])
+      LogicalProject(DEPTNO=[$0], JOB=[$1], ENAME=[$2])
+        LogicalProject(DEPTNO=[$7], JOB=[$2], ENAME=[$1])
+          LogicalTableScan(table=[[CATALOG, SALES, EMP]])
+]]>
+        </Resource>
+    </TestCase>
+    <TestCase name="testDistinctCountGroupingSets2">
+        <Resource name="sql">
+            <![CDATA[select deptno, job, count(distinct ename), sum(sal) from sales.emp
group by rollup(deptno,job)]]>
+        </Resource>
+        <Resource name="planBefore">
+            <![CDATA[
+LogicalProject(DEPTNO=[$0], JOB=[$1], EXPR$2=[$4], EXPR$3=[$5])
+  LogicalProject(DEPTNO=[CASE($2, null, $0)], JOB=[CASE($3, null, $1)], i$DEPTNO=[$2], i$JOB=[$3],
EXPR$2=[$4], EXPR$3=[$5])
+    LogicalAggregate(group=[{0, 1}], groups=[[{0, 1}, {0}, {}]], indicator=[true], EXPR$2=[COUNT(DISTINCT
$2)], EXPR$3=[SUM($3)])
+      LogicalProject(DEPTNO=[$7], JOB=[$2], ENAME=[$1], SAL=[$5])
+        LogicalTableScan(table=[[CATALOG, SALES, EMP]])
+]]>
+        </Resource>
+        <Resource name="planAfter">
+            <![CDATA[
+LogicalProject(DEPTNO=[CASE($2, null, $0)], JOB=[CASE($3, null, $1)], EXPR$2=[$9], EXPR$3=[$4])
+  LogicalJoin(condition=[AND(IS NOT DISTINCT FROM($0, $5), IS NOT DISTINCT FROM($1, $6),
IS NOT DISTINCT FROM($2, $7), IS NOT DISTINCT FROM($3, $8))], joinType=[inner])
+    LogicalAggregate(group=[{0, 1}], groups=[[{0, 1}, {0}, {}]], indicator=[true], EXPR$3=[SUM($3)])
+      LogicalProject(DEPTNO=[$7], JOB=[$2], ENAME=[$1], SAL=[$5])
+        LogicalTableScan(table=[[CATALOG, SALES, EMP]])
+    LogicalAggregate(group=[{0, 1}], groups=[[{0, 1}, {0}, {}]], indicator=[true], EXPR$2=[COUNT($2)])
+      LogicalAggregate(group=[{0, 1, 2}])
+        LogicalProject(DEPTNO=[$7], JOB=[$2], ENAME=[$1])
+          LogicalTableScan(table=[[CATALOG, SALES, EMP]])
+]]>
+        </Resource>
+    </TestCase>
     <TestCase name="testConvertMultiJoinRuleOuterJoins2">
         <Resource name="sql">
             <![CDATA[select * from A right join B on a = b join C on b = c]]>


Mime
View raw message