Return-Path: X-Original-To: archive-asf-public@cust-asf.ponee.io Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id CF92F160C26 for ; Wed, 3 Jan 2018 03:35:07 +0100 (CET) Received: (qmail 25685 invoked by uid 500); 3 Jan 2018 02:35:07 -0000 Mailing-List: contact commits-help@calcite.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@calcite.apache.org Delivered-To: mailing list commits@calcite.apache.org Received: (qmail 25666 invoked by uid 99); 3 Jan 2018 02:35:06 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 03 Jan 2018 02:35:06 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id BC230DFDF4; Wed, 3 Jan 2018 02:35:01 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: jhyde@apache.org To: commits@calcite.apache.org Date: Wed, 03 Jan 2018 02:35:03 -0000 Message-Id: <6cdf2495b1f24d1d81608e7df0ca24bc@git.apache.org> In-Reply-To: References: X-Mailer: ASF-Git Admin Mailer Subject: [3/3] calcite git commit: [CALCITE-2113] Prune the columns coming from the input of an Aggregate by pushing a Project, in both core and Druid adapter (Nishant Bangarwa) archived-at: Wed, 03 Jan 2018 02:35:08 -0000 [CALCITE-2113] Prune the columns coming from the input of an Aggregate by pushing a Project, in both core and Druid adapter (Nishant Bangarwa) Introduce a rule, DruidAggregateExtractProjectRule that transforms Aggregate(DruidQuery) to Aggregate(Project(DruidQuery)). The Project would fix column pruning to Druid in case HepPlanner is used to plan queries. Also a general-purpose rule AggregateExtractProjectRule. Close apache/calcite#594 Project: http://git-wip-us.apache.org/repos/asf/calcite/repo Commit: http://git-wip-us.apache.org/repos/asf/calcite/commit/d3b35a4b Tree: http://git-wip-us.apache.org/repos/asf/calcite/tree/d3b35a4b Diff: http://git-wip-us.apache.org/repos/asf/calcite/diff/d3b35a4b Branch: refs/heads/master Commit: d3b35a4b5c9270f905175fefa312ab5d8207c1b0 Parents: 1e9fd38 Author: Nishant Authored: Fri Dec 29 23:49:25 2017 +0530 Committer: Julian Hyde Committed: Tue Jan 2 17:52:42 2018 -0800 ---------------------------------------------------------------------- .../rel/rules/AggregateExtractProjectRule.java | 145 +++++++++++++++++++ .../apache/calcite/test/RelOptRulesTest.java | 79 +++++++++- .../org/apache/calcite/test/RelOptRulesTest.xml | 82 ++++++++++- .../calcite/adapter/druid/DruidRules.java | 28 ++++ .../org/apache/calcite/test/DruidAdapterIT.java | 4 +- 5 files changed, 328 insertions(+), 10 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/calcite/blob/d3b35a4b/core/src/main/java/org/apache/calcite/rel/rules/AggregateExtractProjectRule.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/calcite/rel/rules/AggregateExtractProjectRule.java b/core/src/main/java/org/apache/calcite/rel/rules/AggregateExtractProjectRule.java new file mode 100644 index 0000000..5bb88ec --- /dev/null +++ b/core/src/main/java/org/apache/calcite/rel/rules/AggregateExtractProjectRule.java @@ -0,0 +1,145 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.calcite.rel.rules; + +import org.apache.calcite.plan.RelOptRule; +import org.apache.calcite.plan.RelOptRuleCall; +import org.apache.calcite.plan.RelOptRuleOperand; +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.core.Aggregate; +import org.apache.calcite.rel.core.AggregateCall; +import org.apache.calcite.rel.core.Project; +import org.apache.calcite.rex.RexNode; +import org.apache.calcite.tools.RelBuilder; +import org.apache.calcite.tools.RelBuilderFactory; +import org.apache.calcite.util.ImmutableBitSet; +import org.apache.calcite.util.mapping.Mapping; +import org.apache.calcite.util.mapping.MappingType; +import org.apache.calcite.util.mapping.Mappings; + +import com.google.common.base.Function; +import com.google.common.base.Predicate; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Iterables; + +import java.util.ArrayList; +import java.util.List; + +import javax.annotation.Nullable; + +/** + * Rule to extract a {@link org.apache.calcite.rel.core.Project} + * from an {@link org.apache.calcite.rel.core.Aggregate} + * and push it down towards the input. + * + *

What projections can be safely pushed down depends upon which fields the + * Aggregate uses. + * + *

To prevent cycles, this rule will not extract a {@code Project} if the + * {@code Aggregate}s input is already a {@code Project}. + */ +public class AggregateExtractProjectRule extends RelOptRule { + + /** Predicate that prevents matching against an {@code Aggregate} whose input + * is already a {@code Project}. This will prevent this rule firing + * repeatedly. */ + private static final Predicate PREDICATE = new Predicate() { + @Override public boolean apply(@Nullable RelNode relNode) { + return !(relNode instanceof Project); + } + }; + + /** + * Creates an AggregateExtractProjectRule. + * + * @param relBuilderFactory Builder for relational expressions + */ + public AggregateExtractProjectRule( + Class aggregateClass, + Class inputClass, + RelBuilderFactory relBuilderFactory) { + this(operand(aggregateClass, operand(inputClass, null, PREDICATE, any())), + relBuilderFactory); + } + + public AggregateExtractProjectRule(RelOptRuleOperand operand, + RelBuilderFactory builderFactory) { + super(operand, builderFactory, null); + } + + public void onMatch(RelOptRuleCall call) { + final Aggregate aggregate = call.rel(0); + final RelNode input = call.rel(1); + // Compute which input fields are used. + // 1. group fields are always used + final ImmutableBitSet.Builder inputFieldsUsed = + aggregate.getGroupSet().rebuild(); + // 2. agg functions + for (AggregateCall aggCall : aggregate.getAggCallList()) { + for (int i : aggCall.getArgList()) { + inputFieldsUsed.set(i); + } + if (aggCall.filterArg >= 0) { + inputFieldsUsed.set(aggCall.filterArg); + } + } + final RelBuilder relBuilder = call.builder().push(input); + final List projects = new ArrayList<>(); + final Mapping mapping = + Mappings.create(MappingType.INVERSE_SURJECTION, + aggregate.getInput().getRowType().getFieldCount(), + inputFieldsUsed.cardinality()); + int j = 0; + for (int i : inputFieldsUsed.build()) { + projects.add(relBuilder.field(i)); + mapping.set(i, j++); + } + + relBuilder.project(projects); + + final ImmutableBitSet newGroupSet = + Mappings.apply(mapping, aggregate.getGroupSet()); + + final ImmutableList newGroupSets = + ImmutableList.copyOf( + Iterables.transform(aggregate.getGroupSets(), + new Function() { + public ImmutableBitSet apply(ImmutableBitSet input) { + return Mappings.apply(mapping, input); + } + })); + final List newAggCallList = new ArrayList<>(); + for (AggregateCall aggCall : aggregate.getAggCallList()) { + final ImmutableList args = + relBuilder.fields( + Mappings.apply2(mapping, aggCall.getArgList())); + final RexNode filterArg = aggCall.filterArg < 0 ? null + : relBuilder.field(Mappings.apply(mapping, aggCall.filterArg)); + newAggCallList.add( + relBuilder.aggregateCall(aggCall.getAggregation(), + aggCall.isDistinct(), aggCall.isApproximate(), + filterArg, aggCall.name, args)); + } + + final RelBuilder.GroupKey groupKey = + relBuilder.groupKey(newGroupSet, newGroupSets); + relBuilder.aggregate(groupKey, newAggCallList); + call.transformTo(relBuilder.build()); + } +} + +// End AggregateExtractProjectRule.java http://git-wip-us.apache.org/repos/asf/calcite/blob/d3b35a4b/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java b/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java index 057954b..d76babe 100644 --- a/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java +++ b/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java @@ -29,19 +29,23 @@ import org.apache.calcite.prepare.Prepare; import org.apache.calcite.rel.RelCollationTraitDef; import org.apache.calcite.rel.RelNode; import org.apache.calcite.rel.RelRoot; +import org.apache.calcite.rel.core.Aggregate; import org.apache.calcite.rel.core.Intersect; import org.apache.calcite.rel.core.Join; import org.apache.calcite.rel.core.JoinRelType; import org.apache.calcite.rel.core.Minus; +import org.apache.calcite.rel.core.Project; import org.apache.calcite.rel.core.RelFactories; import org.apache.calcite.rel.core.Union; import org.apache.calcite.rel.logical.LogicalProject; import org.apache.calcite.rel.logical.LogicalTableModify; +import org.apache.calcite.rel.logical.LogicalTableScan; import org.apache.calcite.rel.metadata.CachingRelMetadataProvider; import org.apache.calcite.rel.metadata.ChainedRelMetadataProvider; import org.apache.calcite.rel.metadata.DefaultRelMetadataProvider; import org.apache.calcite.rel.metadata.RelMetadataProvider; import org.apache.calcite.rel.rules.AggregateExpandDistinctAggregatesRule; +import org.apache.calcite.rel.rules.AggregateExtractProjectRule; import org.apache.calcite.rel.rules.AggregateFilterTransposeRule; import org.apache.calcite.rel.rules.AggregateJoinTransposeRule; import org.apache.calcite.rel.rules.AggregateProjectMergeRule; @@ -96,12 +100,16 @@ import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.rel.type.RelDataTypeFactory; import org.apache.calcite.rex.RexNode; import org.apache.calcite.runtime.Hook; +import org.apache.calcite.runtime.PredicateImpl; import org.apache.calcite.sql.SqlNode; import org.apache.calcite.sql.type.SqlTypeName; import org.apache.calcite.sql.validate.SqlValidator; import org.apache.calcite.sql2rel.SqlToRelConverter; import org.apache.calcite.tools.RelBuilder; +import static org.apache.calcite.plan.RelOptRule.none; +import static org.apache.calcite.plan.RelOptRule.operand; + import com.google.common.base.Function; import com.google.common.collect.ImmutableList; import com.google.common.collect.Lists; @@ -112,6 +120,8 @@ import org.junit.Test; import java.util.Arrays; import java.util.List; +import javax.annotation.Nullable; + import static org.junit.Assert.assertTrue; /** @@ -2492,7 +2502,7 @@ public class RelOptRulesTest extends RelOptTestBase { .check(); } - @Test public void testAggregateProjectMerge() throws Exception { + @Test public void testAggregateProjectMerge() { HepProgram program = new HepProgramBuilder() .addRuleInstance(AggregateProjectMergeRule.INSTANCE) .build(); @@ -2503,7 +2513,7 @@ public class RelOptRulesTest extends RelOptTestBase { + "group by x, y"); } - @Test public void testAggregateGroupingSetsProjectMerge() throws Exception { + @Test public void testAggregateGroupingSetsProjectMerge() { HepProgram program = new HepProgramBuilder() .addRuleInstance(AggregateProjectMergeRule.INSTANCE) .build(); @@ -2514,6 +2524,71 @@ public class RelOptRulesTest extends RelOptTestBase { + "group by rollup(x, y)"); } + @Test public void testAggregateExtractProjectRule() { + final String sql = "select sum(sal)\n" + + "from emp"; + HepProgram pre = new HepProgramBuilder() + .addRuleInstance(AggregateProjectMergeRule.INSTANCE) + .build(); + final AggregateExtractProjectRule rule = + new AggregateExtractProjectRule(Aggregate.class, LogicalTableScan.class, + RelFactories.LOGICAL_BUILDER); + sql(sql).withPre(pre).withRule(rule).check(); + } + + @Test public void testAggregateExtractProjectRuleWithGroupingSets() { + final String sql = "select empno, deptno, sum(sal)\n" + + "from emp\n" + + "group by grouping sets ((empno, deptno),(deptno),(empno))"; + HepProgram pre = new HepProgramBuilder() + .addRuleInstance(AggregateProjectMergeRule.INSTANCE) + .build(); + final AggregateExtractProjectRule rule = + new AggregateExtractProjectRule(Aggregate.class, LogicalTableScan.class, + RelFactories.LOGICAL_BUILDER); + sql(sql).withPre(pre).withRule(rule).check(); + } + + + /** Test with column used in both grouping set and argument to aggregate + * function. */ + @Test public void testAggregateExtractProjectRuleWithGroupingSets2() { + final String sql = "select empno, deptno, sum(empno)\n" + + "from emp\n" + + "group by grouping sets ((empno, deptno),(deptno),(empno))"; + HepProgram pre = new HepProgramBuilder() + .addRuleInstance(AggregateProjectMergeRule.INSTANCE) + .build(); + final AggregateExtractProjectRule rule = + new AggregateExtractProjectRule(Aggregate.class, LogicalTableScan.class, + RelFactories.LOGICAL_BUILDER); + sql(sql).withPre(pre).withRule(rule).check(); + } + + @Test public void testAggregateExtractProjectRuleWithFilter() { + final String sql = "select sum(sal) filter (where empno = 40)\n" + + "from emp"; + HepProgram pre = new HepProgramBuilder() + .addRuleInstance(AggregateProjectMergeRule.INSTANCE) + .build(); + // AggregateProjectMergeRule does not merges Project with Filter. + // Force match Aggregate on top of Project once explicitly in unit test. + final AggregateExtractProjectRule rule = + new AggregateExtractProjectRule( + operand(Aggregate.class, + operand(Project.class, null, + new PredicateImpl() { + int matchCount = 0; + + public boolean test(@Nullable Project project) { + return matchCount++ == 0; + } + }, + none())), + RelFactories.LOGICAL_BUILDER); + sql(sql).withPre(pre).withRule(rule).checkUnchanged(); + } + @Test public void testPullAggregateThroughUnion() { HepProgram program = new HepProgramBuilder() .addRuleInstance(AggregateUnionAggregateRule.INSTANCE) http://git-wip-us.apache.org/repos/asf/calcite/blob/d3b35a4b/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml ---------------------------------------------------------------------- diff --git a/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml b/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml index bde75fe..17041ef 100644 --- a/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml +++ b/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml @@ -16,6 +16,78 @@ See the License for the specific language governing permissions and limitations under the License. --> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - +join sales.dept as d on e.job = d.name]]> - +join sales.dept as d on e.job = d.name]]> RULES = ImmutableList.of(FILTER, PROJECT_FILTER_TRANSPOSE, AGGREGATE_FILTER_TRANSPOSE, AGGREGATE_PROJECT, + PROJECT_EXTRACT_RULE, PROJECT, POST_AGGREGATION_PROJECT, AGGREGATE, @@ -1270,6 +1274,30 @@ public class DruidRules { relBuilderFactory); } } + + /** + * Rule to extract a {@link org.apache.calcite.rel.core.Project} from + * {@link org.apache.calcite.rel.core.Aggregate} on top of + * {@link org.apache.calcite.adapter.druid.DruidQuery} based on the fields + * used in the aggregate. + */ + public static class DruidAggregateExtractProjectRule + extends AggregateExtractProjectRule { + + /** + * Creates a DruidAggregateExtractProjectRule. + * + * @param relBuilderFactory Builder for relational expressions + */ + public DruidAggregateExtractProjectRule( + RelBuilderFactory relBuilderFactory) { + super( + operand(Aggregate.class, + operand(DruidQuery.class, none())), + relBuilderFactory); + } + } + } // End DruidRules.java http://git-wip-us.apache.org/repos/asf/calcite/blob/d3b35a4b/druid/src/test/java/org/apache/calcite/test/DruidAdapterIT.java ---------------------------------------------------------------------- diff --git a/druid/src/test/java/org/apache/calcite/test/DruidAdapterIT.java b/druid/src/test/java/org/apache/calcite/test/DruidAdapterIT.java index fff466f..6108813 100644 --- a/druid/src/test/java/org/apache/calcite/test/DruidAdapterIT.java +++ b/druid/src/test/java/org/apache/calcite/test/DruidAdapterIT.java @@ -3089,8 +3089,8 @@ public class DruidAdapterIT { + "group by \"B\""; String expectedSubExplain = "PLAN=EnumerableInterpreter\n" + " DruidQuery(table=[[foodmart, foodmart]], " - + "intervals=[[1900-01-09T00:00:00.000Z/2992-01-10T00:00:00.000Z]], projects=[[$63, " - + "$89]], groups=[{0}], aggs=[[COUNT($1)]]"; + + "intervals=[[1900-01-09T00:00:00.000Z/2992-01-10T00:00:00.000Z]], projects=[[$89, " + + "$63]], groups=[{1}], aggs=[[COUNT($0)]]"; testCountWithApproxDistinct(true, sql, expectedSubExplain); testCountWithApproxDistinct(false, sql, expectedSubExplain);