From commits-return-125574-archive-asf-public=cust-asf.ponee.io@ignite.apache.org Tue Dec 10 19:40:36 2019 Return-Path: X-Original-To: archive-asf-public@cust-asf.ponee.io Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [207.244.88.153]) by mx-eu-01.ponee.io (Postfix) with SMTP id C83D4180630 for ; Tue, 10 Dec 2019 20:40:35 +0100 (CET) Received: (qmail 75012 invoked by uid 500); 10 Dec 2019 19:40:35 -0000 Mailing-List: contact commits-help@ignite.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@ignite.apache.org Delivered-To: mailing list commits@ignite.apache.org Received: (qmail 75003 invoked by uid 99); 10 Dec 2019 19:40:35 -0000 Received: from ec2-52-202-80-70.compute-1.amazonaws.com (HELO gitbox.apache.org) (52.202.80.70) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 10 Dec 2019 19:40:35 +0000 Received: by gitbox.apache.org (ASF Mail Server at gitbox.apache.org, from userid 33) id D747C8D809; Tue, 10 Dec 2019 19:40:34 +0000 (UTC) Date: Tue, 10 Dec 2019 19:40:34 +0000 To: "commits@ignite.apache.org" Subject: [ignite] branch ignite-12248 updated: planner rethinking, small refactoring MIME-Version: 1.0 Content-Type: text/plain; charset=utf-8 Content-Transfer-Encoding: 8bit Message-ID: <157600683410.18711.5048959903799082410@gitbox.apache.org> From: gvvinblade@apache.org X-Git-Host: gitbox.apache.org X-Git-Repo: ignite X-Git-Refname: refs/heads/ignite-12248 X-Git-Reftype: branch X-Git-Oldrev: b45e813df3989d5edbf1262678381fdc7caec2c5 X-Git-Newrev: aa3a0d94ab5a2733f1266bf5c9b613dad63d6dd4 X-Git-Rev: aa3a0d94ab5a2733f1266bf5c9b613dad63d6dd4 X-Git-NotificationType: ref_changed_plus_diff X-Git-Multimail-Version: 1.5.dev Auto-Submitted: auto-generated This is an automated email from the ASF dual-hosted git repository. gvvinblade pushed a commit to branch ignite-12248 in repository https://gitbox.apache.org/repos/asf/ignite.git The following commit(s) were added to refs/heads/ignite-12248 by this push: new aa3a0d9 planner rethinking, small refactoring aa3a0d9 is described below commit aa3a0d94ab5a2733f1266bf5c9b613dad63d6dd4 Author: Igor Seliverstov AuthorDate: Tue Dec 10 22:40:21 2019 +0300 planner rethinking, small refactoring --- .../query/calcite/exec/ScalarFactory.java | 6 -- .../query/calcite/prepare/IgnitePlanner.java | 6 +- .../query/calcite/prepare/PlannerPhase.java | 8 +-- .../query/calcite/rel/IgniteConvention.java | 9 ++- .../calcite/rule/AbstractVariableConverter.java | 47 -------------- .../query/calcite/rule/FilterConverter.java | 14 ++-- .../query/calcite/rule/IgniteConverter.java | 74 ++++++++++++++++++++++ .../query/calcite/rule/JoinConverter.java | 14 ++-- .../query/calcite/rule/ProjectConverter.java | 14 ++-- .../query/calcite/rule/TableScanConverter.java | 13 ++-- 10 files changed, 116 insertions(+), 89 deletions(-) diff --git a/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/exec/ScalarFactory.java b/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/exec/ScalarFactory.java index 1d4e31d..e3ff512 100644 --- a/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/exec/ScalarFactory.java +++ b/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/exec/ScalarFactory.java @@ -45,8 +45,6 @@ public class ScalarFactory { } public Predicate filterPredicate(DataContext root, RexNode filter, RelDataType rowType) { - System.out.println("filterPredicate for" + filter); - Scalar scalar = rexCompiler.compile(ImmutableList.of(filter), rowType); Context ctx = InterpreterUtils.createContext(root); @@ -54,8 +52,6 @@ public class ScalarFactory { } public Function projectExpression(DataContext root, List projects, RelDataType rowType) { - System.out.println("joinExpression for" + projects); - Scalar scalar = rexCompiler.compile(projects, rowType); Context ctx = InterpreterUtils.createContext(root); int count = projects.size(); @@ -64,8 +60,6 @@ public class ScalarFactory { } public BiFunction joinExpression(DataContext root, RexNode expression, RelDataType leftType, RelDataType rightType) { - System.out.println("joinExpression for" + expression); - RelDataType rowType = combinedType(leftType, rightType); Scalar scalar = rexCompiler.compile(ImmutableList.of(expression), rowType); diff --git a/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/prepare/IgnitePlanner.java b/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/prepare/IgnitePlanner.java index 5c12ffa..2f35a65 100644 --- a/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/prepare/IgnitePlanner.java +++ b/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/prepare/IgnitePlanner.java @@ -69,7 +69,6 @@ import org.apache.calcite.tools.Planner; import org.apache.calcite.tools.Program; import org.apache.calcite.tools.Programs; import org.apache.calcite.tools.RelBuilder; -import org.apache.calcite.tools.RuleSet; import org.apache.calcite.tools.ValidationException; import org.apache.calcite.util.Pair; import org.apache.ignite.internal.processors.query.calcite.metadata.IgniteMetadata; @@ -322,7 +321,6 @@ public class IgnitePlanner implements Planner, RelOptTable.ViewExpander { ready(); RelTraitSet toTraits = targetTraits.simplify(); - RuleSet rules = plannerPhase.getRules(Commons.plannerContext(context)); input.accept(new MetaDataProviderModifier(metadataProvider)); @@ -332,7 +330,7 @@ public class IgnitePlanner implements Planner, RelOptTable.ViewExpander { case HEP: final HepProgramBuilder programBuilder = new HepProgramBuilder(); - for (RelOptRule rule : rules) { + for (RelOptRule rule : plannerPhase.getRules(Commons.plannerContext(context))) { programBuilder.addRuleInstance(rule); } @@ -348,7 +346,7 @@ public class IgnitePlanner implements Planner, RelOptTable.ViewExpander { break; case VOLCANO: - Program program = Programs.of(rules); + Program program = Programs.of(plannerPhase.getRules(Commons.plannerContext(context))); output = program.run(planner, input, toTraits, ImmutableList.of(), ImmutableList.of()); diff --git a/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/prepare/PlannerPhase.java b/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/prepare/PlannerPhase.java index 906c20e..14ef394 100644 --- a/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/prepare/PlannerPhase.java +++ b/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/prepare/PlannerPhase.java @@ -43,10 +43,10 @@ public enum PlannerPhase { OPTIMIZATION("Main optimization phase") { @Override public RuleSet getRules(PlannerContext ctx) { return RuleSets.ofList( - new TableScanConverter(), - new JoinConverter(), - new ProjectConverter(), - new FilterConverter()); + TableScanConverter.INSTANCE, + JoinConverter.INSTANCE, + ProjectConverter.INSTANCE, + FilterConverter.INSTANCE); } }; diff --git a/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/rel/IgniteConvention.java b/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/rel/IgniteConvention.java index c4fad9f..a0f7878 100644 --- a/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/rel/IgniteConvention.java +++ b/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/rel/IgniteConvention.java @@ -17,6 +17,7 @@ package org.apache.ignite.internal.processors.query.calcite.rel; import org.apache.calcite.plan.Convention; +import org.apache.calcite.plan.ConventionTraitDef; import org.apache.calcite.plan.RelOptPlanner; import org.apache.calcite.plan.RelTraitSet; import org.apache.calcite.plan.volcano.AbstractConverter; @@ -25,17 +26,21 @@ import org.apache.calcite.plan.volcano.AbstractConverter; * */ public class IgniteConvention extends Convention.Impl { - public static final Convention INSTANCE = new IgniteConvention(); + public static final IgniteConvention INSTANCE = new IgniteConvention(); private IgniteConvention() { super("IGNITE", IgniteRel.class); } + @Override public ConventionTraitDef getTraitDef() { + return ConventionTraitDef.INSTANCE; + } + @Override public void register(RelOptPlanner planner) { planner.addRule(AbstractConverter.ExpandConversionRule.INSTANCE); } @Override public boolean useAbstractConvertersForConversion(RelTraitSet fromTraits, RelTraitSet toTraits) { - return fromTraits.contains(INSTANCE) && toTraits.contains(INSTANCE); + return toTraits.contains(INSTANCE) && fromTraits.contains(INSTANCE); } } diff --git a/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/rule/AbstractVariableConverter.java b/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/rule/AbstractVariableConverter.java deleted file mode 100644 index 02c7d53..0000000 --- a/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/rule/AbstractVariableConverter.java +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Copyright 2019 GridGain Systems, Inc. and Contributors. - * - * Licensed under the GridGain Community Edition License (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.gridgain.com/products/software/community-edition/gridgain-community-edition-license - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.ignite.internal.processors.query.calcite.rule; - -import java.util.List; -import org.apache.calcite.plan.RelOptRuleCall; -import org.apache.calcite.plan.RelTrait; -import org.apache.calcite.rel.RelNode; -import org.apache.calcite.rel.convert.ConverterRule; -import org.apache.ignite.internal.util.typedef.F; - -/** - * - */ -public abstract class AbstractVariableConverter extends ConverterRule { - protected AbstractVariableConverter(Class clazz, RelTrait in, RelTrait out, String descriptionPrefix) { - super(clazz, in, out, descriptionPrefix); - } - - @Override public void onMatch(RelOptRuleCall call) { - RelNode rel = call.rel(0); - if (rel.getTraitSet().contains(getInTrait())) { - for (RelNode newRel : convert(rel, false)) - call.transformTo(newRel); - } - } - - @Override public RelNode convert(RelNode rel) { - return F.first(convert(rel, true)); - } - - public abstract List convert(RelNode rel, boolean firstOnly); -} diff --git a/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/rule/FilterConverter.java b/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/rule/FilterConverter.java index 6a6ce47..56587ba 100644 --- a/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/rule/FilterConverter.java +++ b/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/rule/FilterConverter.java @@ -17,10 +17,10 @@ package org.apache.ignite.internal.processors.query.calcite.rule; import java.util.List; -import org.apache.calcite.plan.Convention; import org.apache.calcite.plan.RelOptCluster; import org.apache.calcite.plan.RelTraitSet; import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.convert.ConverterRule; import org.apache.calcite.rel.logical.LogicalFilter; import org.apache.calcite.rel.metadata.RelMetadataQuery; import org.apache.ignite.internal.processors.query.calcite.metadata.IgniteMdDerivedDistribution; @@ -28,17 +28,18 @@ import org.apache.ignite.internal.processors.query.calcite.rel.IgniteConvention; import org.apache.ignite.internal.processors.query.calcite.rel.IgniteFilter; import org.apache.ignite.internal.processors.query.calcite.trait.IgniteDistribution; import org.apache.ignite.internal.processors.query.calcite.util.Commons; -import org.apache.ignite.internal.util.typedef.F; /** * */ -public class FilterConverter extends AbstractVariableConverter { +public class FilterConverter extends IgniteConverter { + public static final ConverterRule INSTANCE = new FilterConverter(); + public FilterConverter() { - super(LogicalFilter.class, Convention.NONE, IgniteConvention.INSTANCE, "FilterConverter"); + super(LogicalFilter.class, "FilterConverter"); } - @Override public List convert(RelNode rel, boolean firstOnly) { + @Override protected List convert0(RelNode rel) { LogicalFilter filter = (LogicalFilter) rel; RelNode input = convert(filter.getInput(), IgniteConvention.INSTANCE); @@ -48,8 +49,7 @@ public class FilterConverter extends AbstractVariableConverter { List distrs = IgniteMdDerivedDistribution.deriveDistributions(input, IgniteConvention.INSTANCE, mq); - return firstOnly ? F.asList(create(filter, input, F.first(distrs))) : - Commons.transform(distrs, d -> create(filter, input, d)); + return Commons.transform(distrs, d -> create(filter, input, d)); } private static IgniteFilter create(LogicalFilter filter, RelNode input, IgniteDistribution distr) { diff --git a/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/rule/IgniteConverter.java b/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/rule/IgniteConverter.java new file mode 100644 index 0000000..4653fec --- /dev/null +++ b/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/rule/IgniteConverter.java @@ -0,0 +1,74 @@ +/* + * Copyright 2019 GridGain Systems, Inc. and Contributors. + * + * Licensed under the GridGain Community Edition License (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.gridgain.com/products/software/community-edition/gridgain-community-edition-license + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.query.calcite.rule; + +import com.google.common.collect.ImmutableMap; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import org.apache.calcite.plan.Convention; +import org.apache.calcite.plan.RelOptPlanner; +import org.apache.calcite.plan.RelOptRuleCall; +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.convert.ConverterRule; +import org.apache.ignite.internal.processors.query.calcite.rel.IgniteConvention; +import org.apache.ignite.internal.util.typedef.F; + +/** + * + */ +public abstract class IgniteConverter extends ConverterRule { + protected IgniteConverter(Class clazz, String descriptionPrefix) { + super(clazz, Convention.NONE, IgniteConvention.INSTANCE, descriptionPrefix); + } + + @Override public void onMatch(RelOptRuleCall call) { + RelNode rel = call.rel(0); + if (rel.getTraitSet().contains(Convention.NONE)) { + List rels = convert0(rel); + if (F.isEmpty(rels)) + return; + + Map equiv = ImmutableMap.of(); + + if (rels.size() > 1) { + equiv = new HashMap<>(); + + for (int i = 1; i < rels.size(); i++) { + equiv.put(rels.get(i), rel); + } + } + + call.transformTo(F.first(rels), equiv); + } + } + + @Override public RelNode convert(RelNode rel) { + List converted = convert0(rel); + + if (converted.size() > 1) { + RelOptPlanner planner = rel.getCluster().getPlanner(); + + for (int i = 1; i < converted.size(); i++) + planner.ensureRegistered(converted.get(i), rel); + } + + return F.first(converted); + } + + protected abstract List convert0(RelNode rel); +} diff --git a/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/rule/JoinConverter.java b/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/rule/JoinConverter.java index 00a2bd9..b10fe20 100644 --- a/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/rule/JoinConverter.java +++ b/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/rule/JoinConverter.java @@ -17,10 +17,10 @@ package org.apache.ignite.internal.processors.query.calcite.rule; import java.util.List; -import org.apache.calcite.plan.Convention; import org.apache.calcite.plan.RelOptCluster; import org.apache.calcite.plan.RelTraitSet; import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.convert.ConverterRule; import org.apache.calcite.rel.logical.LogicalJoin; import org.apache.calcite.rel.metadata.RelMetadataQuery; import org.apache.ignite.internal.processors.query.calcite.metadata.IgniteMdDerivedDistribution; @@ -29,17 +29,18 @@ import org.apache.ignite.internal.processors.query.calcite.rel.IgniteJoin; import org.apache.ignite.internal.processors.query.calcite.trait.IgniteDistribution; import org.apache.ignite.internal.processors.query.calcite.trait.IgniteDistributions; import org.apache.ignite.internal.processors.query.calcite.util.Commons; -import org.apache.ignite.internal.util.typedef.F; /** * */ -public class JoinConverter extends AbstractVariableConverter { +public class JoinConverter extends IgniteConverter { + public static final ConverterRule INSTANCE = new JoinConverter(); + public JoinConverter() { - super(LogicalJoin.class, Convention.NONE, IgniteConvention.INSTANCE, "JoinConverter"); + super(LogicalJoin.class, "JoinConverter"); } - @Override public List convert(RelNode rel, boolean firstOnly) { + @Override protected List convert0(RelNode rel) { LogicalJoin join = (LogicalJoin) rel; RelNode left = convert(join.getLeft(), IgniteConvention.INSTANCE); @@ -53,8 +54,7 @@ public class JoinConverter extends AbstractVariableConverter { List suggestions = IgniteDistributions.suggestJoin(leftTraits, rightTraits, join.analyzeCondition(), join.getJoinType()); - return firstOnly ? F.asList(create(join, left, right, F.first(suggestions))) : - Commons.transform(suggestions, s -> create(join, left, right, s)); + return Commons.transform(suggestions, s -> create(join, left, right, s)); } private static RelNode create(LogicalJoin join, RelNode left, RelNode right, IgniteDistributions.BiSuggestion suggest) { diff --git a/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/rule/ProjectConverter.java b/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/rule/ProjectConverter.java index da281a0..2fe1147 100644 --- a/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/rule/ProjectConverter.java +++ b/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/rule/ProjectConverter.java @@ -17,10 +17,10 @@ package org.apache.ignite.internal.processors.query.calcite.rule; import java.util.List; -import org.apache.calcite.plan.Convention; import org.apache.calcite.plan.RelOptCluster; import org.apache.calcite.plan.RelTraitSet; import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.convert.ConverterRule; import org.apache.calcite.rel.logical.LogicalProject; import org.apache.calcite.rel.metadata.RelMetadataQuery; import org.apache.ignite.internal.processors.query.calcite.metadata.IgniteMdDerivedDistribution; @@ -29,17 +29,18 @@ import org.apache.ignite.internal.processors.query.calcite.rel.IgniteConvention; import org.apache.ignite.internal.processors.query.calcite.rel.IgniteProject; import org.apache.ignite.internal.processors.query.calcite.trait.IgniteDistribution; import org.apache.ignite.internal.processors.query.calcite.util.Commons; -import org.apache.ignite.internal.util.typedef.F; /** * */ -public class ProjectConverter extends AbstractVariableConverter { +public class ProjectConverter extends IgniteConverter { + public static final ConverterRule INSTANCE = new ProjectConverter(); + public ProjectConverter() { - super(LogicalProject.class, Convention.NONE, IgniteConvention.INSTANCE, "ProjectConverter"); + super(LogicalProject.class, "ProjectConverter"); } - @Override public List convert(RelNode rel, boolean firstOnly) { + @Override protected List convert0(RelNode rel) { LogicalProject project = (LogicalProject) rel; RelNode input = convert(project.getInput(), IgniteConvention.INSTANCE); @@ -49,8 +50,7 @@ public class ProjectConverter extends AbstractVariableConverter { List distrs = IgniteMdDerivedDistribution.deriveDistributions(input, IgniteConvention.INSTANCE, mq); - return firstOnly ? F.asList(create(project, input, F.first(distrs))) : - Commons.transform(distrs, d -> create(project, input, d)); + return Commons.transform(distrs, d -> create(project, input, d)); } private static IgniteProject create(LogicalProject project, RelNode input, IgniteDistribution distr) { diff --git a/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/rule/TableScanConverter.java b/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/rule/TableScanConverter.java index 7665a78..2d5aea0 100644 --- a/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/rule/TableScanConverter.java +++ b/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/rule/TableScanConverter.java @@ -16,26 +16,29 @@ package org.apache.ignite.internal.processors.query.calcite.rule; -import org.apache.calcite.plan.Convention; +import java.util.List; import org.apache.calcite.plan.RelTraitSet; import org.apache.calcite.rel.RelNode; import org.apache.calcite.rel.convert.ConverterRule; import org.apache.calcite.rel.logical.LogicalTableScan; import org.apache.ignite.internal.processors.query.calcite.rel.IgniteConvention; import org.apache.ignite.internal.processors.query.calcite.rel.IgniteTableScan; +import org.apache.ignite.internal.util.typedef.F; /** * */ -public class TableScanConverter extends ConverterRule { +public class TableScanConverter extends IgniteConverter { + public static final ConverterRule INSTANCE = new TableScanConverter(); + public TableScanConverter() { - super(LogicalTableScan.class, Convention.NONE, IgniteConvention.INSTANCE, "TableScanConverter"); + super(LogicalTableScan.class, "TableScanConverter"); } - @Override public RelNode convert(RelNode rel) { + @Override protected List convert0(RelNode rel) { LogicalTableScan scan = (LogicalTableScan) rel; RelTraitSet traitSet = scan.getTraitSet().replace(IgniteConvention.INSTANCE); - return new IgniteTableScan(rel.getCluster(), traitSet, scan.getTable()); + return F.asList(new IgniteTableScan(rel.getCluster(), traitSet, scan.getTable())); } }