Return-Path: X-Original-To: apmail-tajo-commits-archive@minotaur.apache.org Delivered-To: apmail-tajo-commits-archive@minotaur.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 38A12185FF for ; Thu, 13 Aug 2015 13:35:38 +0000 (UTC) Received: (qmail 49141 invoked by uid 500); 13 Aug 2015 13:35:32 -0000 Delivered-To: apmail-tajo-commits-archive@tajo.apache.org Received: (qmail 49113 invoked by uid 500); 13 Aug 2015 13:35:31 -0000 Mailing-List: contact commits-help@tajo.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@tajo.apache.org Delivered-To: mailing list commits@tajo.apache.org Received: (qmail 49100 invoked by uid 99); 13 Aug 2015 13:35:31 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Thu, 13 Aug 2015 13:35:31 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id BB0BFE6838; Thu, 13 Aug 2015 13:35:31 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: jihoonson@apache.org To: commits@tajo.apache.org Message-Id: <1b8a863c2276402cadec9d6f2a73c646@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: tajo git commit: TAJO-1561: Query which contains join condition in "OR" clause does not finish. Date: Thu, 13 Aug 2015 13:35:31 +0000 (UTC) Repository: tajo Updated Branches: refs/heads/master 9114485b5 -> f0ab0ca20 TAJO-1561: Query which contains join condition in "OR" clause does not finish. Closes #685 Project: http://git-wip-us.apache.org/repos/asf/tajo/repo Commit: http://git-wip-us.apache.org/repos/asf/tajo/commit/f0ab0ca2 Tree: http://git-wip-us.apache.org/repos/asf/tajo/tree/f0ab0ca2 Diff: http://git-wip-us.apache.org/repos/asf/tajo/diff/f0ab0ca2 Branch: refs/heads/master Commit: f0ab0ca208a341625bc0334b061d63c6e0960194 Parents: 9114485 Author: Jihoon Son Authored: Thu Aug 13 22:35:04 2015 +0900 Committer: Jihoon Son Committed: Thu Aug 13 22:35:04 2015 +0900 ---------------------------------------------------------------------- CHANGES | 3 + .../tajo/engine/query/TestSelectQuery.java | 27 +++ .../testSelectWithCommonQuals1.1.plan | 7 + .../testSelectWithCommonQuals1.1.result | 27 +++ .../testSelectWithCommonQuals2.1.plan | 7 + .../testSelectWithCommonQuals2.1.result | 27 +++ .../testSelectWithCommonQuals3.1.plan | 7 + .../testSelectWithCommonQuals3.1.result | 7 + .../tajo/plan/expr/BasicEvalNodeVisitor.java | 23 +- .../apache/tajo/plan/expr/EvalNodeVisitor2.java | 2 +- .../org/apache/tajo/plan/expr/EvalTreeUtil.java | 18 +- .../BaseLogicalPlanRewriteRuleProvider.java | 7 +- .../rules/CommonConditionReduceRule.java | 221 +++++++++++++++++++ .../plan/rewrite/rules/FilterPushDownRule.java | 15 +- .../org/apache/tajo/plan/util/IndexUtil.java | 2 +- .../tajo/plan/verifier/ExprsVerifier.java | 5 +- 16 files changed, 366 insertions(+), 39 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tajo/blob/f0ab0ca2/CHANGES ---------------------------------------------------------------------- diff --git a/CHANGES b/CHANGES index 3498cc3..66d9bee 100644 --- a/CHANGES +++ b/CHANGES @@ -216,6 +216,9 @@ Release 0.11.0 - unreleased BUG FIXES + TAJO-1561: Query which contains join condition in "OR" clause does not finish. + (jihoon) + TAJO-1752: NameResolver cannot find nested records properly. (jihoon) TAJO-1732: During filter push down phase, join conditions are not set properly. http://git-wip-us.apache.org/repos/asf/tajo/blob/f0ab0ca2/tajo-core/src/test/java/org/apache/tajo/engine/query/TestSelectQuery.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestSelectQuery.java b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestSelectQuery.java index 1351505..27c2fcd 100644 --- a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestSelectQuery.java +++ b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestSelectQuery.java @@ -748,4 +748,31 @@ public class TestSelectQuery extends QueryTestCaseBase { assertResultSet(res); cleanupQuery(res); } + + @Test + @Option(withExplain = true) + @SimpleTest ( + queries = @QuerySpec("select * from nation where (n_regionkey = 1 or n_name is not null) and (n_regionkey = 1 or n_comment is not null)") + ) + public void testSelectWithCommonQuals1() throws Exception { + runSimpleTests(); + } + + @Test + @Option(withExplain = true) + @SimpleTest( + queries = @QuerySpec("select * from nation where (n_regionkey = 1 or n_name is not null) and (n_regionkey = 1 or n_name is not null)") + ) + public void testSelectWithCommonQuals2() throws Exception { + runSimpleTests(); + } + + @Test + @Option(withExplain = true) + @SimpleTest( + queries = @QuerySpec("select * from nation where (n_regionkey = 1 and n_name is not null) or (n_regionkey = 1 and n_comment is not null)") + ) + public void testSelectWithCommonQuals3() throws Exception { + runSimpleTests(); + } } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/tajo/blob/f0ab0ca2/tajo-core/src/test/resources/results/TestSelectQuery/testSelectWithCommonQuals1.1.plan ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/resources/results/TestSelectQuery/testSelectWithCommonQuals1.1.plan b/tajo-core/src/test/resources/results/TestSelectQuery/testSelectWithCommonQuals1.1.plan new file mode 100644 index 0000000..d49b01e --- /dev/null +++ b/tajo-core/src/test/resources/results/TestSelectQuery/testSelectWithCommonQuals1.1.plan @@ -0,0 +1,7 @@ +explain +------------------------------- +SCAN(0) on default.nation + => filter: default.nation.n_regionkey (INT4) = 1 OR default.nation.n_name (TEXT) IS NOT NULL AND default.nation.n_comment (TEXT) IS NOT NULL + => target list: default.nation.n_comment (TEXT), default.nation.n_name (TEXT), default.nation.n_nationkey (INT4), default.nation.n_regionkey (INT4) + => out schema: {(4) default.nation.n_comment (TEXT), default.nation.n_name (TEXT), default.nation.n_nationkey (INT4), default.nation.n_regionkey (INT4)} + => in schema: {(4) default.nation.n_comment (TEXT), default.nation.n_name (TEXT), default.nation.n_nationkey (INT4), default.nation.n_regionkey (INT4)} http://git-wip-us.apache.org/repos/asf/tajo/blob/f0ab0ca2/tajo-core/src/test/resources/results/TestSelectQuery/testSelectWithCommonQuals1.1.result ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/resources/results/TestSelectQuery/testSelectWithCommonQuals1.1.result b/tajo-core/src/test/resources/results/TestSelectQuery/testSelectWithCommonQuals1.1.result new file mode 100644 index 0000000..91f0969 --- /dev/null +++ b/tajo-core/src/test/resources/results/TestSelectQuery/testSelectWithCommonQuals1.1.result @@ -0,0 +1,27 @@ +n_nationkey,n_name,n_regionkey,n_comment +------------------------------- +0,ALGERIA,0, haggle. carefully final deposits detect slyly agai +1,ARGENTINA,1,al foxes promise slyly according to the regular accounts. bold requests alon +2,BRAZIL,1,y alongside of the pending deposits. carefully special packages are about the ironic forges. slyly special +3,CANADA,1,eas hang ironic, silent packages. slyly regular packages are furiously over the tithes. fluffily bold +4,EGYPT,4,y above the carefully unusual theodolites. final dugouts are quickly across the furiously regular d +5,ETHIOPIA,0,ven packages wake quickly. regu +6,FRANCE,3,refully final requests. regular, ironi +7,GERMANY,3,l platelets. regular accounts x-ray: unusual, regular acco +8,INDIA,2,ss excuses cajole slyly across the packages. deposits print aroun +9,INDONESIA,2, slyly express asymptotes. regular deposits haggle slyly. carefully ironic hockey players sleep blithely. carefull +10,IRAN,4,efully alongside of the slyly final dependencies. +11,IRAQ,4,nic deposits boost atop the quickly final requests? quickly regula +12,JAPAN,2,ously. final, express gifts cajole a +13,JORDAN,4,ic deposits are blithely about the carefully regular pa +14,KENYA,0, pending excuses haggle furiously deposits. pending, express pinto beans wake fluffily past t +15,MOROCCO,0,rns. blithely bold courts among the closely regular packages use furiously bold platelets? +16,MOZAMBIQUE,0,s. ironic, unusual asymptotes wake blithely r +17,PERU,1,platelets. blithely pending dependencies use fluffily across the even pinto beans. carefully silent accoun +18,CHINA,2,c dependencies. furiously express notornis sleep slyly regular accounts. ideas sleep. depos +19,ROMANIA,3,ular asymptotes are about the furious multipliers. express dependencies nag above the ironically ironic account +20,SAUDI ARABIA,4,ts. silent requests haggle. closely express packages sleep across the blithely +21,VIETNAM,2,hely enticingly express accounts. even, final +22,RUSSIA,3, requests against the platelets use never according to the quickly regular pint +23,UNITED KINGDOM,3,eans boost carefully special requests. accounts are. carefull +24,UNITED STATES,1,y final packages. slow foxes cajole quickly. quickly silent platelets breach ironic accounts. unusual pinto be http://git-wip-us.apache.org/repos/asf/tajo/blob/f0ab0ca2/tajo-core/src/test/resources/results/TestSelectQuery/testSelectWithCommonQuals2.1.plan ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/resources/results/TestSelectQuery/testSelectWithCommonQuals2.1.plan b/tajo-core/src/test/resources/results/TestSelectQuery/testSelectWithCommonQuals2.1.plan new file mode 100644 index 0000000..fa47fda --- /dev/null +++ b/tajo-core/src/test/resources/results/TestSelectQuery/testSelectWithCommonQuals2.1.plan @@ -0,0 +1,7 @@ +explain +------------------------------- +SCAN(0) on default.nation + => filter: default.nation.n_regionkey (INT4) = 1 OR default.nation.n_name (TEXT) IS NOT NULL + => target list: default.nation.n_comment (TEXT), default.nation.n_name (TEXT), default.nation.n_nationkey (INT4), default.nation.n_regionkey (INT4) + => out schema: {(4) default.nation.n_comment (TEXT), default.nation.n_name (TEXT), default.nation.n_nationkey (INT4), default.nation.n_regionkey (INT4)} + => in schema: {(4) default.nation.n_comment (TEXT), default.nation.n_name (TEXT), default.nation.n_nationkey (INT4), default.nation.n_regionkey (INT4)} http://git-wip-us.apache.org/repos/asf/tajo/blob/f0ab0ca2/tajo-core/src/test/resources/results/TestSelectQuery/testSelectWithCommonQuals2.1.result ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/resources/results/TestSelectQuery/testSelectWithCommonQuals2.1.result b/tajo-core/src/test/resources/results/TestSelectQuery/testSelectWithCommonQuals2.1.result new file mode 100644 index 0000000..91f0969 --- /dev/null +++ b/tajo-core/src/test/resources/results/TestSelectQuery/testSelectWithCommonQuals2.1.result @@ -0,0 +1,27 @@ +n_nationkey,n_name,n_regionkey,n_comment +------------------------------- +0,ALGERIA,0, haggle. carefully final deposits detect slyly agai +1,ARGENTINA,1,al foxes promise slyly according to the regular accounts. bold requests alon +2,BRAZIL,1,y alongside of the pending deposits. carefully special packages are about the ironic forges. slyly special +3,CANADA,1,eas hang ironic, silent packages. slyly regular packages are furiously over the tithes. fluffily bold +4,EGYPT,4,y above the carefully unusual theodolites. final dugouts are quickly across the furiously regular d +5,ETHIOPIA,0,ven packages wake quickly. regu +6,FRANCE,3,refully final requests. regular, ironi +7,GERMANY,3,l platelets. regular accounts x-ray: unusual, regular acco +8,INDIA,2,ss excuses cajole slyly across the packages. deposits print aroun +9,INDONESIA,2, slyly express asymptotes. regular deposits haggle slyly. carefully ironic hockey players sleep blithely. carefull +10,IRAN,4,efully alongside of the slyly final dependencies. +11,IRAQ,4,nic deposits boost atop the quickly final requests? quickly regula +12,JAPAN,2,ously. final, express gifts cajole a +13,JORDAN,4,ic deposits are blithely about the carefully regular pa +14,KENYA,0, pending excuses haggle furiously deposits. pending, express pinto beans wake fluffily past t +15,MOROCCO,0,rns. blithely bold courts among the closely regular packages use furiously bold platelets? +16,MOZAMBIQUE,0,s. ironic, unusual asymptotes wake blithely r +17,PERU,1,platelets. blithely pending dependencies use fluffily across the even pinto beans. carefully silent accoun +18,CHINA,2,c dependencies. furiously express notornis sleep slyly regular accounts. ideas sleep. depos +19,ROMANIA,3,ular asymptotes are about the furious multipliers. express dependencies nag above the ironically ironic account +20,SAUDI ARABIA,4,ts. silent requests haggle. closely express packages sleep across the blithely +21,VIETNAM,2,hely enticingly express accounts. even, final +22,RUSSIA,3, requests against the platelets use never according to the quickly regular pint +23,UNITED KINGDOM,3,eans boost carefully special requests. accounts are. carefull +24,UNITED STATES,1,y final packages. slow foxes cajole quickly. quickly silent platelets breach ironic accounts. unusual pinto be http://git-wip-us.apache.org/repos/asf/tajo/blob/f0ab0ca2/tajo-core/src/test/resources/results/TestSelectQuery/testSelectWithCommonQuals3.1.plan ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/resources/results/TestSelectQuery/testSelectWithCommonQuals3.1.plan b/tajo-core/src/test/resources/results/TestSelectQuery/testSelectWithCommonQuals3.1.plan new file mode 100644 index 0000000..90adfc4 --- /dev/null +++ b/tajo-core/src/test/resources/results/TestSelectQuery/testSelectWithCommonQuals3.1.plan @@ -0,0 +1,7 @@ +explain +------------------------------- +SCAN(0) on default.nation + => filter: default.nation.n_regionkey (INT4) = 1 AND default.nation.n_name (TEXT) IS NOT NULL OR default.nation.n_comment (TEXT) IS NOT NULL + => target list: default.nation.n_comment (TEXT), default.nation.n_name (TEXT), default.nation.n_nationkey (INT4), default.nation.n_regionkey (INT4) + => out schema: {(4) default.nation.n_comment (TEXT), default.nation.n_name (TEXT), default.nation.n_nationkey (INT4), default.nation.n_regionkey (INT4)} + => in schema: {(4) default.nation.n_comment (TEXT), default.nation.n_name (TEXT), default.nation.n_nationkey (INT4), default.nation.n_regionkey (INT4)} http://git-wip-us.apache.org/repos/asf/tajo/blob/f0ab0ca2/tajo-core/src/test/resources/results/TestSelectQuery/testSelectWithCommonQuals3.1.result ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/resources/results/TestSelectQuery/testSelectWithCommonQuals3.1.result b/tajo-core/src/test/resources/results/TestSelectQuery/testSelectWithCommonQuals3.1.result new file mode 100644 index 0000000..85bf5bb --- /dev/null +++ b/tajo-core/src/test/resources/results/TestSelectQuery/testSelectWithCommonQuals3.1.result @@ -0,0 +1,7 @@ +n_nationkey,n_name,n_regionkey,n_comment +------------------------------- +1,ARGENTINA,1,al foxes promise slyly according to the regular accounts. bold requests alon +2,BRAZIL,1,y alongside of the pending deposits. carefully special packages are about the ironic forges. slyly special +3,CANADA,1,eas hang ironic, silent packages. slyly regular packages are furiously over the tithes. fluffily bold +17,PERU,1,platelets. blithely pending dependencies use fluffily across the even pinto beans. carefully silent accoun +24,UNITED STATES,1,y final packages. slow foxes cajole quickly. quickly silent platelets breach ironic accounts. unusual pinto be http://git-wip-us.apache.org/repos/asf/tajo/blob/f0ab0ca2/tajo-plan/src/main/java/org/apache/tajo/plan/expr/BasicEvalNodeVisitor.java ---------------------------------------------------------------------- diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/expr/BasicEvalNodeVisitor.java b/tajo-plan/src/main/java/org/apache/tajo/plan/expr/BasicEvalNodeVisitor.java index 128cc7c..59be24a 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/expr/BasicEvalNodeVisitor.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/expr/BasicEvalNodeVisitor.java @@ -18,7 +18,6 @@ package org.apache.tajo.plan.expr; -import org.apache.tajo.exception.TajoInternalError; import org.apache.tajo.exception.UnsupportedException; import java.util.Stack; @@ -26,7 +25,7 @@ import java.util.Stack; public class BasicEvalNodeVisitor implements EvalNodeVisitor2 { @Override - public RESULT visitChild(CONTEXT context, EvalNode evalNode, Stack stack) { + public RESULT visit(CONTEXT context, EvalNode evalNode, Stack stack) { RESULT result; switch (evalNode.getType()) { // Column and Value reference expressions @@ -146,15 +145,15 @@ public class BasicEvalNodeVisitor implements EvalNodeVisitor2 stack) { stack.push(unaryEval); - RESULT result = visitChild(context, unaryEval.getChild(), stack); + RESULT result = visit(context, unaryEval.getChild(), stack); stack.pop(); return result; } private RESULT visitDefaultBinaryEval(CONTEXT context, BinaryEval binaryEval, Stack stack) { stack.push(binaryEval); - RESULT result = visitChild(context, binaryEval.getLeftExpr(), stack); - visitChild(context, binaryEval.getRightExpr(), stack); + RESULT result = visit(context, binaryEval.getLeftExpr(), stack); + visit(context, binaryEval.getRightExpr(), stack); stack.pop(); return result; } @@ -164,7 +163,7 @@ public class BasicEvalNodeVisitor implements EvalNodeVisitor2 implements EvalNodeVisitor2 stack) { stack.push(evalNode); - RESULT result = visitChild(context, evalNode.getPredicand(), stack); - visitChild(context, evalNode.getBegin(), stack); - visitChild(context, evalNode.getEnd(), stack); + RESULT result = visit(context, evalNode.getPredicand(), stack); + visit(context, evalNode.getBegin(), stack); + visit(context, evalNode.getEnd(), stack); return result; } @@ -278,7 +277,7 @@ public class BasicEvalNodeVisitor implements EvalNodeVisitor2 implements EvalNodeVisitor2 stack) { RESULT result; stack.push(evalNode); - result = visitChild(context, evalNode.getCondition(), stack); - visitChild(context, evalNode.getResult(), stack); + result = visit(context, evalNode.getCondition(), stack); + visit(context, evalNode.getResult(), stack); stack.pop(); return result; } http://git-wip-us.apache.org/repos/asf/tajo/blob/f0ab0ca2/tajo-plan/src/main/java/org/apache/tajo/plan/expr/EvalNodeVisitor2.java ---------------------------------------------------------------------- diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/expr/EvalNodeVisitor2.java b/tajo-plan/src/main/java/org/apache/tajo/plan/expr/EvalNodeVisitor2.java index bae193a..43729ac 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/expr/EvalNodeVisitor2.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/expr/EvalNodeVisitor2.java @@ -21,7 +21,7 @@ package org.apache.tajo.plan.expr; import java.util.Stack; public interface EvalNodeVisitor2 { - RESULT visitChild(CONTEXT context, EvalNode evalNode, Stack stack); + RESULT visit(CONTEXT context, EvalNode evalNode, Stack stack); // Column and Value reference expressions RESULT visitConst(CONTEXT context, ConstEval evalNode, Stack stack); http://git-wip-us.apache.org/repos/asf/tajo/blob/f0ab0ca2/tajo-plan/src/main/java/org/apache/tajo/plan/expr/EvalTreeUtil.java ---------------------------------------------------------------------- diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/expr/EvalTreeUtil.java b/tajo-plan/src/main/java/org/apache/tajo/plan/expr/EvalTreeUtil.java index 51534e1..89fd81b 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/expr/EvalTreeUtil.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/expr/EvalTreeUtil.java @@ -46,7 +46,7 @@ public class EvalTreeUtil { public static int replace(EvalNode expr, EvalNode targetExpr, EvalNode tobeReplaced) { EvalReplaceVisitor replacer = new EvalReplaceVisitor(targetExpr, tobeReplaced); ReplaceContext context = new ReplaceContext(); - replacer.visitChild(context, expr, new Stack()); + replacer.visit(context, expr, new Stack()); return context.countOfReplaces; } @@ -64,8 +64,8 @@ public class EvalTreeUtil { } @Override - public EvalNode visitChild(ReplaceContext context, EvalNode evalNode, Stack stack) { - super.visitChild(context, evalNode, stack); + public EvalNode visit(ReplaceContext context, EvalNode evalNode, Stack stack) { + super.visit(context, evalNode, stack); if (evalNode.equals(target)) { context.countOfReplaces++; @@ -511,13 +511,13 @@ public class EvalTreeUtil { public static Collection findEvalsByType(EvalNode evalNode, EvalType type) { EvalFinder finder = new EvalFinder(type); - finder.visitChild(null, evalNode, new Stack()); + finder.visit(null, evalNode, new Stack()); return (Collection) finder.evalNodes; } public static Collection findOuterJoinSensitiveEvals(EvalNode evalNode) { OuterJoinSensitiveEvalFinder finder = new OuterJoinSensitiveEvalFinder(); - finder.visitChild(null, evalNode, new Stack()); + finder.visit(null, evalNode, new Stack()); return (Collection) finder.evalNodes; } @@ -530,8 +530,8 @@ public class EvalTreeUtil { } @Override - public Object visitChild(Object context, EvalNode evalNode, Stack stack) { - super.visitChild(context, evalNode, stack); + public Object visit(Object context, EvalNode evalNode, Stack stack) { + super.visit(context, evalNode, stack); if (evalNode.type == targetType) { evalNodes.add(evalNode); @@ -549,8 +549,8 @@ public class EvalTreeUtil { private List evalNodes = TUtil.newList(); @Override - public Object visitChild(Object context, EvalNode evalNode, Stack stack) { - super.visitChild(context, evalNode, stack); + public Object visit(Object context, EvalNode evalNode, Stack stack) { + super.visit(context, evalNode, stack); if (evalNode.type == EvalType.CASE) { evalNodes.add(evalNode); http://git-wip-us.apache.org/repos/asf/tajo/blob/f0ab0ca2/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/BaseLogicalPlanRewriteRuleProvider.java ---------------------------------------------------------------------- diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/BaseLogicalPlanRewriteRuleProvider.java b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/BaseLogicalPlanRewriteRuleProvider.java index dcfd6bf..f73aa54 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/BaseLogicalPlanRewriteRuleProvider.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/BaseLogicalPlanRewriteRuleProvider.java @@ -19,10 +19,7 @@ package org.apache.tajo.plan.rewrite; import org.apache.tajo.conf.TajoConf; -import org.apache.tajo.plan.rewrite.rules.AccessPathRewriter; -import org.apache.tajo.plan.rewrite.rules.FilterPushDownRule; -import org.apache.tajo.plan.rewrite.rules.PartitionedTableRewriter; -import org.apache.tajo.plan.rewrite.rules.ProjectionPushDownRule; +import org.apache.tajo.plan.rewrite.rules.*; import org.apache.tajo.util.TUtil; import java.util.Collection; @@ -42,6 +39,8 @@ public class BaseLogicalPlanRewriteRuleProvider extends LogicalPlanRewriteRulePr public Collection> getPreRules() { List> rules = TUtil.newList(); + rules.add(CommonConditionReduceRule.class); + if (systemConf.getBoolVar(TajoConf.ConfVars.$TEST_FILTER_PUSHDOWN_ENABLED)) { rules.add(FilterPushDownRule.class); } http://git-wip-us.apache.org/repos/asf/tajo/blob/f0ab0ca2/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/CommonConditionReduceRule.java ---------------------------------------------------------------------- diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/CommonConditionReduceRule.java b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/CommonConditionReduceRule.java new file mode 100644 index 0000000..ac4ef81 --- /dev/null +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/CommonConditionReduceRule.java @@ -0,0 +1,221 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.plan.rewrite.rules; + +import org.apache.tajo.exception.TajoException; +import org.apache.tajo.plan.LogicalPlan; +import org.apache.tajo.plan.expr.*; +import org.apache.tajo.plan.logical.JoinNode; +import org.apache.tajo.plan.logical.LogicalNode; +import org.apache.tajo.plan.logical.NodeType; +import org.apache.tajo.plan.logical.SelectionNode; +import org.apache.tajo.plan.rewrite.LogicalPlanRewriteRule; +import org.apache.tajo.plan.rewrite.LogicalPlanRewriteRuleContext; +import org.apache.tajo.plan.visitor.BasicLogicalPlanVisitor; +import org.apache.tajo.util.TUtil; + +import java.util.Set; +import java.util.Stack; + +/** + * Condition reduce rule reduces the query predicate based on distributivity. + * For example, the query + * + * SELECT * + * FROM t + * WHERE (t.a = 1 OR t.b = 10) AND (t.a = 1 OR t.c = 100) + * + * is converted into + * + * SELECT * + * FROM t + * WHERE t.a = 1 OR (t.b = 10 AND t.c = 100). + * + */ +public class CommonConditionReduceRule implements LogicalPlanRewriteRule { + private Rewriter rewriter; + + @Override + public String getName() { + return "CommonConditionReduceRule"; + } + + @Override + public boolean isEligible(LogicalPlanRewriteRuleContext context) { + for (LogicalPlan.QueryBlock block : context.getPlan().getQueryBlocks()) { + if (block.hasNode(NodeType.SELECTION) || block.hasNode(NodeType.JOIN)) { + rewriter = new Rewriter(context.getPlan()); + return true; + } + } + return false; + } + + @Override + public LogicalPlan rewrite(LogicalPlanRewriteRuleContext context) throws TajoException { + rewriter.visit(null, context.getPlan(), context.getPlan().getRootBlock()); + return context.getPlan(); + } + + /** + * Rewriter simply triggers rewriting evals while visiting logical nodes. + */ + private final static class Rewriter extends BasicLogicalPlanVisitor { + EvalRewriter evalRewriter; + + public Rewriter(LogicalPlan plan) { + evalRewriter = new EvalRewriter(plan); + } + + @Override + public LogicalNode visitFilter(Object context, LogicalPlan plan, LogicalPlan.QueryBlock block, + SelectionNode selNode, Stack stack) throws TajoException { + selNode.setQual(evalRewriter.visit(null, selNode.getQual(), new Stack())); + return null; + } + + @Override + public LogicalNode visitJoin(Object context, LogicalPlan plan, LogicalPlan.QueryBlock block, + JoinNode joinNode, Stack stack) throws TajoException { + if (joinNode.hasJoinQual()) { + joinNode.setJoinQual(evalRewriter.visit(null, joinNode.getJoinQual(), new Stack())); + } + return null; + } + } + + /** + * EvalRewriter is responsible for rewriting evals based on distributivity. + */ + private static class EvalRewriter extends SimpleEvalNodeVisitor { + final LogicalPlan plan; + + public EvalRewriter(LogicalPlan plan) { + this.plan = plan; + } + + @Override + protected EvalNode visitUnaryEval(Object context, Stack stack, UnaryEval unaryEval) { + stack.push(unaryEval); + EvalNode child = unaryEval.getChild(); + visit(context, child, stack); + if (child.getType() == EvalType.AND || child.getType() == EvalType.OR) { + unaryEval.setChild(rewrite((BinaryEval) child)); + } + stack.pop(); + return unaryEval; + } + + @Override + protected EvalNode visitBinaryEval(Object context, Stack stack, BinaryEval binaryEval) { + stack.push(binaryEval); + EvalNode child = binaryEval.getLeftExpr(); + visit(context, child, stack); + if (child.getType() == EvalType.AND || child.getType() == EvalType.OR) { + binaryEval.setLeftExpr(rewrite((BinaryEval) child)); + } + + child = binaryEval.getRightExpr(); + visit(context, child, stack); + if (child.getType() == EvalType.AND || child.getType() == EvalType.OR) { + binaryEval.setRightExpr(rewrite((BinaryEval) child)); + } + + EvalNode result = rewrite(binaryEval); + + stack.pop(); + return result; + } + + @Override + protected EvalNode visitDefaultFunctionEval(Object context, Stack stack, FunctionEval functionEval) { + stack.push(functionEval); + if (functionEval.getArgs() != null) { + EvalNode [] args = functionEval.getArgs(); + for (int i = 0; i < args.length; i++) { + visit(context, args[i], stack); + if (args[i].getType() == EvalType.AND || args[i].getType() == EvalType.OR) { + functionEval.setArg(i, rewrite((BinaryEval) args[i])); + } + } + } + stack.pop(); + return functionEval; + } + + private EvalNode rewrite(BinaryEval evalNode) { + // Example qual: ( a | b ) ^ ( a | c ) + EvalType outerType = evalNode.getType(); // type of the outer operation. ex) ^ + EvalType innerType = outerType == EvalType.AND ? EvalType.OR : EvalType.AND; // type of the inner operation. ex) | + + EvalNode finalQual; + + if (evalNode.getLeftExpr().getType() == innerType && + evalNode.getRightExpr().getType() == innerType) { + BinaryEval leftChild = evalNode.getLeftExpr(); + BinaryEval rightChild = evalNode.getRightExpr(); + + // Find common quals from the left and right children. + Set commonQuals = TUtil.newHashSet(); + Set childrenOfLeft = TUtil.newHashSet(leftChild.getLeftExpr(), leftChild.getRightExpr()); + for (int i = 0; i < 2; i++) { + if (childrenOfLeft.contains(rightChild.getChild(i))) { + commonQuals.add(rightChild.getChild(i)); + } + } + + if (commonQuals.size() == 2) { + // Ex) ( a | b ) ^ ( a | b ) + // Current binary eval has the same left and right children, so it is useless. + // Connect the parent of the current eval and one of the children directly. + finalQual = leftChild; + plan.addHistory("Common condition is reduced."); + + } else if (commonQuals.size() == 1) { + // A single common qual is found. + // ( a | b ) ^ ( a | c ) -> a | (b ^ c) + EvalNode commonQual = commonQuals.iterator().next(); + EvalNode nonCommonQual; + + if (leftChild.getLeftExpr().equals(commonQual)) { + nonCommonQual = leftChild.getRightExpr(); + } else { + nonCommonQual = leftChild.getLeftExpr(); + } + + if (rightChild.getLeftExpr().equals(commonQual)) { + nonCommonQual = new BinaryEval(outerType, nonCommonQual, rightChild.getRightExpr()); + } else { + nonCommonQual = new BinaryEval(outerType, nonCommonQual, rightChild.getLeftExpr()); + } + + finalQual = new BinaryEval(innerType, commonQual, nonCommonQual); + plan.addHistory("Common condition is reduced."); + } else { + finalQual = evalNode; + } + } else { + finalQual = evalNode; + } + + return finalQual; + } + + } +} http://git-wip-us.apache.org/repos/asf/tajo/blob/f0ab0ca2/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/FilterPushDownRule.java ---------------------------------------------------------------------- diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/FilterPushDownRule.java b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/FilterPushDownRule.java index 30ef7a8..524d3ea 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/FilterPushDownRule.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/FilterPushDownRule.java @@ -24,25 +24,23 @@ import com.google.common.collect.ImmutableSet; import com.google.common.collect.Sets; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; -import org.apache.tajo.OverridableConf; import org.apache.tajo.algebra.JoinType; import org.apache.tajo.catalog.*; import org.apache.tajo.datum.Datum; -import org.apache.tajo.catalog.CatalogUtil; -import org.apache.tajo.catalog.Column; -import org.apache.tajo.catalog.Schema; -import org.apache.tajo.catalog.TableDesc; import org.apache.tajo.exception.TajoException; import org.apache.tajo.exception.TajoInternalError; -import org.apache.tajo.plan.*; +import org.apache.tajo.plan.InvalidQueryException; +import org.apache.tajo.plan.LogicalPlan; import org.apache.tajo.plan.LogicalPlan.QueryBlock; +import org.apache.tajo.plan.LogicalPlanner; +import org.apache.tajo.plan.Target; import org.apache.tajo.plan.expr.*; import org.apache.tajo.plan.logical.*; +import org.apache.tajo.plan.rewrite.LogicalPlanRewriteRule; import org.apache.tajo.plan.rewrite.LogicalPlanRewriteRuleContext; import org.apache.tajo.plan.rewrite.rules.FilterPushDownRule.FilterPushDownContext; import org.apache.tajo.plan.rewrite.rules.IndexScanInfo.SimplePredicate; import org.apache.tajo.plan.util.IndexUtil; -import org.apache.tajo.plan.rewrite.LogicalPlanRewriteRule; import org.apache.tajo.plan.util.PlannerUtil; import org.apache.tajo.plan.visitor.BasicLogicalPlanVisitor; import org.apache.tajo.util.TUtil; @@ -171,8 +169,7 @@ public class FilterPushDownRule extends BasicLogicalPlanVisitor stack) throws TajoException { + JoinNode joinNode, Stack stack) throws TajoException { Set onPredicates = TUtil.newHashSet(); if (joinNode.hasJoinQual()) { onPredicates.addAll(TUtil.newHashSet(AlgebraicUtil.toConjunctiveNormalFormArray(joinNode.getJoinQual()))); http://git-wip-us.apache.org/repos/asf/tajo/blob/f0ab0ca2/tajo-plan/src/main/java/org/apache/tajo/plan/util/IndexUtil.java ---------------------------------------------------------------------- diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/util/IndexUtil.java b/tajo-plan/src/main/java/org/apache/tajo/plan/util/IndexUtil.java index 73b33d5..8f847e0 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/util/IndexUtil.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/util/IndexUtil.java @@ -38,7 +38,7 @@ public class IndexUtil { public static List getAllEqualEvals(EvalNode qual) { EvalTreeUtil.EvalFinder finder = new EvalTreeUtil.EvalFinder(EvalType.EQUAL); - finder.visitChild(null, qual, new Stack()); + finder.visit(null, qual, new Stack()); return finder.getEvalNodes(); } http://git-wip-us.apache.org/repos/asf/tajo/blob/f0ab0ca2/tajo-plan/src/main/java/org/apache/tajo/plan/verifier/ExprsVerifier.java ---------------------------------------------------------------------- diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/verifier/ExprsVerifier.java b/tajo-plan/src/main/java/org/apache/tajo/plan/verifier/ExprsVerifier.java index eaf8775..6c6fb1d 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/verifier/ExprsVerifier.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/verifier/ExprsVerifier.java @@ -23,7 +23,6 @@ import org.apache.tajo.error.Errors; import org.apache.tajo.exception.TajoException; import org.apache.tajo.exception.TajoInternalError; import org.apache.tajo.exception.UndefinedOperatorException; -import org.apache.tajo.plan.PlanningException; import org.apache.tajo.plan.expr.*; import org.apache.tajo.plan.logical.LogicalNode; @@ -48,7 +47,7 @@ public class ExprsVerifier extends BasicEvalNodeVisitor()); + instance.visit(state, expression, new Stack()); Set referredColumns = EvalTreeUtil.findUniqueColumns(expression); for (Column referredColumn : referredColumns) { if (!currentNode.getInSchema().contains(referredColumn)) { @@ -247,7 +246,7 @@ public class ExprsVerifier extends BasicEvalNodeVisitor