Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id 5AD58200C37 for ; Sun, 19 Mar 2017 23:55:32 +0100 (CET) Received: by cust-asf.ponee.io (Postfix) id 5946D160B6D; Sun, 19 Mar 2017 22:55:32 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id 17C87160B7D for ; Sun, 19 Mar 2017 23:55:29 +0100 (CET) Received: (qmail 21481 invoked by uid 500); 19 Mar 2017 22:55:29 -0000 Mailing-List: contact commits-help@asterixdb.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@asterixdb.apache.org Delivered-To: mailing list commits@asterixdb.apache.org Received: (qmail 21471 invoked by uid 99); 19 Mar 2017 22:55:27 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Sun, 19 Mar 2017 22:55:27 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id BF905DFD73; Sun, 19 Mar 2017 22:55:27 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: wangsaeu@apache.org To: commits@asterixdb.apache.org Date: Sun, 19 Mar 2017 22:55:28 -0000 Message-Id: In-Reply-To: <2871f890841949219b33fdc99633ec36@git.apache.org> References: <2871f890841949219b33fdc99633ec36@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [2/2] asterixdb git commit: Index-only plan step 3: Top-down Select and Join transformation rule archived-at: Sun, 19 Mar 2017 22:55:32 -0000 Index-only plan step 3: Top-down Select and Join transformation rule - Converted IntroduceSelectAccessMethodRule and IntroduceJoinAccessMethodRule from bottom-up approach to top-down approach from the root operator. - Index-only plan needs to verify the variables that are live in the select or join condition are the only variables to be used afterwards unless a variable is generated after the select or join operator. - In order to keep this information, top-down approach needs to be introduced. Change-Id: I60a2a61eb46851d4c16c8f17447e3ac9b0aca778 Reviewed-on: https://asterix-gerrit.ics.uci.edu/1551 Tested-by: Jenkins BAD: Jenkins Integration-Tests: Jenkins Reviewed-by: Yingyi Bu Project: http://git-wip-us.apache.org/repos/asf/asterixdb/repo Commit: http://git-wip-us.apache.org/repos/asf/asterixdb/commit/c4eb6564 Tree: http://git-wip-us.apache.org/repos/asf/asterixdb/tree/c4eb6564 Diff: http://git-wip-us.apache.org/repos/asf/asterixdb/diff/c4eb6564 Branch: refs/heads/master Commit: c4eb65649b251f6a3403fb7301a40d0dab47418c Parents: d8b5972 Author: Taewoo Kim Authored: Fri Mar 17 19:16:11 2017 -0700 Committer: Taewoo Kim Committed: Sun Mar 19 15:55:01 2017 -0700 ---------------------------------------------------------------------- .../am/AbstractIntroduceAccessMethodRule.java | 40 +- .../rules/am/AccessMethodAnalysisContext.java | 65 +++- .../optimizer/rules/am/AccessMethodUtils.java | 61 +-- .../optimizer/rules/am/BTreeAccessMethod.java | 24 +- .../optimizer/rules/am/IAccessMethod.java | 2 +- .../rules/am/IntroduceJoinAccessMethodRule.java | 370 +++++++++++++------ .../am/IntroduceLSMComponentFilterRule.java | 9 +- .../am/IntroduceSelectAccessMethodRule.java | 248 ++++++++----- .../rules/am/InvertedIndexAccessMethod.java | 34 +- .../rules/am/OptimizableOperatorSubTree.java | 34 +- .../optimizer/rules/am/RTreeAccessMethod.java | 10 +- .../asterix/common/exceptions/ErrorCode.java | 8 + .../main/resources/asx_errormsg/en.properties | 8 + .../hyracks/api/exceptions/ErrorCode.java | 2 +- 14 files changed, 606 insertions(+), 309 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/asterixdb/blob/c4eb6564/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/AbstractIntroduceAccessMethodRule.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/AbstractIntroduceAccessMethodRule.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/AbstractIntroduceAccessMethodRule.java index d7fb0ac..5179adc 100644 --- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/AbstractIntroduceAccessMethodRule.java +++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/AbstractIntroduceAccessMethodRule.java @@ -73,7 +73,7 @@ import com.google.common.collect.ImmutableSet; */ public abstract class AbstractIntroduceAccessMethodRule implements IAlgebraicRewriteRule { - private MetadataProvider metadataProvider; + protected MetadataProvider metadataProvider; // Function Identifier sets that retain the original field variable through each function's arguments private final ImmutableSet funcIDSetThatRetainFieldName = @@ -100,7 +100,8 @@ public abstract class AbstractIntroduceAccessMethodRule implements IAlgebraicRew } @Override - public boolean rewritePre(Mutable opRef, IOptimizationContext context) { + public boolean rewritePre(Mutable opRef, IOptimizationContext context) + throws AlgebricksException { return false; } @@ -132,7 +133,7 @@ public abstract class AbstractIntroduceAccessMethodRule implements IAlgebraicRew pruneIndexCandidates(entry.getKey(), amCtx, context, typeEnvironment); // Remove access methods for which there are definitely no // applicable indexes. - if (amCtx.indexExprsAndVars.isEmpty()) { + if (amCtx.isIndexExprsAndVarsEmpty()) { amIt.remove(); } } @@ -143,7 +144,7 @@ public abstract class AbstractIntroduceAccessMethodRule implements IAlgebraicRew * process by making it more systematic. */ protected Pair chooseBestIndex(Map analyzedAMs) { - List> list = chooseAllIndex(analyzedAMs); + List> list = chooseAllIndexes(analyzedAMs); return list.isEmpty() ? null : list.get(0); } @@ -155,7 +156,7 @@ public abstract class AbstractIntroduceAccessMethodRule implements IAlgebraicRew * [InvertedIndexAccessMethod, IndexType.SINGLE_PARTITION_WORD_INVIX || SINGLE_PARTITION_NGRAM_INVIX || * LENGTH_PARTITIONED_WORD_INVIX || LENGTH_PARTITIONED_NGRAM_INVIX] */ - protected List> chooseAllIndex( + protected List> chooseAllIndexes( Map analyzedAMs) { List> result = new ArrayList<>(); // Use variables (fields) to the index types map to check which type of indexes are applied for the vars. @@ -165,7 +166,7 @@ public abstract class AbstractIntroduceAccessMethodRule implements IAlgebraicRew Map.Entry amEntry = amIt.next(); AccessMethodAnalysisContext analysisCtx = amEntry.getValue(); Iterator>>> indexIt = - analysisCtx.indexExprsAndVars.entrySet().iterator(); + analysisCtx.getIteratorForIndexExprsAndVars(); while (indexIt.hasNext()) { Map.Entry>> indexEntry = indexIt.next(); IAccessMethod chosenAccessMethod = amEntry.getKey(); @@ -209,7 +210,7 @@ public abstract class AbstractIntroduceAccessMethodRule implements IAlgebraicRew public void pruneIndexCandidates(IAccessMethod accessMethod, AccessMethodAnalysisContext analysisCtx, IOptimizationContext context, IVariableTypeEnvironment typeEnvironment) throws AlgebricksException { Iterator>>> indexExprAndVarIt = - analysisCtx.indexExprsAndVars.entrySet().iterator(); + analysisCtx.getIteratorForIndexExprsAndVars(); // Used to keep track of matched expressions (added for prefix search) int numMatchedKeys = 0; ArrayList matchedExpressions = new ArrayList<>(); @@ -229,7 +230,7 @@ public abstract class AbstractIntroduceAccessMethodRule implements IAlgebraicRew Iterator> exprsAndVarIter = indexExprAndVarEntry.getValue().iterator(); while (exprsAndVarIter.hasNext()) { final Pair exprAndVarIdx = exprsAndVarIter.next(); - final IOptimizableFuncExpr optFuncExpr = analysisCtx.matchedFuncExprs.get(exprAndVarIdx.first); + final IOptimizableFuncExpr optFuncExpr = analysisCtx.getMatchedFuncExpr(exprAndVarIdx.first); // If expr is not optimizable by concrete index then remove // expr and continue. if (!accessMethod.exprIsOptimizable(index, optFuncExpr)) { @@ -357,7 +358,7 @@ public abstract class AbstractIntroduceAccessMethodRule implements IAlgebraicRew continue; } } - analysisCtx.indexNumMatchedKeys.put(index, new Integer(numMatchedKeys)); + analysisCtx.putNumberOfMatchedKeys(index, Integer.valueOf(numMatchedKeys)); } } @@ -381,13 +382,13 @@ public abstract class AbstractIntroduceAccessMethodRule implements IAlgebraicRew * * @throws AlgebricksException */ - protected boolean analyzeCondition(ILogicalExpression cond, List assignsAndUnnests, + protected boolean analyzeSelectOrJoinOpConditionAndUpdateAnalyzedAM(ILogicalExpression cond, + List assignsAndUnnests, Map analyzedAMs, IOptimizationContext context, IVariableTypeEnvironment typeEnvironment) throws AlgebricksException { AbstractFunctionCallExpression funcExpr = (AbstractFunctionCallExpression) cond; FunctionIdentifier funcIdent = funcExpr.getFunctionIdentifier(); - // Don't consider optimizing a disjunctive condition with an index (too - // complicated for now). + // TODO: We don't consider a disjunctive condition with an index yet since it's complex. if (funcIdent == AlgebricksBuiltinFunctions.OR) { return false; } else if (funcIdent == AlgebricksBuiltinFunctions.AND) { @@ -400,14 +401,15 @@ public abstract class AbstractIntroduceAccessMethodRule implements IAlgebraicRew continue; } AbstractFunctionCallExpression argFuncExpr = (AbstractFunctionCallExpression) argExpr; - boolean matchFound = - analyzeFunctionExpr(argFuncExpr, assignsAndUnnests, analyzedAMs, context, typeEnvironment); + boolean matchFound = analyzeFunctionExprAndUpdateAnalyzedAM(argFuncExpr, assignsAndUnnests, analyzedAMs, + context, typeEnvironment); found = found || matchFound; } return found; } else { // For single function or "NOT" case: - return analyzeFunctionExpr(funcExpr, assignsAndUnnests, analyzedAMs, context, typeEnvironment); + return analyzeFunctionExprAndUpdateAnalyzedAM(funcExpr, assignsAndUnnests, analyzedAMs, context, + typeEnvironment); } } @@ -418,7 +420,7 @@ public abstract class AbstractIntroduceAccessMethodRule implements IAlgebraicRew * * @throws AlgebricksException */ - protected boolean analyzeFunctionExpr(AbstractFunctionCallExpression funcExpr, + protected boolean analyzeFunctionExprAndUpdateAnalyzedAM(AbstractFunctionCallExpression funcExpr, List assignsAndUnnests, Map analyzedAMs, IOptimizationContext context, IVariableTypeEnvironment typeEnvironment) throws AlgebricksException { @@ -443,8 +445,8 @@ public abstract class AbstractIntroduceAccessMethodRule implements IAlgebraicRew } // Analyzes the funcExpr's arguments to see if the accessMethod is // truly applicable. - boolean matchFound = accessMethod.analyzeFuncExprArgs(funcExpr, assignsAndUnnests, analysisCtx, context, - typeEnvironment); + boolean matchFound = accessMethod.analyzeFuncExprArgsAndUpdateAnalysisCtx(funcExpr, assignsAndUnnests, + analysisCtx, context, typeEnvironment); if (matchFound) { // If we've used the current new context placeholder, replace it // with a new one. @@ -510,7 +512,7 @@ public abstract class AbstractIntroduceAccessMethodRule implements IAlgebraicRew datasetRecordVar = datasetVars.get(datasetVars.size() - 1); } } - for (IOptimizableFuncExpr optFuncExpr : analysisCtx.matchedFuncExprs) { + for (IOptimizableFuncExpr optFuncExpr : analysisCtx.getMatchedFuncExprs()) { // Try to match variables from optFuncExpr to assigns or unnests. for (int assignOrUnnestIndex = 0; assignOrUnnestIndex < subTree.getAssignsAndUnnests() .size(); assignOrUnnestIndex++) { http://git-wip-us.apache.org/repos/asf/asterixdb/blob/c4eb6564/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/AccessMethodAnalysisContext.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/AccessMethodAnalysisContext.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/AccessMethodAnalysisContext.java index ca5da98..16ee6d1 100644 --- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/AccessMethodAnalysisContext.java +++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/AccessMethodAnalysisContext.java @@ -19,14 +19,14 @@ package org.apache.asterix.optimizer.rules.am; import java.util.ArrayList; +import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.TreeMap; -import org.apache.commons.lang3.mutable.Mutable; - import org.apache.asterix.metadata.entities.Dataset; import org.apache.asterix.metadata.entities.Index; +import org.apache.commons.lang3.mutable.Mutable; import org.apache.hyracks.algebricks.common.utils.Pair; import org.apache.hyracks.algebricks.core.algebra.base.ILogicalOperator; import org.apache.hyracks.algebricks.core.algebra.expressions.ScalarFunctionCallExpression; @@ -36,37 +36,70 @@ import org.apache.hyracks.algebricks.core.algebra.expressions.ScalarFunctionCall */ public class AccessMethodAnalysisContext { - public List matchedFuncExprs = new ArrayList(); + private List matchedFuncExprs = new ArrayList(); - // Contains candidate indexes and a list of (integer,integer) tuples that index into matchedFuncExprs and matched variable inside this expr. - // We are mapping from candidate indexes to a list of function expressions + // Contains candidate indexes and a list of (integer,integer) tuples that index into matchedFuncExprs and + // matched variable inside this expr. We are mapping from candidate indexes to a list of function expressions // that match one of the index's expressions. - public Map>> indexExprsAndVars = new TreeMap>>(); + private Map>> indexExprsAndVars = + new TreeMap>>(); // Maps from index to the dataset it is indexing. - public Map indexDatasetMap = new TreeMap(); + private Map indexDatasetMap = new TreeMap(); // Maps from an index to the number of matched fields in the query plan (for performing prefix search) - public Map indexNumMatchedKeys = new TreeMap(); + private Map indexNumMatchedKeys = new TreeMap(); // variables for resetting null placeholder for left-outer-join private Mutable lojGroupbyOpRef = null; private ScalarFunctionCallExpression lojIsNullFuncInGroupBy = null; public void addIndexExpr(Dataset dataset, Index index, Integer exprIndex, Integer varIndex) { - List> exprs = indexExprsAndVars.get(index); + List> exprs = getIndexExprsFromIndexExprsAndVars(index); if (exprs == null) { exprs = new ArrayList>(); - indexExprsAndVars.put(index, exprs); + putIndexExprToIndexExprsAndVars(index, exprs); } exprs.add(new Pair(exprIndex, varIndex)); - indexDatasetMap.put(index, dataset); + putDatasetIntoIndexDatasetMap(index, dataset); + } + + public List getMatchedFuncExprs() { + return matchedFuncExprs; + } + + public IOptimizableFuncExpr getMatchedFuncExpr(int index) { + return matchedFuncExprs.get(index); + } + + public void addMatchedFuncExpr(IOptimizableFuncExpr optFuncExpr) { + matchedFuncExprs.add(optFuncExpr); } - public List> getIndexExprs(Index index) { + public Iterator>>> getIteratorForIndexExprsAndVars() { + return indexExprsAndVars.entrySet().iterator(); + } + + public boolean isIndexExprsAndVarsEmpty() { + return indexExprsAndVars.isEmpty(); + } + + public List> getIndexExprsFromIndexExprsAndVars(Index index) { return indexExprsAndVars.get(index); } + public void putIndexExprToIndexExprsAndVars(Index index, List> exprs) { + indexExprsAndVars.put(index, exprs); + } + + public Integer getNumberOfMatchedKeys(Index index) { + return indexNumMatchedKeys.get(index); + } + + public void putNumberOfMatchedKeys(Index index, Integer numMatchedKeys) { + indexNumMatchedKeys.put(index, numMatchedKeys); + } + public void setLOJGroupbyOpRef(Mutable opRef) { lojGroupbyOpRef = opRef; } @@ -83,4 +116,12 @@ public class AccessMethodAnalysisContext { return lojIsNullFuncInGroupBy; } + public Dataset getDatasetFromIndexDatasetMap(Index idx) { + return indexDatasetMap.get(idx); + } + + public void putDatasetIntoIndexDatasetMap(Index idx, Dataset ds) { + indexDatasetMap.put(idx, ds); + } + } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/c4eb6564/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/AccessMethodUtils.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/AccessMethodUtils.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/AccessMethodUtils.java index 7743d9f..e059574 100644 --- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/AccessMethodUtils.java +++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/AccessMethodUtils.java @@ -129,9 +129,9 @@ public class AccessMethodUtils { return ConstantExpressionUtil.getBooleanConstant(expr.getValue()); } - public static boolean analyzeFuncExprArgsForOneConstAndVar(AbstractFunctionCallExpression funcExpr, - AccessMethodAnalysisContext analysisCtx, IOptimizationContext context, - IVariableTypeEnvironment typeEnvironment) throws AlgebricksException { + public static boolean analyzeFuncExprArgsForOneConstAndVarAndUpdateAnalysisCtx( + AbstractFunctionCallExpression funcExpr, AccessMethodAnalysisContext analysisCtx, + IOptimizationContext context, IVariableTypeEnvironment typeEnvironment) throws AlgebricksException { ILogicalExpression constExpression = null; IAType constantExpressionType = null; LogicalVariable fieldVar = null; @@ -180,16 +180,38 @@ public class AccessMethodUtils { } else { return false; } - OptimizableFuncExpr newOptFuncExpr = new OptimizableFuncExpr(funcExpr, fieldVar, constExpression, - constantExpressionType); - for (IOptimizableFuncExpr optFuncExpr : analysisCtx.matchedFuncExprs) { + + // Updates the given Analysis Context by adding a new optimizable function expression. + constructNewOptFuncExprAndAddToAnalysisCtx(funcExpr, fieldVar, constExpression, constantExpressionType, + analysisCtx); + return true; + } + + private static void constructNewOptFuncExprAndAddToAnalysisCtx(AbstractFunctionCallExpression funcExpr, + LogicalVariable fieldVar, ILogicalExpression expression, IAType expressionType, + AccessMethodAnalysisContext analysisCtx) { + OptimizableFuncExpr newOptFuncExpr = + new OptimizableFuncExpr(funcExpr, fieldVar, expression, expressionType); + addNewOptFuncExprToAnalysisCtx(funcExpr, newOptFuncExpr, analysisCtx); + } + + private static void constructNewOptFuncExprAndAddToAnalysisCtx(AbstractFunctionCallExpression funcExpr, + LogicalVariable[] fieldVars, ILogicalExpression[] expressions, IAType[] expressionTypes, + AccessMethodAnalysisContext analysisCtx) { + OptimizableFuncExpr newOptFuncExpr = new OptimizableFuncExpr(funcExpr, fieldVars, expressions, expressionTypes); + addNewOptFuncExprToAnalysisCtx(funcExpr, newOptFuncExpr, analysisCtx); + + } + + private static void addNewOptFuncExprToAnalysisCtx(AbstractFunctionCallExpression funcExpr, + OptimizableFuncExpr newOptFuncExpr, AccessMethodAnalysisContext analysisCtx) { + for (IOptimizableFuncExpr optFuncExpr : analysisCtx.getMatchedFuncExprs()) { //avoid additional optFuncExpressions in case of a join if (optFuncExpr.getFuncExpr().equals(funcExpr)) { - return true; + return; } } - analysisCtx.matchedFuncExprs.add(newOptFuncExpr); - return true; + analysisCtx.addMatchedFuncExpr(newOptFuncExpr); } /** @@ -247,7 +269,7 @@ public class AccessMethodUtils { } } - public static boolean analyzeFuncExprArgsForTwoVars(AbstractFunctionCallExpression funcExpr, + public static boolean analyzeFuncExprArgsForTwoVarsAndUpdateAnalysisCtx(AbstractFunctionCallExpression funcExpr, AccessMethodAnalysisContext analysisCtx) { LogicalVariable fieldVar1 = null; LogicalVariable fieldVar2 = null; @@ -260,15 +282,10 @@ public class AccessMethodUtils { } else { return false; } - OptimizableFuncExpr newOptFuncExpr = new OptimizableFuncExpr(funcExpr, - new LogicalVariable[] { fieldVar1, fieldVar2 }, new ILogicalExpression[0], new IAType[0]); - for (IOptimizableFuncExpr optFuncExpr : analysisCtx.matchedFuncExprs) { - //avoid additional optFuncExpressions in case of a join - if (optFuncExpr.getFuncExpr().equals(funcExpr)) { - return true; - } - } - analysisCtx.matchedFuncExprs.add(newOptFuncExpr); + + // Updates the given Analysis Context by adding a new optimizable function expression. + constructNewOptFuncExprAndAddToAnalysisCtx(funcExpr, new LogicalVariable[] { fieldVar1, fieldVar2 }, + new ILogicalExpression[0], new IAType[0], analysisCtx); return true; } @@ -451,13 +468,13 @@ public class AccessMethodUtils { */ public static IOptimizableFuncExpr chooseFirstOptFuncExpr(Index chosenIndex, AccessMethodAnalysisContext analysisCtx) { - List> indexExprs = analysisCtx.getIndexExprs(chosenIndex); + List> indexExprs = analysisCtx.getIndexExprsFromIndexExprsAndVars(chosenIndex); int firstExprIndex = indexExprs.get(0).first; - return analysisCtx.matchedFuncExprs.get(firstExprIndex); + return analysisCtx.getMatchedFuncExpr(firstExprIndex); } public static int chooseFirstOptFuncVar(Index chosenIndex, AccessMethodAnalysisContext analysisCtx) { - List> indexExprs = analysisCtx.getIndexExprs(chosenIndex); + List> indexExprs = analysisCtx.getIndexExprsFromIndexExprsAndVars(chosenIndex); return indexExprs.get(0).second; } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/c4eb6564/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/BTreeAccessMethod.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/BTreeAccessMethod.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/BTreeAccessMethod.java index 3acf1f5..0f6012f 100644 --- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/BTreeAccessMethod.java +++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/BTreeAccessMethod.java @@ -99,14 +99,14 @@ public class BTreeAccessMethod implements IAccessMethod { } @Override - public boolean analyzeFuncExprArgs(AbstractFunctionCallExpression funcExpr, + public boolean analyzeFuncExprArgsAndUpdateAnalysisCtx(AbstractFunctionCallExpression funcExpr, List assignsAndUnnests, AccessMethodAnalysisContext analysisCtx, IOptimizationContext context, IVariableTypeEnvironment typeEnvironment) throws AlgebricksException { boolean matches = - AccessMethodUtils.analyzeFuncExprArgsForOneConstAndVar( + AccessMethodUtils.analyzeFuncExprArgsForOneConstAndVarAndUpdateAnalysisCtx( funcExpr, analysisCtx, context, typeEnvironment); if (!matches) { - matches = AccessMethodUtils.analyzeFuncExprArgsForTwoVars(funcExpr, analysisCtx); + matches = AccessMethodUtils.analyzeFuncExprArgsForTwoVarsAndUpdateAnalysisCtx(funcExpr, analysisCtx); } return matches; } @@ -175,7 +175,7 @@ public class BTreeAccessMethod implements IAccessMethod { Mutable conditionRef = joinOp.getCondition(); // Determine if the index is applicable on the left or right side // (if both, we arbitrarily prefer the left side). - Dataset dataset = analysisCtx.indexDatasetMap.get(chosenIndex); + Dataset dataset = analysisCtx.getDatasetFromIndexDatasetMap(chosenIndex); OptimizableOperatorSubTree indexSubTree; OptimizableOperatorSubTree probeSubTree; // We assume that the left subtree is the outer branch and the right subtree is the inner branch. @@ -225,17 +225,16 @@ public class BTreeAccessMethod implements IAccessMethod { @Override public ILogicalOperator createSecondaryToPrimaryPlan(Mutable conditionRef, OptimizableOperatorSubTree indexSubTree, OptimizableOperatorSubTree probeSubTree, Index chosenIndex, - AccessMethodAnalysisContext analysisCtx, boolean retainInput, boolean retainNull, - boolean requiresBroadcast, IOptimizationContext context) throws AlgebricksException { + AccessMethodAnalysisContext analysisCtx, boolean retainInput, boolean retainNull, boolean requiresBroadcast, + IOptimizationContext context) throws AlgebricksException { Dataset dataset = indexSubTree.getDataset(); ARecordType recordType = indexSubTree.getRecordType(); ARecordType metaRecordType = indexSubTree.getMetaRecordType(); // we made sure indexSubTree has datasource scan AbstractDataSourceOperator dataSourceOp = (AbstractDataSourceOperator) indexSubTree.getDataSourceRef().getValue(); - List> exprAndVarList = analysisCtx.indexExprsAndVars.get(chosenIndex); - List matchedFuncExprs = analysisCtx.matchedFuncExprs; - int numSecondaryKeys = analysisCtx.indexNumMatchedKeys.get(chosenIndex); + List> exprAndVarList = analysisCtx.getIndexExprsFromIndexExprsAndVars(chosenIndex); + int numSecondaryKeys = analysisCtx.getNumberOfMatchedKeys(chosenIndex); // List of function expressions that will be replaced by the secondary-index search. // These func exprs will be removed from the select condition at the very end of this method. Set replacedFuncExprs = new HashSet<>(); @@ -267,15 +266,14 @@ public class BTreeAccessMethod implements IAccessMethod { for (Pair exprIndex : exprAndVarList) { // Position of the field of matchedFuncExprs.get(exprIndex) in the chosen index's indexed exprs. - IOptimizableFuncExpr optFuncExpr = matchedFuncExprs.get(exprIndex.first); + IOptimizableFuncExpr optFuncExpr = analysisCtx.getMatchedFuncExpr(exprIndex.first); int keyPos = indexOf(optFuncExpr.getFieldName(0), chosenIndex.getKeyFieldNames()); if (keyPos < 0 && optFuncExpr.getNumLogicalVars() > 1) { // If we are optimizing a join, the matching field may be the second field name. keyPos = indexOf(optFuncExpr.getFieldName(1), chosenIndex.getKeyFieldNames()); } if (keyPos < 0) { - throw new AlgebricksException( - "Could not match optimizable function expression to any index field name."); + throw CompilationException.create(ErrorCode.NO_INDEX_FIELD_NAME_FOR_GIVEN_FUNC_EXPR); } Pair returnedSearchKeyExpr = AccessMethodUtils.createSearchKeyExpr(optFuncExpr, indexSubTree, probeSubTree); @@ -420,7 +418,7 @@ public class BTreeAccessMethod implements IAccessMethod { } if (!couldntFigureOut) { // Remember to remove this funcExpr later. - replacedFuncExprs.add(matchedFuncExprs.get(exprIndex.first).getFuncExpr()); + replacedFuncExprs.add(analysisCtx.getMatchedFuncExpr(exprIndex.first).getFuncExpr()); } if (doneWithExprs) { break; http://git-wip-us.apache.org/repos/asf/asterixdb/blob/c4eb6564/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/IAccessMethod.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/IAccessMethod.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/IAccessMethod.java index d249b96..870b425 100644 --- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/IAccessMethod.java +++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/IAccessMethod.java @@ -57,7 +57,7 @@ public interface IAccessMethod extends Comparable { * otherwise * @throws AlgebricksException */ - boolean analyzeFuncExprArgs(AbstractFunctionCallExpression funcExpr, + boolean analyzeFuncExprArgsAndUpdateAnalysisCtx(AbstractFunctionCallExpression funcExpr, List assignsAndUnnests, AccessMethodAnalysisContext analysisCtx, IOptimizationContext context, IVariableTypeEnvironment typeEnvironment) throws AlgebricksException; http://git-wip-us.apache.org/repos/asf/asterixdb/blob/c4eb6564/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/IntroduceJoinAccessMethodRule.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/IntroduceJoinAccessMethodRule.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/IntroduceJoinAccessMethodRule.java index 73bbbfc..7fc7902 100644 --- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/IntroduceJoinAccessMethodRule.java +++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/IntroduceJoinAccessMethodRule.java @@ -23,7 +23,8 @@ import java.util.Iterator; import java.util.List; import java.util.Map; -import org.apache.asterix.metadata.declared.MetadataProvider; +import org.apache.asterix.algebra.operators.CommitOperator; +import org.apache.asterix.metadata.entities.Dataset; import org.apache.asterix.metadata.entities.Index; import org.apache.commons.lang3.mutable.Mutable; import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException; @@ -39,6 +40,7 @@ import org.apache.hyracks.algebricks.core.algebra.expressions.ScalarFunctionCall import org.apache.hyracks.algebricks.core.algebra.functions.FunctionIdentifier; import org.apache.hyracks.algebricks.core.algebra.operators.logical.AbstractBinaryJoinOperator; import org.apache.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator; +import org.apache.hyracks.algebricks.core.algebra.operators.logical.DelegateOperator; import org.apache.hyracks.algebricks.core.algebra.operators.logical.GroupByOperator; import org.apache.hyracks.algebricks.core.algebra.operators.logical.InnerJoinOperator; import org.apache.hyracks.algebricks.core.algebra.operators.logical.LeftOuterJoinOperator; @@ -53,7 +55,8 @@ import org.apache.hyracks.algebricks.core.algebra.util.OperatorPropertiesUtil; * This rule tries to utilize an index on the inner relation. * If that's not possible, it stops transforming the given join into an index-nested-loop join. * Replaces the above pattern with the following simplified plan: - * (select) <-- (assign) <-- (btree search) <-- (sort) <-- (unnest(index search)) <-- (assign) <-- (datasource scan | unnest-map) + * (select) <-- (assign) <-- (btree search) <-- (sort) <-- (unnest(index search)) <-- (assign) <-- (A) + * (A) <-- (datasource scan | unnest-map) * The sort is optional, and some access methods may choose not to sort. * Note that for some index-based optimizations we do not remove the triggering * condition from the join, since the secondary index may only act as a filter, and the @@ -76,7 +79,7 @@ import org.apache.hyracks.algebricks.core.algebra.util.OperatorPropertiesUtil; public class IntroduceJoinAccessMethodRule extends AbstractIntroduceAccessMethodRule { protected Mutable joinRef = null; - protected AbstractBinaryJoinOperator join = null; + protected AbstractBinaryJoinOperator joinOp = null; protected AbstractFunctionCallExpression joinCond = null; protected final OptimizableOperatorSubTree leftSubTree = new OptimizableOperatorSubTree(); protected final OptimizableOperatorSubTree rightSubTree = new OptimizableOperatorSubTree(); @@ -85,7 +88,7 @@ public class IntroduceJoinAccessMethodRule extends AbstractIntroduceAccessMethod protected boolean hasGroupBy = true; // Register access methods. - protected static Map> accessMethods = new HashMap>(); + protected static Map> accessMethods = new HashMap<>(); static { registerAccessMethod(BTreeAccessMethod.INSTANCE, accessMethods); @@ -93,90 +96,61 @@ public class IntroduceJoinAccessMethodRule extends AbstractIntroduceAccessMethod registerAccessMethod(InvertedIndexAccessMethod.INSTANCE, accessMethods); } + /** + * Recursively check the given plan from the root operator to transform a plan + * with JOIN or LEFT-OUTER-JOIN operator into an index-utilized plan. + */ + @Override - public boolean rewritePost(Mutable opRef, IOptimizationContext context) + public boolean rewritePre(Mutable opRef, IOptimizationContext context) throws AlgebricksException { clear(); setMetadataDeclarations(context); - // Match operator pattern and initialize optimizable sub trees. - if (!matchesOperatorPattern(opRef, context)) { - return false; - } - // Analyze condition on those optimizable subtrees that have a datasource scan. - Map analyzedAMs = new HashMap(); - boolean matchInLeftSubTree = false; - boolean matchInRightSubTree = false; - if (leftSubTree.hasDataSource()) { - matchInLeftSubTree = analyzeCondition(joinCond, leftSubTree.getAssignsAndUnnests(), analyzedAMs, context, - typeEnvironment); - } - if (rightSubTree.hasDataSource()) { - matchInRightSubTree = analyzeCondition(joinCond, rightSubTree.getAssignsAndUnnests(), analyzedAMs, context, - typeEnvironment); - } - if (!matchInLeftSubTree && !matchInRightSubTree) { + AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue(); + + // Already checked? + if (context.checkIfInDontApplySet(this, op)) { return false; } - // Set dataset and type metadata. - MetadataProvider metadataProvider = (MetadataProvider) context.getMetadataProvider(); - boolean checkLeftSubTreeMetadata = false; - boolean checkRightSubTreeMetadata = false; - if (matchInLeftSubTree) { - checkLeftSubTreeMetadata = leftSubTree.setDatasetAndTypeMetadata(metadataProvider); - } - if (matchInRightSubTree) { - checkRightSubTreeMetadata = rightSubTree.setDatasetAndTypeMetadata(metadataProvider); - } - if (!checkLeftSubTreeMetadata && !checkRightSubTreeMetadata) { + // Check whether this operator is the root, which is DISTRIBUTE_RESULT or SINK since + // we start the process from the root operator. + if (op.getOperatorTag() != LogicalOperatorTag.DISTRIBUTE_RESULT + && op.getOperatorTag() != LogicalOperatorTag.SINK + && op.getOperatorTag() != LogicalOperatorTag.DELEGATE_OPERATOR) { return false; } - if (checkLeftSubTreeMetadata) { - fillSubTreeIndexExprs(leftSubTree, analyzedAMs, context); - } - if (checkRightSubTreeMetadata) { - fillSubTreeIndexExprs(rightSubTree, analyzedAMs, context); - } - pruneIndexCandidates(analyzedAMs, context, typeEnvironment); - // We only consider indexes from the inner branch (right subTree). - // If no index is available, then we stop this optimization. - removeIndexCandidatesFromOuterBranch(analyzedAMs); - - // Choose an index from the inner branch that will be used. - Pair chosenIndex = chooseBestIndex(analyzedAMs); - if (chosenIndex == null) { - context.addToDontApplySet(this, join); + if (op.getOperatorTag() == LogicalOperatorTag.DELEGATE_OPERATOR + && !(((DelegateOperator) op).getDelegate() instanceof CommitOperator)) { return false; } - // Apply plan transformation using chosen index. - AccessMethodAnalysisContext analysisCtx = analyzedAMs.get(chosenIndex.first); + // Recursively check the given plan whether the desired pattern exists in it. + // If so, try to optimize the plan. + boolean planTransformed = checkAndApplyJoinTransformation(opRef, context); - //For LOJ with GroupBy, prepare objects to reset LOJ nullPlaceHolderVariable in GroupByOp - if (isLeftOuterJoin && hasGroupBy) { - analysisCtx.setLOJGroupbyOpRef(opRef); - ScalarFunctionCallExpression isNullFuncExpr = AccessMethodUtils - .findLOJIsMissingFuncInGroupBy((GroupByOperator) opRef.getValue()); - analysisCtx.setLOJIsNullFuncInGroupBy(isNullFuncExpr); + if (joinOp != null) { + // We found an optimization here. Don't need to optimize this operator again. + context.addToDontApplySet(this, joinOp); } - // At this point, we are sure that only an index from the inner branch is going to be used. - // So, the left subtree is the outer branch and the right subtree is the inner branch. - boolean res = chosenIndex.first.applyJoinPlanTransformation(joinRef, leftSubTree, rightSubTree, - chosenIndex.second, analysisCtx, context, isLeftOuterJoin, hasGroupBy); - if (res) { + if (!planTransformed) { + return false; + } else { OperatorPropertiesUtil.typeOpRec(opRef, context); } - context.addToDontApplySet(this, join); - return res; + + return planTransformed; } /** - * Removes indexes from the optimizer's consideration for this rule. + * Removes indexes from the outer branch from the optimizer's consideration for this rule, + * since we only use indexes from the inner branch. */ - protected void removeIndexCandidatesFromOuterBranch(Map analyzedAMs) { + protected void pruneIndexCandidatesFromOuterBranch(Map analyzedAMs) { + // Inner branch is the right side branch of the given JOIN operator. String innerDataset = null; if (rightSubTree.getDataset() != null) { innerDataset = rightSubTree.getDataset().getDatasetName(); @@ -188,8 +162,7 @@ public class IntroduceJoinAccessMethodRule extends AbstractIntroduceAccessMethod AccessMethodAnalysisContext amCtx = entry.getValue(); // Fetch index, expression, and variables. - Iterator>>> indexIt = amCtx.indexExprsAndVars.entrySet() - .iterator(); + Iterator>>> indexIt = amCtx.getIteratorForIndexExprsAndVars(); while (indexIt.hasNext()) { Map.Entry>> indexExprAndVarEntry = indexIt.next(); @@ -198,10 +171,10 @@ public class IntroduceJoinAccessMethodRule extends AbstractIntroduceAccessMethod while (exprsAndVarIter.hasNext()) { Pair exprAndVarIdx = exprsAndVarIter.next(); - IOptimizableFuncExpr optFuncExpr = amCtx.matchedFuncExprs.get(exprAndVarIdx.first); + IOptimizableFuncExpr optFuncExpr = amCtx.getMatchedFuncExpr(exprAndVarIdx.first); - // Does this index come from the inner branch? - // We check the dataset name and the subtree to make sure the index is applicable. + // We check the dataset name and the subtree to make sure + // that this index come from the inner branch. if (indexExprAndVarEntry.getKey().getDatasetName().equals(innerDataset)) { if (optFuncExpr.getOperatorSubTree(exprAndVarIdx.second).equals(rightSubTree)) { indexFromInnerBranch = true; @@ -209,8 +182,8 @@ public class IntroduceJoinAccessMethodRule extends AbstractIntroduceAccessMethod } } - // If the index does not come from the inner branch, - // We do not consider this index. + // If the given index does not come from the inner branch, + // prune this index so that the optimizer doesn't consider this index in this rule. if (!indexFromInnerBranch) { indexIt.remove(); } @@ -218,50 +191,11 @@ public class IntroduceJoinAccessMethodRule extends AbstractIntroduceAccessMethod } } - protected boolean matchesOperatorPattern(Mutable opRef, IOptimizationContext context) - throws AlgebricksException { - // First check that the operator is a join and its condition is a function call. - AbstractLogicalOperator op1 = (AbstractLogicalOperator) opRef.getValue(); - if (context.checkIfInDontApplySet(this, op1)) { - return false; - } - - boolean isInnerJoin = isInnerJoin(op1); - isLeftOuterJoin = isLeftOuterJoin(op1); - - if (!isInnerJoin && !isLeftOuterJoin) { - return false; - } - - // Set and analyze select. - if (isInnerJoin) { - joinRef = opRef; - join = (InnerJoinOperator) op1; - } else { - joinRef = op1.getInputs().get(0); - join = (LeftOuterJoinOperator) joinRef.getValue(); - } - - typeEnvironment = context.getOutputTypeEnvironment(join); - // Check that the select's condition is a function call. - ILogicalExpression condExpr = join.getCondition().getValue(); - if (condExpr.getExpressionTag() != LogicalExpressionTag.FUNCTION_CALL) { - return false; - } - joinCond = (AbstractFunctionCallExpression) condExpr; - boolean leftSubTreeInitialized = leftSubTree.initFromSubTree(join.getInputs().get(0)); - boolean rightSubTreeInitialized = rightSubTree.initFromSubTree(join.getInputs().get(1)); - if (!leftSubTreeInitialized || !rightSubTreeInitialized) { - return false; - } - - // One of the subtrees must have a datasource scan. - if (leftSubTree.hasDataSourceScan() || rightSubTree.hasDataSourceScan()) { - return true; - } - return false; - } - + /** + * Checks whether the given operator is LEFTOUTERJOIN. + * If so, also checks that GROUPBY is placed after LEFTOUTERJOIN. + */ + // Check whether (Groupby)? <-- Leftouterjoin private boolean isLeftOuterJoin(AbstractLogicalOperator op1) { if (op1.getInputs().size() != 1) { return false; @@ -277,6 +211,9 @@ public class IntroduceJoinAccessMethodRule extends AbstractIntroduceAccessMethod return true; } + /** + * Checks whether the given operator is INNERJOIN. + */ private boolean isInnerJoin(AbstractLogicalOperator op1) { return op1.getOperatorTag() == LogicalOperatorTag.INNERJOIN; } @@ -288,8 +225,207 @@ public class IntroduceJoinAccessMethodRule extends AbstractIntroduceAccessMethod private void clear() { joinRef = null; - join = null; + joinOp = null; joinCond = null; isLeftOuterJoin = false; } + + /** + * Recursively traverse the given plan and check whether a INNERJOIN or LEFTOUTERJOIN operator exists. + * If one is found, maintain the path from the root to the given join operator and + * optimize the path from the given join operator to the EMPTY_TUPLE_SOURCE operator + * if it is not already optimized. + */ + protected boolean checkAndApplyJoinTransformation(Mutable opRef, IOptimizationContext context) + throws AlgebricksException { + AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue(); + boolean joinFoundAndOptimizationApplied; + + // Check the current operator pattern to see whether it is a JOIN or not. + boolean isThisOpInnerJoin = isInnerJoin(op); + boolean isThisOpLeftOuterJoin = isLeftOuterJoin(op); + boolean isParentOpGroupBy = hasGroupBy; + + Mutable joinRefFromThisOp = null; + AbstractBinaryJoinOperator joinOpFromThisOp = null; + + if (isThisOpInnerJoin) { + // Set join operator. + joinRef = opRef; + joinOp = (InnerJoinOperator) op; + joinRefFromThisOp = opRef; + joinOpFromThisOp = (InnerJoinOperator) op; + } else if (isThisOpLeftOuterJoin) { + // Set left-outer-join op. + // The current operator is GROUP and the child of this op is LEFTOUERJOIN. + joinRef = op.getInputs().get(0); + joinOp = (LeftOuterJoinOperator) joinRef.getValue(); + joinRefFromThisOp = op.getInputs().get(0); + joinOpFromThisOp = (LeftOuterJoinOperator) joinRefFromThisOp.getValue(); + } + + // Recursively check the plan and try to optimize it. We first check the children of the given operator + // to make sure an earlier join in the path is optimized first. + for (Mutable inputOpRef : op.getInputs()) { + joinFoundAndOptimizationApplied = checkAndApplyJoinTransformation(inputOpRef, context); + if (joinFoundAndOptimizationApplied) { + return true; + } + } + + // For a JOIN case, try to transform the given plan. + if (isThisOpInnerJoin || isThisOpLeftOuterJoin) { + // Restore the information from this operator since it might have been be set to null + // if there are other join operators in the earlier path. + joinRef = joinRefFromThisOp; + joinOp = joinOpFromThisOp; + + boolean continueCheck = true; + + // Already checked? If not, this operator may be optimized. + if (context.checkIfInDontApplySet(this, joinOp)) { + continueCheck = false; + } + + // For each access method, this contains the information about + // whether an available index can be applicable or not. + Map analyzedAMs = null; + if (continueCheck) { + analyzedAMs = new HashMap<>(); + } + + // Check the condition of JOIN operator is a function call since only function call can be transformed + // using available indexes. If so, initialize the subtree information that will be used later to decide + // whether the given plan is truly optimizable or not. + if (continueCheck && !checkJoinOpConditionAndInitSubTree(context)) { + continueCheck = false; + } + + // Analyze the condition of SELECT operator and initialize analyzedAMs. + // Check whether the function in the SELECT operator can be truly transformed. + boolean matchInLeftSubTree = false; + boolean matchInRightSubTree = false; + if (continueCheck) { + if (leftSubTree.hasDataSource()) { + matchInLeftSubTree = analyzeSelectOrJoinOpConditionAndUpdateAnalyzedAM(joinCond, + leftSubTree.getAssignsAndUnnests(), analyzedAMs, context, typeEnvironment); + } + if (rightSubTree.hasDataSource()) { + matchInRightSubTree = analyzeSelectOrJoinOpConditionAndUpdateAnalyzedAM(joinCond, + rightSubTree.getAssignsAndUnnests(), analyzedAMs, context, typeEnvironment); + } + } + + // Find the dataset from the data-source and the record type of the dataset from the metadata. + // This will be used to find an applicable index on the dataset. + boolean checkLeftSubTreeMetadata = false; + boolean checkRightSubTreeMetadata = false; + if (continueCheck && (matchInLeftSubTree || matchInRightSubTree)) { + // Set dataset and type metadata. + if (matchInLeftSubTree) { + checkLeftSubTreeMetadata = leftSubTree.setDatasetAndTypeMetadata(metadataProvider); + } + if (matchInRightSubTree) { + checkRightSubTreeMetadata = rightSubTree.setDatasetAndTypeMetadata(metadataProvider); + } + } + + if (continueCheck && (checkLeftSubTreeMetadata || checkRightSubTreeMetadata)) { + // Map variables to the applicable indexes and find the field name and type. + // Then find the applicable indexes for the variables used in the JOIN condition. + if (checkLeftSubTreeMetadata) { + fillSubTreeIndexExprs(leftSubTree, analyzedAMs, context); + } + if (checkRightSubTreeMetadata) { + fillSubTreeIndexExprs(rightSubTree, analyzedAMs, context); + } + + // Prune the access methods based on the function expression and access methods. + pruneIndexCandidates(analyzedAMs, context, typeEnvironment); + + // If the right subtree (inner branch) has indexes, one of those indexes will be used. + // Remove the indexes from the outer branch in the optimizer's consideration list for this rule. + pruneIndexCandidatesFromOuterBranch(analyzedAMs); + + // We are going to use indexes from the inner branch. + // If no index is available, then we stop here. + Pair chosenIndex = chooseBestIndex(analyzedAMs); + if (chosenIndex == null) { + context.addToDontApplySet(this, joinOp); + continueCheck = false; + } + + if (continueCheck) { + // Apply plan transformation using chosen index. + AccessMethodAnalysisContext analysisCtx = analyzedAMs.get(chosenIndex.first); + + // For LOJ with GroupBy, prepare objects to reset LOJ nullPlaceHolderVariable + // in GroupByOp. + if (isThisOpLeftOuterJoin && isParentOpGroupBy) { + analysisCtx.setLOJGroupbyOpRef(opRef); + ScalarFunctionCallExpression isNullFuncExpr = AccessMethodUtils + .findLOJIsMissingFuncInGroupBy((GroupByOperator) opRef.getValue()); + analysisCtx.setLOJIsNullFuncInGroupBy(isNullFuncExpr); + } + + Dataset indexDataset = analysisCtx.getDatasetFromIndexDatasetMap(chosenIndex.second); + + // We assume that the left subtree is the outer branch and the right subtree + // is the inner branch. This assumption holds true since we only use an index + // from the right subtree. The following is just a sanity check. + if (!rightSubTree.hasDataSourceScan() + && !indexDataset.getDatasetName().equals(rightSubTree.getDataset().getDatasetName())) { + return false; + } + + // Finally, try to apply plan transformation using chosen index. + boolean res = chosenIndex.first.applyJoinPlanTransformation(joinRef, leftSubTree, rightSubTree, + chosenIndex.second, analysisCtx, context, isThisOpLeftOuterJoin, isParentOpGroupBy); + + // If the plan transformation is successful, we don't need to traverse the plan + // any more, since if there are more JOIN operators, the next trigger on this plan + // will find them. + if (res) { + return res; + } + } + } + + joinRef = null; + joinOp = null; + } + + return false; + } + + /** + * After the pattern is matched, check the condition and initialize the data sources from the both sub trees. + * + * @throws AlgebricksException + */ + protected boolean checkJoinOpConditionAndInitSubTree(IOptimizationContext context) throws AlgebricksException { + + typeEnvironment = context.getOutputTypeEnvironment(joinOp); + + // Check that the join's condition is a function call. + ILogicalExpression condExpr = joinOp.getCondition().getValue(); + if (condExpr.getExpressionTag() != LogicalExpressionTag.FUNCTION_CALL) { + return false; + } + joinCond = (AbstractFunctionCallExpression) condExpr; + + boolean leftSubTreeInitialized = leftSubTree.initFromSubTree(joinOp.getInputs().get(0)); + boolean rightSubTreeInitialized = rightSubTree.initFromSubTree(joinOp.getInputs().get(1)); + + if (!leftSubTreeInitialized || !rightSubTreeInitialized) { + return false; + } + + // One of the subtrees must have a datasource scan. + if (leftSubTree.hasDataSourceScan() || rightSubTree.hasDataSourceScan()) { + return true; + } + return false; + } + } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/c4eb6564/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/IntroduceLSMComponentFilterRule.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/IntroduceLSMComponentFilterRule.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/IntroduceLSMComponentFilterRule.java index 5f9b4ab..18641e5 100644 --- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/IntroduceLSMComponentFilterRule.java +++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/IntroduceLSMComponentFilterRule.java @@ -83,7 +83,7 @@ public class IntroduceLSMComponentFilterRule implements IAlgebraicRewriteRule { typeEnvironment = context.getOutputTypeEnvironment(op); ILogicalExpression condExpr = ((SelectOperator) op).getCondition().getValue(); AccessMethodAnalysisContext analysisCtx = analyzeCondition(condExpr, context, typeEnvironment); - if (analysisCtx.matchedFuncExprs.isEmpty()) { + if (analysisCtx.getMatchedFuncExprs().isEmpty()) { return false; } @@ -106,8 +106,8 @@ public class IntroduceLSMComponentFilterRule implements IAlgebraicRewriteRule { List optFuncExprs = new ArrayList<>(); - for (int i = 0; i < analysisCtx.matchedFuncExprs.size(); i++) { - IOptimizableFuncExpr optFuncExpr = analysisCtx.matchedFuncExprs.get(i); + for (int i = 0; i < analysisCtx.getMatchedFuncExprs().size(); i++) { + IOptimizableFuncExpr optFuncExpr = analysisCtx.getMatchedFuncExpr(i); boolean found = findMacthedExprFieldName(optFuncExpr, op, dataset, recType, datasetIndexes, context); if (found && optFuncExpr.getFieldName(0).equals(filterFieldName)) { optFuncExprs.add(optFuncExpr); @@ -299,7 +299,8 @@ public class IntroduceLSMComponentFilterRule implements IAlgebraicRewriteRule { if (funcIdent == AlgebricksBuiltinFunctions.LE || funcIdent == AlgebricksBuiltinFunctions.GE || funcIdent == AlgebricksBuiltinFunctions.LT || funcIdent == AlgebricksBuiltinFunctions.GT || funcIdent == AlgebricksBuiltinFunctions.EQ) { - AccessMethodUtils.analyzeFuncExprArgsForOneConstAndVar(funcExpr, analysisCtx, context, typeEnvironment); + AccessMethodUtils.analyzeFuncExprArgsForOneConstAndVarAndUpdateAnalysisCtx(funcExpr, analysisCtx, context, + typeEnvironment); } } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/c4eb6564/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/IntroduceSelectAccessMethodRule.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/IntroduceSelectAccessMethodRule.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/IntroduceSelectAccessMethodRule.java index 1d332b6..d95b278 100644 --- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/IntroduceSelectAccessMethodRule.java +++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/IntroduceSelectAccessMethodRule.java @@ -89,7 +89,7 @@ public class IntroduceSelectAccessMethodRule extends AbstractIntroduceAccessMeth protected List> afterSelectRefs = null; // Register access methods. - protected static Map> accessMethods = new HashMap>(); + protected static Map> accessMethods = new HashMap<>(); static { registerAccessMethod(BTreeAccessMethod.INSTANCE, accessMethods); @@ -102,32 +102,35 @@ public class IntroduceSelectAccessMethodRule extends AbstractIntroduceAccessMeth * with SELECT operator into an index-utilized plan. */ @Override - public boolean rewritePost(Mutable opRef, IOptimizationContext context) + public boolean rewritePre(Mutable opRef, IOptimizationContext context) throws AlgebricksException { clear(); setMetadataDeclarations(context); AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue(); + + // Already checked? if (context.checkIfInDontApplySet(this, op)) { return false; } - //We start at the top of the plan + // We start at the top of the plan. Thus, check whether this operator is the root, + // which is DISTRIBUTE_RESULT, SINK, or COMMIT since we start the process from the root operator. if (op.getOperatorTag() != LogicalOperatorTag.DISTRIBUTE_RESULT && op.getOperatorTag() != LogicalOperatorTag.SINK && op.getOperatorTag() != LogicalOperatorTag.DELEGATE_OPERATOR) { return false; } + if (op.getOperatorTag() == LogicalOperatorTag.DELEGATE_OPERATOR && !(((DelegateOperator) op).getDelegate() instanceof CommitOperator)) { return false; } afterSelectRefs = new ArrayList<>(); - // Recursively check the given plan whether the desired pattern exists in it. // If so, try to optimize the plan. - boolean planTransformed = checkAndApplyTheSelectTransformationRule(opRef, context); + boolean planTransformed = checkAndApplyTheSelectTransformation(opRef, context); if (selectOp != null) { // We found an optimization here. Don't need to optimize this operator again. @@ -135,78 +138,44 @@ public class IntroduceSelectAccessMethodRule extends AbstractIntroduceAccessMeth } if (!planTransformed) { - // We found an optimization here. Don't need to optimize this operator again. return false; } else { OperatorPropertiesUtil.typeOpRec(opRef, context); - } return planTransformed; } - protected boolean checkAndApplyTheSelectTransformationRule(Mutable opRef, - IOptimizationContext context) throws AlgebricksException { - AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue(); - - // Match operator pattern and initialize operator members. - if (matchesOperatorPattern(opRef, context)) { - // Analyze select condition. - Map analyzedAMs = new TreeMap<>(); - if (!analyzeCondition(selectCond, subTree.getAssignsAndUnnests(), analyzedAMs, context, typeEnvironment)) { - return false; - } - - // Set dataset and type metadata. - if (!subTree.setDatasetAndTypeMetadata((MetadataProvider) context.getMetadataProvider())) { - return false; - } - - fillSubTreeIndexExprs(subTree, analyzedAMs, context); - pruneIndexCandidates(analyzedAMs, context, typeEnvironment); - - // Choose index to be applied. - List> chosenIndexes = chooseAllIndex(analyzedAMs); - if (chosenIndexes == null || chosenIndexes.isEmpty()) { - context.addToDontApplySet(this, selectOp); - return false; - } - - // Apply plan transformation using chosen index. - boolean res = intersectAllSecondaryIndexes(chosenIndexes, analyzedAMs, context); - - context.addToDontApplySet(this, selectOp); - if (res) { - OperatorPropertiesUtil.typeOpRec(opRef, context); - return res; - } - selectRef = null; - selectOp = null; - afterSelectRefs.add(opRef); - - } else { - // This is not a SELECT operator. Remember operators - afterSelectRefs.add(opRef); - - } - // Recursively check the plan and try to optimize it. - boolean selectFoundAndOptimizationApplied = false; - for (Mutable inputOpRef : op.getInputs()) { - boolean foundHere = checkAndApplyTheSelectTransformationRule(inputOpRef, context); - if (foundHere) { - selectFoundAndOptimizationApplied = true; - } + /** + * Check that the given SELECT condition is a function call. + * Call initSubTree() to initialize the optimizable subtree that collects information from + * the operators below the given SELECT operator. + * In order to transform the given plan, a datasource should be configured + * since we are going to transform a datasource into an unnest-map operator. + */ + protected boolean checkSelectOpConditionAndInitSubTree(IOptimizationContext context) throws AlgebricksException { + // Set and analyze select. + ILogicalExpression condExpr = selectOp.getCondition().getValue(); + typeEnvironment = context.getOutputTypeEnvironment(selectOp); + if (condExpr.getExpressionTag() != LogicalExpressionTag.FUNCTION_CALL) { + return false; } + selectCond = (AbstractFunctionCallExpression) condExpr; - // Clean the path after SELECT operator by removing the current operator in the list. - afterSelectRefs.remove(opRef); - return selectFoundAndOptimizationApplied; - + // Initialize the subtree information. + // Match and put assign, unnest, and datasource information. + boolean res = subTree.initFromSubTree(selectOp.getInputs().get(0)); + return res && subTree.hasDataSourceScan(); } + /** + * Construct all applicable secondary index-based access paths in the given selection plan and + * intersect them using INTERSECT operator to guide to the common primary index search. + * In case where the applicable index is one, we only construct one path. + */ private boolean intersectAllSecondaryIndexes(List> chosenIndexes, Map analyzedAMs, IOptimizationContext context) - throws AlgebricksException { + throws AlgebricksException { Pair chosenIndex = null; Optional> primaryIndex = chosenIndexes.stream() .filter(pair -> pair.second.isPrimaryIndex()).findFirst(); @@ -235,22 +204,26 @@ public class IntroduceSelectAccessMethodRule extends AbstractIntroduceAccessMeth .getExecutionMode() == ExecutionMode.UNPARTITIONED, context)); } - ILogicalOperator primaryUnnest = connectAll2ndarySearchPlanWithIntersect(subRoots, context); + // Connect each secondary index utilization plan to a common intersect operator. + ILogicalOperator primaryUnnestOp = connectAll2ndarySearchPlanWithIntersect(subRoots, context); - subTree.getDataSourceRef().setValue(primaryUnnest); - return primaryUnnest != null; + subTree.getDataSourceRef().setValue(primaryUnnestOp); + return primaryUnnestOp != null; } + /** + * Connect each secondary index utilization plan to a common INTERSECT operator. + */ private ILogicalOperator connectAll2ndarySearchPlanWithIntersect(List subRoots, IOptimizationContext context) throws AlgebricksException { ILogicalOperator lop = subRoots.get(0); List> inputVars = new ArrayList<>(subRoots.size()); for (int i = 0; i < subRoots.size(); i++) { if (lop.getOperatorTag() != subRoots.get(i).getOperatorTag()) { - throw new AlgebricksException("The data source root should have the same operator type"); + throw new AlgebricksException("The data source root should have the same operator type."); } if (lop.getInputs().size() != 1) { - throw new AlgebricksException("The primary search has multiple input"); + throw new AlgebricksException("The primary search has multiple inputs."); } ILogicalOperator curRoot = subRoots.get(i); @@ -259,7 +232,8 @@ public class IntroduceSelectAccessMethodRule extends AbstractIntroduceAccessMeth for (Pair> orderExpression : order .getOrderExpressions()) { if (orderExpression.second.getValue().getExpressionTag() != LogicalExpressionTag.VARIABLE) { - throw new AlgebricksException("It should not happen, the order by expression is not variables"); + throw new AlgebricksException( + "The order by expression should be variables, but they aren't variables."); } VariableReferenceExpression orderedVar = (VariableReferenceExpression) orderExpression.second .getValue(); @@ -278,31 +252,128 @@ public class IntroduceSelectAccessMethodRule extends AbstractIntroduceAccessMeth return lop; } - protected boolean matchesOperatorPattern(Mutable opRef, IOptimizationContext context) - throws AlgebricksException { - // First check that the operator is a select and its condition is a function call. - AbstractLogicalOperator op1 = (AbstractLogicalOperator) opRef.getValue(); - if (context.checkIfInDontApplySet(this, op1)) { - return false; + /** + * Recursively traverse the given plan and check whether a SELECT operator exists. + * If one is found, maintain the path from the root to SELECT operator and + * optimize the path from the SELECT operator to the EMPTY_TUPLE_SOURCE operator + * if it is not already optimized. + */ + protected boolean checkAndApplyTheSelectTransformation(Mutable opRef, + IOptimizationContext context) throws AlgebricksException { + AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue(); + boolean selectFoundAndOptimizationApplied; + boolean isSelectOp = false; + + Mutable selectRefFromThisOp = null; + SelectOperator selectOpFromThisOp = null; + + // Check the current operator pattern to see whether it is a JOIN or not. + if (op.getOperatorTag() == LogicalOperatorTag.SELECT) { + selectRef = opRef; + selectOp = (SelectOperator) op; + selectRefFromThisOp = opRef; + selectOpFromThisOp = (SelectOperator) op; + isSelectOp = true; + } else { + // This is not a SELECT operator. Remember this operator. + afterSelectRefs.add(opRef); } - if (op1.getOperatorTag() != LogicalOperatorTag.SELECT) { - return false; + + // Recursively check the plan and try to optimize it. We first check the children of the given operator + // to make sure an earlier select in the path is optimized first. + for (Mutable inputOpRef : op.getInputs()) { + selectFoundAndOptimizationApplied = checkAndApplyTheSelectTransformation(inputOpRef, context); + if (selectFoundAndOptimizationApplied) { + return true; + } } - // Set and analyze select. - selectRef = opRef; - selectOp = (SelectOperator) op1; - typeEnvironment = context.getOutputTypeEnvironment(op1); - // Check that the select's condition is a function call. - ILogicalExpression condExpr = selectOp.getCondition().getValue(); - if (condExpr.getExpressionTag() != LogicalExpressionTag.FUNCTION_CALL) { - return false; + // Traverse the plan until we find a SELECT operator. + if (isSelectOp) { + // Restore the information from this operator since it might have been be set to null + // if there are other select operators in the earlier path. + selectRef = selectRefFromThisOp; + selectOp = selectOpFromThisOp; + + // Decides the plan transformation check needs to be continued. + // This variable is needed since we can't just return false + // in order to keep this operator in the afterSelectRefs list. + boolean continueCheck = true; + + // Already checked this SELECT operator? If not, this operator may be optimized. + if (context.checkIfInDontApplySet(this, selectOp)) { + continueCheck = false; + } + + // For each access method, contains the information about + // whether an available index can be applicable or not. + Map analyzedAMs = null; + if (continueCheck) { + analyzedAMs = new TreeMap<>(); + } + + // Check the condition of SELECT operator is a function call since + // only function call can be transformed using available indexes. + // If so, initialize the subtree information that will be used later to decide whether + // the given plan is truly optimizable or not. + if (continueCheck && !checkSelectOpConditionAndInitSubTree(context)) { + continueCheck = false; + } + + // Analyze the condition of SELECT operator and initialize analyzedAMs. + // Check whether the function in the SELECT operator can be truly transformed. + if (continueCheck && !analyzeSelectOrJoinOpConditionAndUpdateAnalyzedAM(selectCond, + subTree.getAssignsAndUnnests(), analyzedAMs, context, typeEnvironment)) { + continueCheck = false; + } + + // Find the dataset from the data-source and + // the record type of the dataset from the metadata. + // This will be used to find an applicable index on the dataset. + if (continueCheck && !subTree.setDatasetAndTypeMetadata((MetadataProvider) context.getMetadataProvider())) { + continueCheck = false; + } + + if (continueCheck) { + // Map variables to the applicable indexes and find the field name and type. + // Then find the applicable indexes for the variables used in the SELECT condition. + fillSubTreeIndexExprs(subTree, analyzedAMs, context); + + // Prune the access methods based on the function expression and access methods. + pruneIndexCandidates(analyzedAMs, context, typeEnvironment); + + // Choose all indexes that will be applied. + List> chosenIndexes = chooseAllIndexes(analyzedAMs); + + if (chosenIndexes == null || chosenIndexes.isEmpty()) { + // We can't apply any index for this SELECT operator + context.addToDontApplySet(this, selectRef.getValue()); + return false; + } + + // Apply plan transformation using chosen index. + boolean res = intersectAllSecondaryIndexes(chosenIndexes, analyzedAMs, context); + context.addToDontApplySet(this, selectOp); + + if (res) { + OperatorPropertiesUtil.typeOpRec(opRef, context); + return res; + } + } + + selectRef = null; + selectOp = null; + afterSelectRefs.add(opRef); } - selectCond = (AbstractFunctionCallExpression) condExpr; - boolean res = subTree.initFromSubTree(op1.getInputs().get(0)); - return res && subTree.hasDataSourceScan(); + + // Clean the path after SELECT operator by removing the current operator in the list. + afterSelectRefs.remove(opRef); + + return false; + } + @Override public Map> getAccessMethods() { return accessMethods; @@ -313,5 +384,6 @@ public class IntroduceSelectAccessMethodRule extends AbstractIntroduceAccessMeth selectRef = null; selectOp = null; selectCond = null; + subTree.reset(); } } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/c4eb6564/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/InvertedIndexAccessMethod.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/InvertedIndexAccessMethod.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/InvertedIndexAccessMethod.java index 7328f97..a02679d 100644 --- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/InvertedIndexAccessMethod.java +++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/InvertedIndexAccessMethod.java @@ -28,6 +28,8 @@ import java.util.Map; import org.apache.asterix.common.annotations.SkipSecondaryIndexSearchExpressionAnnotation; import org.apache.asterix.common.config.DatasetConfig.IndexType; import org.apache.asterix.common.exceptions.AsterixException; +import org.apache.asterix.common.exceptions.CompilationException; +import org.apache.asterix.common.exceptions.ErrorCode; import org.apache.asterix.dataflow.data.common.ExpressionTypeComputer; import org.apache.asterix.formats.nontagged.BinaryTokenizerFactoryProvider; import org.apache.asterix.lang.common.util.FunctionUtil; @@ -135,17 +137,17 @@ public class InvertedIndexAccessMethod implements IAccessMethod { } @Override - public boolean analyzeFuncExprArgs(AbstractFunctionCallExpression funcExpr, + public boolean analyzeFuncExprArgsAndUpdateAnalysisCtx(AbstractFunctionCallExpression funcExpr, List assignsAndUnnests, AccessMethodAnalysisContext analysisCtx, IOptimizationContext context, IVariableTypeEnvironment typeEnvironment) throws AlgebricksException { if (funcExpr.getFunctionIdentifier() == BuiltinFunctions.STRING_CONTAINS || funcExpr.getFunctionIdentifier() == BuiltinFunctions.FULLTEXT_CONTAINS || funcExpr.getFunctionIdentifier() == BuiltinFunctions.FULLTEXT_CONTAINS_WO_OPTION) { - boolean matches = AccessMethodUtils.analyzeFuncExprArgsForOneConstAndVar(funcExpr, analysisCtx, context, - typeEnvironment); + boolean matches = AccessMethodUtils.analyzeFuncExprArgsForOneConstAndVarAndUpdateAnalysisCtx(funcExpr, + analysisCtx, context, typeEnvironment); if (!matches) { - matches = AccessMethodUtils.analyzeFuncExprArgsForTwoVars(funcExpr, analysisCtx); + matches = AccessMethodUtils.analyzeFuncExprArgsForTwoVarsAndUpdateAnalysisCtx(funcExpr, analysisCtx); } return matches; } @@ -265,13 +267,13 @@ public class InvertedIndexAccessMethod implements IAccessMethod { OptimizableFuncExpr newOptFuncExpr = new OptimizableFuncExpr(funcExpr, new LogicalVariable[] { fieldVarExpr1, fieldVarExpr2 }, new ILogicalExpression[] { arg3 }, new IAType[] { (IAType) ExpressionTypeComputer.INSTANCE.getType(arg3, null, null) }); - for (IOptimizableFuncExpr optFuncExpr : analysisCtx.matchedFuncExprs) { + for (IOptimizableFuncExpr optFuncExpr : analysisCtx.getMatchedFuncExprs()) { //avoid additional optFuncExpressions in case of a join if (optFuncExpr.getFuncExpr().equals(funcExpr)) { return true; } } - analysisCtx.matchedFuncExprs.add(newOptFuncExpr); + analysisCtx.addMatchedFuncExpr(newOptFuncExpr); return true; } @@ -315,13 +317,13 @@ public class InvertedIndexAccessMethod implements IAccessMethod { new ILogicalExpression[] { constArg, arg3 }, new IAType[] { (IAType) ExpressionTypeComputer.INSTANCE.getType(constArg, null, null), (IAType) ExpressionTypeComputer.INSTANCE.getType(arg3, null, null) }); - for (IOptimizableFuncExpr optFuncExpr : analysisCtx.matchedFuncExprs) { + for (IOptimizableFuncExpr optFuncExpr : analysisCtx.getMatchedFuncExprs()) { //avoid additional optFuncExpressions in case of a join if (optFuncExpr.getFuncExpr().equals(funcExpr)) { return true; } } - analysisCtx.matchedFuncExprs.add(newOptFuncExpr); + analysisCtx.addMatchedFuncExpr(newOptFuncExpr); return true; } @@ -466,7 +468,7 @@ public class InvertedIndexAccessMethod implements IAccessMethod { AccessMethodAnalysisContext analysisCtx, IOptimizationContext context, boolean isLeftOuterJoin, boolean hasGroupBy) throws AlgebricksException { // Figure out if the index is applicable on the left or right side (if both, we arbitrarily prefer the left side). - Dataset dataset = analysisCtx.indexDatasetMap.get(chosenIndex); + Dataset dataset = analysisCtx.getDatasetFromIndexDatasetMap(chosenIndex); OptimizableOperatorSubTree indexSubTree; OptimizableOperatorSubTree probeSubTree; @@ -595,7 +597,7 @@ public class InvertedIndexAccessMethod implements IAccessMethod { ILogicalExpression joinCond, IOptimizableFuncExpr optFuncExpr, List originalSubTreePKs, List surrogateSubTreePKs, IOptimizationContext context) throws AlgebricksException { - probeSubTree.getPrimaryKeyVars(originalSubTreePKs); + probeSubTree.getPrimaryKeyVars(null, originalSubTreePKs); // Create two copies of the original probe subtree. // The first copy, which becomes the new probe subtree, will retain the primary-key and secondary-search key variables, @@ -775,7 +777,7 @@ public class InvertedIndexAccessMethod implements IAccessMethod { break; } default: { - throw new AlgebricksException("Only strings, ordered and unordered list types supported."); + throw CompilationException.create(ErrorCode.NO_SUPPORTED_TYPE); } } @@ -828,7 +830,7 @@ public class InvertedIndexAccessMethod implements IAccessMethod { typeTag = ((AUnionType) type).getActualType().getTypeTag(); } if (typeTag != ATypeTag.ORDEREDLIST && typeTag != ATypeTag.STRING && typeTag != ATypeTag.UNORDEREDLIST) { - throw new AlgebricksException("Only ordered lists, string, and unordered lists types supported."); + throw CompilationException.create(ErrorCode.NO_SUPPORTED_TYPE); } } jobGenParams.setSearchKeyType(typeTag); @@ -1226,7 +1228,7 @@ public class InvertedIndexAccessMethod implements IAccessMethod { index.getGramLength(), prePost, false); } default: { - throw new AlgebricksException("Tokenizer not applicable to index kind '" + index.getIndexType() + "'."); + throw CompilationException.create(ErrorCode.NO_TOKENIZER_FOR_TYPE, index.getIndexType()); } } } @@ -1271,12 +1273,12 @@ public class InvertedIndexAccessMethod implements IAccessMethod { } } default: { - throw new AlgebricksException("Incompatible search modifier '" + searchModifierType - + "' for index type '" + index.getIndexType() + "'"); + throw CompilationException.create(ErrorCode.INCOMPATIBLE_SEARCH_MODIFIER, + searchModifierType, index.getIndexType()); } } default: - throw new AlgebricksException("Unknown search modifier type '" + searchModifierType + "'."); + throw CompilationException.create(ErrorCode.UNKNOWN_SEARCH_MODIFIER, searchModifierType); } }