asterixdb-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ti...@apache.org
Subject asterixdb git commit: ASTERIXDB-1573 Allow Extension of Rewrite Rules
Date Wed, 26 Oct 2016 00:06:50 GMT
Repository: asterixdb
Updated Branches:
  refs/heads/master 34d75c418 -> 109e48515


ASTERIXDB-1573 Allow Extension of Rewrite Rules

Add IRuleSetFactory which builds rewrite Rule Set
Can be extended by Asterix Extensions

Removed some obsolete extension code
Change-Id: I7f59dea86b0ef4ee9d31b56766a97bd2b34ec02c
Reviewed-on: https://asterix-gerrit.ics.uci.edu/1304
Tested-by: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Integration-Tests: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Reviewed-by: Till Westmann <tillw@apache.org>


Project: http://git-wip-us.apache.org/repos/asf/asterixdb/repo
Commit: http://git-wip-us.apache.org/repos/asf/asterixdb/commit/109e4851
Tree: http://git-wip-us.apache.org/repos/asf/asterixdb/tree/109e4851
Diff: http://git-wip-us.apache.org/repos/asf/asterixdb/diff/109e4851

Branch: refs/heads/master
Commit: 109e48515676ef04afb7d66e92e02c650f02ac44
Parents: 34d75c4
Author: Steven Glenn Jacobs <sjaco002@ucr.edu>
Authored: Mon Oct 24 09:40:31 2016 -0700
Committer: Till Westmann <tillw@apache.org>
Committed: Tue Oct 25 17:06:14 2016 -0700

----------------------------------------------------------------------
 .../asterix/algebra/base/ILangExtension.java    |  23 --
 .../extension/ExtensionFunctionIdentifier.java  |  78 -------
 .../extension/IAlgebraExtensionManager.java     |  11 -
 .../provider/AqlCompilationProvider.java        |   5 +
 .../provider/DefaultRuleSetFactory.java         |  86 ++++++++
 .../provider/ILangCompilationProvider.java      |   5 +
 .../compiler/provider/IRuleSetFactory.java      |  43 ++++
 .../provider/SqlppCompilationProvider.java      |   5 +
 .../asterix/optimizer/base/RuleCollections.java |   9 +-
 .../optimizer/rules/UnnestToDataScanRule.java   | 218 +++++++++----------
 .../apache/asterix/api/common/APIFramework.java |  61 +-----
 .../asterix/api/java/AsterixJavaClient.java     |   2 +-
 .../app/cc/CompilerExtensionManager.java        |  25 +--
 .../app/external/FeedWorkCollection.java        |   2 +-
 .../DefaultStatementExecutorFactory.java        |   9 +-
 .../asterix/app/translator/QueryTranslator.java |  13 +-
 .../org/apache/asterix/drivers/AsterixCLI.java  |   2 +-
 .../asterix/drivers/AsterixClientDriver.java    |   2 +-
 .../asterix/drivers/AsterixWebServer.java       |   2 +-
 .../aql/translator/QueryTranslatorTest.java     |   2 +-
 .../org/apache/asterix/test/dml/DmlTest.java    |   2 +-
 .../asterix/test/optimizer/OptimizerTest.java   |   8 +-
 22 files changed, 272 insertions(+), 341 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/asterixdb/blob/109e4851/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/algebra/base/ILangExtension.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/algebra/base/ILangExtension.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/algebra/base/ILangExtension.java
index ac3bc03..4803691 100644
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/algebra/base/ILangExtension.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/algebra/base/ILangExtension.java
@@ -21,13 +21,6 @@ package org.apache.asterix.algebra.base;
 
 import org.apache.asterix.common.api.IExtension;
 import org.apache.asterix.compiler.provider.ILangCompilationProvider;
-import org.apache.commons.lang3.mutable.Mutable;
-import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
-import org.apache.hyracks.algebricks.core.algebra.base.ILogicalExpression;
-import org.apache.hyracks.algebricks.core.algebra.base.ILogicalOperator;
-import org.apache.hyracks.algebricks.core.algebra.base.IOptimizationContext;
-import org.apache.hyracks.algebricks.core.algebra.expressions.AbstractFunctionCallExpression;
-import org.apache.hyracks.algebricks.core.algebra.operators.logical.UnnestOperator;
 
 /**
  * An interface for language extensions
@@ -45,20 +38,4 @@ public interface ILangExtension extends IExtension {
     }
 
     ILangCompilationProvider getLangCompilationProvider(Language lang);
-
-    //TODO(amoudi/yingyi) this is not a good way to extend re-write rules. introduce rewrite-rule-provider.
-    /**
-     * Called by the compiler when the unnest function is an extension function.
-     * Provides a way to add additional types of datasources
-     *
-     * @param opRef
-     * @param context
-     * @param unnestOp
-     * @param unnestExpr
-     * @param functionCallExpr
-     * @return true if transformation was successful, false otherwise
-     * @throws AlgebricksException
-     */
-    boolean unnestToDataScan(Mutable<ILogicalOperator> opRef, IOptimizationContext context, UnnestOperator unnestOp,
-            ILogicalExpression unnestExpr, AbstractFunctionCallExpression functionCallExpr) throws AlgebricksException;
 }

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/109e4851/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/algebra/extension/ExtensionFunctionIdentifier.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/algebra/extension/ExtensionFunctionIdentifier.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/algebra/extension/ExtensionFunctionIdentifier.java
deleted file mode 100644
index ae67337..0000000
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/algebra/extension/ExtensionFunctionIdentifier.java
+++ /dev/null
@@ -1,78 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.asterix.algebra.extension;
-
-import org.apache.asterix.common.api.ExtensionId;
-import org.apache.hyracks.algebricks.core.algebra.functions.FunctionIdentifier;
-
-/**
- * an Identifier for an extension function
- */
-public class ExtensionFunctionIdentifier extends FunctionIdentifier {
-    //TODO (till): remove this classs
-
-    private static final long serialVersionUID = 1L;
-    private final ExtensionId extensionId;
-
-    /**
-     * Create an identifier for an external function
-     *
-     * @param namespace
-     * @param name
-     * @param extensionId
-     */
-    public ExtensionFunctionIdentifier(String namespace, String name, ExtensionId extensionId) {
-        super(namespace, name);
-        this.extensionId = extensionId;
-    }
-
-    /**
-     * Create an identifier for an external function
-     *
-     * @param namespace
-     * @param name
-     * @param arity
-     * @param extensionId
-     */
-    public ExtensionFunctionIdentifier(String namespace, String name, int arity, ExtensionId extensionId) {
-        super(namespace, name, arity);
-        this.extensionId = extensionId;
-    }
-
-    public ExtensionId getExtensionId() {
-        return extensionId;
-    }
-
-    @Override
-    public int hashCode() {
-        return super.hashCode() + extensionId.hashCode();
-    }
-
-    @Override
-    public boolean equals(Object o) {
-        if (this == o) {
-            return true;
-        }
-        if (o instanceof ExtensionFunctionIdentifier) {
-            ExtensionFunctionIdentifier oId = (ExtensionFunctionIdentifier) o;
-            return super.equals(o) && extensionId.equals(oId.getExtensionId());
-        }
-        return false;
-    }
-}

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/109e4851/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/algebra/extension/IAlgebraExtensionManager.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/algebra/extension/IAlgebraExtensionManager.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/algebra/extension/IAlgebraExtensionManager.java
index 0a8402c..d9641df 100644
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/algebra/extension/IAlgebraExtensionManager.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/algebra/extension/IAlgebraExtensionManager.java
@@ -18,17 +18,6 @@
  */
 package org.apache.asterix.algebra.extension;
 
-import org.apache.commons.lang3.mutable.Mutable;
-import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
-import org.apache.hyracks.algebricks.core.algebra.base.ILogicalExpression;
-import org.apache.hyracks.algebricks.core.algebra.base.ILogicalOperator;
-import org.apache.hyracks.algebricks.core.algebra.base.IOptimizationContext;
-import org.apache.hyracks.algebricks.core.algebra.expressions.AbstractFunctionCallExpression;
-import org.apache.hyracks.algebricks.core.algebra.operators.logical.UnnestOperator;
-
 public interface IAlgebraExtensionManager {
 
-    boolean unnestToDataScan(Mutable<ILogicalOperator> opRef, IOptimizationContext context, UnnestOperator unnestOp,
-            ILogicalExpression unnestExpr, AbstractFunctionCallExpression functionCallExpr) throws AlgebricksException;
-
 }

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/109e4851/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/compiler/provider/AqlCompilationProvider.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/compiler/provider/AqlCompilationProvider.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/compiler/provider/AqlCompilationProvider.java
index 00affa8..af0ba73 100644
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/compiler/provider/AqlCompilationProvider.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/compiler/provider/AqlCompilationProvider.java
@@ -49,4 +49,9 @@ public class AqlCompilationProvider implements ILangCompilationProvider {
         return new AqlExpressionToPlanTranslatorFactory();
     }
 
+    @Override
+    public IRuleSetFactory getRuleSetFactory() {
+        return new DefaultRuleSetFactory();
+    }
+
 }

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/109e4851/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/compiler/provider/DefaultRuleSetFactory.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/compiler/provider/DefaultRuleSetFactory.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/compiler/provider/DefaultRuleSetFactory.java
new file mode 100644
index 0000000..f6526c4
--- /dev/null
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/compiler/provider/DefaultRuleSetFactory.java
@@ -0,0 +1,86 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.compiler.provider;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.asterix.optimizer.base.RuleCollections;
+import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
+import org.apache.hyracks.algebricks.common.utils.Pair;
+import org.apache.hyracks.algebricks.compiler.rewriter.rulecontrollers.SequentialFixpointRuleController;
+import org.apache.hyracks.algebricks.compiler.rewriter.rulecontrollers.SequentialOnceRuleController;
+import org.apache.hyracks.algebricks.core.rewriter.base.AbstractRuleController;
+import org.apache.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
+
+public class DefaultRuleSetFactory implements IRuleSetFactory {
+
+    @Override
+    public List<Pair<AbstractRuleController, List<IAlgebraicRewriteRule>>> getLogicalRewrites()
+            throws AlgebricksException {
+        return buildLogical();
+    }
+
+    @Override
+    public List<Pair<AbstractRuleController, List<IAlgebraicRewriteRule>>> getPhysicalRewrites() {
+        return buildPhysical();
+    }
+
+    public static List<Pair<AbstractRuleController, List<IAlgebraicRewriteRule>>> buildLogical() {
+        List<Pair<AbstractRuleController, List<IAlgebraicRewriteRule>>> defaultLogicalRewrites = new ArrayList<>();
+        SequentialFixpointRuleController seqCtrlNoDfs = new SequentialFixpointRuleController(false);
+        SequentialFixpointRuleController seqCtrlFullDfs = new SequentialFixpointRuleController(true);
+        SequentialOnceRuleController seqOnceCtrl = new SequentialOnceRuleController(true);
+        defaultLogicalRewrites.add(new Pair<>(seqOnceCtrl, RuleCollections.buildInitialTranslationRuleCollection()));
+        defaultLogicalRewrites.add(new Pair<>(seqOnceCtrl, RuleCollections.buildTypeInferenceRuleCollection()));
+        defaultLogicalRewrites.add(new Pair<>(seqOnceCtrl, RuleCollections.buildAutogenerateIDRuleCollection()));
+        defaultLogicalRewrites
+            .add(new Pair<>(seqCtrlFullDfs, RuleCollections.buildNormalizationRuleCollection()));
+        defaultLogicalRewrites
+                .add(new Pair<>(seqCtrlNoDfs, RuleCollections.buildCondPushDownAndJoinInferenceRuleCollection()));
+        defaultLogicalRewrites.add(new Pair<>(seqCtrlFullDfs, RuleCollections.buildLoadFieldsRuleCollection()));
+        // fj
+        defaultLogicalRewrites.add(new Pair<>(seqCtrlFullDfs, RuleCollections.buildFuzzyJoinRuleCollection()));
+        //
+        defaultLogicalRewrites.add(new Pair<>(seqCtrlFullDfs, RuleCollections.buildNormalizationRuleCollection()));
+        defaultLogicalRewrites
+                .add(new Pair<>(seqCtrlNoDfs, RuleCollections.buildCondPushDownAndJoinInferenceRuleCollection()));
+        defaultLogicalRewrites.add(new Pair<>(seqCtrlFullDfs, RuleCollections.buildLoadFieldsRuleCollection()));
+        defaultLogicalRewrites.add(new Pair<>(seqOnceCtrl, RuleCollections.buildDataExchangeRuleCollection()));
+        defaultLogicalRewrites.add(new Pair<>(seqCtrlNoDfs, RuleCollections.buildConsolidationRuleCollection()));
+        defaultLogicalRewrites.add(new Pair<>(seqCtrlNoDfs, RuleCollections.buildAccessMethodRuleCollection()));
+        defaultLogicalRewrites.add(new Pair<>(seqCtrlNoDfs, RuleCollections.buildPlanCleanupRuleCollection()));
+
+        //put TXnRuleCollection!
+        return defaultLogicalRewrites;
+    }
+
+    public static List<Pair<AbstractRuleController, List<IAlgebraicRewriteRule>>> buildPhysical() {
+        List<Pair<AbstractRuleController, List<IAlgebraicRewriteRule>>> defaultPhysicalRewrites = new ArrayList<>();
+        SequentialOnceRuleController seqOnceCtrl = new SequentialOnceRuleController(true);
+        SequentialOnceRuleController seqOnceTopLevel = new SequentialOnceRuleController(false);
+        defaultPhysicalRewrites
+                .add(new Pair<>(seqOnceCtrl, RuleCollections.buildPhysicalRewritesAllLevelsRuleCollection()));
+        defaultPhysicalRewrites
+                .add(new Pair<>(seqOnceTopLevel, RuleCollections.buildPhysicalRewritesTopLevelRuleCollection()));
+        defaultPhysicalRewrites.add(new Pair<>(seqOnceCtrl, RuleCollections.prepareForJobGenRuleCollection()));
+        return defaultPhysicalRewrites;
+    }
+
+}

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/109e4851/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/compiler/provider/ILangCompilationProvider.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/compiler/provider/ILangCompilationProvider.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/compiler/provider/ILangCompilationProvider.java
index fbb55e9..f658be9 100644
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/compiler/provider/ILangCompilationProvider.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/compiler/provider/ILangCompilationProvider.java
@@ -45,4 +45,9 @@ public interface ILangCompilationProvider {
      */
     public ILangExpressionToPlanTranslatorFactory getExpressionToPlanTranslatorFactory();
 
+    /**
+     * @return the rule set factory of a language implementation
+     */
+    public IRuleSetFactory getRuleSetFactory();
+
 }

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/109e4851/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/compiler/provider/IRuleSetFactory.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/compiler/provider/IRuleSetFactory.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/compiler/provider/IRuleSetFactory.java
new file mode 100644
index 0000000..af07355
--- /dev/null
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/compiler/provider/IRuleSetFactory.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.compiler.provider;
+
+import java.util.List;
+
+import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
+import org.apache.hyracks.algebricks.common.utils.Pair;
+import org.apache.hyracks.algebricks.core.rewriter.base.AbstractRuleController;
+import org.apache.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
+
+public interface IRuleSetFactory {
+
+    /**
+     * @return the logical rewrites
+     * @throws AlgebricksException
+     */
+    public List<Pair<AbstractRuleController, List<IAlgebraicRewriteRule>>> getLogicalRewrites()
+            throws AlgebricksException;
+
+    /**
+     * @return the physical rewrites
+     */
+    public List<Pair<AbstractRuleController, List<IAlgebraicRewriteRule>>> getPhysicalRewrites()
+            throws AlgebricksException;
+
+}

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/109e4851/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/compiler/provider/SqlppCompilationProvider.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/compiler/provider/SqlppCompilationProvider.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/compiler/provider/SqlppCompilationProvider.java
index 2b270df..70f8f92 100644
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/compiler/provider/SqlppCompilationProvider.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/compiler/provider/SqlppCompilationProvider.java
@@ -49,4 +49,9 @@ public class SqlppCompilationProvider implements ILangCompilationProvider {
         return new SqlppExpressionToPlanTranslatorFactory();
     }
 
+    @Override
+    public IRuleSetFactory getRuleSetFactory() {
+        return new DefaultRuleSetFactory();
+    }
+
 }

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/109e4851/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/base/RuleCollections.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/base/RuleCollections.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/base/RuleCollections.java
index 56a3bd0..3efc4dc 100644
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/base/RuleCollections.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/base/RuleCollections.java
@@ -22,7 +22,6 @@ package org.apache.asterix.optimizer.base;
 import java.util.LinkedList;
 import java.util.List;
 
-import org.apache.asterix.algebra.extension.IAlgebraExtensionManager;
 import org.apache.asterix.optimizer.rules.AddEquivalenceClassForRecordConstructorRule;
 import org.apache.asterix.optimizer.rules.AsterixExtractFunctionsFromJoinConditionRule;
 import org.apache.asterix.optimizer.rules.AsterixInlineVariablesRule;
@@ -66,10 +65,10 @@ import org.apache.asterix.optimizer.rules.RemoveRedundantListifyRule;
 import org.apache.asterix.optimizer.rules.RemoveRedundantSelectRule;
 import org.apache.asterix.optimizer.rules.RemoveSortInFeedIngestionRule;
 import org.apache.asterix.optimizer.rules.RemoveUnusedOneToOneEquiJoinRule;
-import org.apache.asterix.optimizer.rules.SetupCommitExtensionOpRule;
 import org.apache.asterix.optimizer.rules.ResolveVariableRule;
 import org.apache.asterix.optimizer.rules.SetAsterixPhysicalOperatorsRule;
 import org.apache.asterix.optimizer.rules.SetClosedRecordConstructorsRule;
+import org.apache.asterix.optimizer.rules.SetupCommitExtensionOpRule;
 import org.apache.asterix.optimizer.rules.SimilarityCheckRule;
 import org.apache.asterix.optimizer.rules.SweepIllegalNonfunctionalFunctions;
 import org.apache.asterix.optimizer.rules.UnnestToDataScanRule;
@@ -155,9 +154,7 @@ public final class RuleCollections {
         return autogen;
     }
 
-    //TODO(amoudi/yingyi): refactor this to use a provider instead of passing the extensionManager
-    public static final List<IAlgebraicRewriteRule> buildNormalizationRuleCollection(
-            IAlgebraExtensionManager algebraExtensionManager) {
+    public static final List<IAlgebraicRewriteRule> buildNormalizationRuleCollection() {
         List<IAlgebraicRewriteRule> normalization = new LinkedList<>();
         normalization.add(new ResolveVariableRule());
         normalization.add(new IntroduceUnnestForCollectionToSequenceRule());
@@ -183,7 +180,7 @@ public final class RuleCollections {
         normalization.add(new ListifyUnnestingFunctionRule());
         normalization.add(new ConstantFoldingRule());
         normalization.add(new RemoveRedundantSelectRule());
-        normalization.add(new UnnestToDataScanRule(algebraExtensionManager));
+        normalization.add(new UnnestToDataScanRule());
         normalization.add(new MetaFunctionToMetaVariableRule());
         normalization.add(new FuzzyEqRule());
         normalization.add(new SimilarityCheckRule());

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/109e4851/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/UnnestToDataScanRule.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/UnnestToDataScanRule.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/UnnestToDataScanRule.java
index c033214..3503f56 100644
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/UnnestToDataScanRule.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/UnnestToDataScanRule.java
@@ -21,7 +21,6 @@ package org.apache.asterix.optimizer.rules;
 import java.util.ArrayList;
 import java.util.List;
 
-import org.apache.asterix.algebra.extension.IAlgebraExtensionManager;
 import org.apache.asterix.common.config.DatasetConfig.DatasetType;
 import org.apache.asterix.external.feed.watch.FeedActivityDetails;
 import org.apache.asterix.external.util.ExternalDataUtils;
@@ -68,11 +67,6 @@ import org.apache.hyracks.algebricks.core.algebra.properties.FunctionalDependenc
 import org.apache.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
 
 public class UnnestToDataScanRule implements IAlgebraicRewriteRule {
-    private final IAlgebraExtensionManager algebraExtensionManager;
-
-    public UnnestToDataScanRule(IAlgebraExtensionManager algebraExtensionManager) {
-        this.algebraExtensionManager = algebraExtensionManager;
-    }
 
     @Override
     public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context)
@@ -89,124 +83,114 @@ public class UnnestToDataScanRule implements IAlgebraicRewriteRule {
         }
         UnnestOperator unnest = (UnnestOperator) op;
         ILogicalExpression unnestExpr = unnest.getExpressionRef().getValue();
+        if (unnestExpr.getExpressionTag() != LogicalExpressionTag.FUNCTION_CALL) {
+            return false;
+        }
+        return handleFunction(opRef, context, unnest, (AbstractFunctionCallExpression) unnestExpr);
+    }
 
-        if (unnestExpr.getExpressionTag() == LogicalExpressionTag.FUNCTION_CALL) {
-            AbstractFunctionCallExpression f = (AbstractFunctionCallExpression) unnestExpr;
-            FunctionIdentifier fid = f.getFunctionIdentifier();
-
-            if (fid.equals(AsterixBuiltinFunctions.DATASET)) {
-                if (unnest.getPositionalVariable() != null) {
-                    // TODO remove this after enabling the support of positional variables in data scan
-                    throw new AlgebricksException("No positional variables are allowed over datasets.");
-                }
-                ILogicalExpression expr = f.getArguments().get(0).getValue();
-                if (expr.getExpressionTag() != LogicalExpressionTag.CONSTANT) {
-                    return false;
-                }
-                ConstantExpression ce = (ConstantExpression) expr;
-                IAlgebricksConstantValue acv = ce.getValue();
-                if (!(acv instanceof AsterixConstantValue)) {
-                    return false;
-                }
-                AsterixConstantValue acv2 = (AsterixConstantValue) acv;
-                if (acv2.getObject().getType().getTypeTag() != ATypeTag.STRING) {
-                    return false;
-                }
-                String datasetArg = ((AString) acv2.getObject()).getStringValue();
-
-                AqlMetadataProvider metadataProvider = (AqlMetadataProvider) context.getMetadataProvider();
-                Pair<String, String> datasetReference = parseDatasetReference(metadataProvider, datasetArg);
-                String dataverseName = datasetReference.first;
-                String datasetName = datasetReference.second;
-                Dataset dataset = metadataProvider.findDataset(dataverseName, datasetName);
-                if (dataset == null) {
-                    throw new AlgebricksException(
-                            "Could not find dataset " + datasetName + " in dataverse " + dataverseName);
-                }
-
-                AqlSourceId asid = new AqlSourceId(dataverseName, datasetName);
-                List<LogicalVariable> variables = new ArrayList<LogicalVariable>();
-
-                if (dataset.getDatasetType() == DatasetType.INTERNAL) {
-                    int numPrimaryKeys = DatasetUtils.getPartitioningKeys(dataset).size();
-                    for (int i = 0; i < numPrimaryKeys; i++) {
-                        variables.add(context.newVar());
-                    }
-                }
-                variables.add(unnest.getVariable());
-                AqlDataSource dataSource = metadataProvider.findDataSource(asid);
-                boolean hasMeta = dataSource.hasMeta();
-                if (hasMeta) {
+    protected boolean handleFunction(Mutable<ILogicalOperator> opRef, IOptimizationContext context,
+            UnnestOperator unnest, AbstractFunctionCallExpression f) throws AlgebricksException {
+        FunctionIdentifier fid = f.getFunctionIdentifier();
+        if (fid.equals(AsterixBuiltinFunctions.DATASET)) {
+            if (unnest.getPositionalVariable() != null) {
+                // TODO remove this after enabling the support of positional variables in data scan
+                throw new AlgebricksException("No positional variables are allowed over datasets.");
+            }
+            ILogicalExpression expr = f.getArguments().get(0).getValue();
+            if (expr.getExpressionTag() != LogicalExpressionTag.CONSTANT) {
+                return false;
+            }
+            ConstantExpression ce = (ConstantExpression) expr;
+            IAlgebricksConstantValue acv = ce.getValue();
+            if (!(acv instanceof AsterixConstantValue)) {
+                return false;
+            }
+            AsterixConstantValue acv2 = (AsterixConstantValue) acv;
+            if (acv2.getObject().getType().getTypeTag() != ATypeTag.STRING) {
+                return false;
+            }
+            String datasetArg = ((AString) acv2.getObject()).getStringValue();
+            AqlMetadataProvider metadataProvider = (AqlMetadataProvider) context.getMetadataProvider();
+            Pair<String, String> datasetReference = parseDatasetReference(metadataProvider, datasetArg);
+            String dataverseName = datasetReference.first;
+            String datasetName = datasetReference.second;
+            Dataset dataset = metadataProvider.findDataset(dataverseName, datasetName);
+            if (dataset == null) {
+                throw new AlgebricksException(
+                        "Could not find dataset " + datasetName + " in dataverse " + dataverseName);
+            }
+            AqlSourceId asid = new AqlSourceId(dataverseName, datasetName);
+            List<LogicalVariable> variables = new ArrayList<>();
+            if (dataset.getDatasetType() == DatasetType.INTERNAL) {
+                int numPrimaryKeys = DatasetUtils.getPartitioningKeys(dataset).size();
+                for (int i = 0; i < numPrimaryKeys; i++) {
                     variables.add(context.newVar());
                 }
-                DataSourceScanOperator scan = new DataSourceScanOperator(variables, dataSource);
-                List<Mutable<ILogicalOperator>> scanInpList = scan.getInputs();
-                scanInpList.addAll(unnest.getInputs());
-                opRef.setValue(scan);
-                addPrimaryKey(variables, dataSource, context);
-                context.computeAndSetTypeEnvironmentForOperator(scan);
-
-                // Adds equivalence classes --- one equivalent class between a primary key
-                // variable and a record field-access expression.
-                IAType[] schemaTypes = dataSource.getSchemaTypes();
-                ARecordType recordType = (ARecordType) (hasMeta ? schemaTypes[schemaTypes.length - 2]
-                        : schemaTypes[schemaTypes.length - 1]);
-                ARecordType metaRecordType = (ARecordType) (hasMeta ? schemaTypes[schemaTypes.length - 1] : null);
-                EquivalenceClassUtils.addEquivalenceClassesForPrimaryIndexAccess(scan, variables, recordType,
-                        metaRecordType, dataset, context);
-                return true;
-            } else if (fid.equals(AsterixBuiltinFunctions.FEED_COLLECT)) {
-                if (unnest.getPositionalVariable() != null) {
-                    throw new AlgebricksException("No positional variables are allowed over feeds.");
-                }
-
-                String dataverse = ConstantExpressionUtil.getStringArgument(f, 0);
-                String sourceFeedName = ConstantExpressionUtil.getStringArgument(f, 1);
-                String getTargetFeed = ConstantExpressionUtil.getStringArgument(f, 2);
-                String subscriptionLocation = ConstantExpressionUtil.getStringArgument(f, 3);
-                String targetDataset = ConstantExpressionUtil.getStringArgument(f, 4);
-                String outputType = ConstantExpressionUtil.getStringArgument(f, 5);
-
-                AqlMetadataProvider metadataProvider = (AqlMetadataProvider) context.getMetadataProvider();
-
-                AqlSourceId asid = new AqlSourceId(dataverse, getTargetFeed);
-                String policyName = metadataProvider.getConfig().get(FeedActivityDetails.FEED_POLICY_NAME);
-                FeedPolicyEntity policy = metadataProvider.findFeedPolicy(dataverse, policyName);
+            }
+            variables.add(unnest.getVariable());
+            AqlDataSource dataSource = metadataProvider.findDataSource(asid);
+            boolean hasMeta = dataSource.hasMeta();
+            if (hasMeta) {
+                variables.add(context.newVar());
+            }
+            DataSourceScanOperator scan = new DataSourceScanOperator(variables, dataSource);
+            List<Mutable<ILogicalOperator>> scanInpList = scan.getInputs();
+            scanInpList.addAll(unnest.getInputs());
+            opRef.setValue(scan);
+            addPrimaryKey(variables, dataSource, context);
+            context.computeAndSetTypeEnvironmentForOperator(scan);
+            // Adds equivalence classes --- one equivalent class between a primary key
+            // variable and a record field-access expression.
+            IAType[] schemaTypes = dataSource.getSchemaTypes();
+            ARecordType recordType = (ARecordType) (hasMeta ? schemaTypes[schemaTypes.length - 2]
+                    : schemaTypes[schemaTypes.length - 1]);
+            ARecordType metaRecordType = (ARecordType) (hasMeta ? schemaTypes[schemaTypes.length - 1] : null);
+            EquivalenceClassUtils.addEquivalenceClassesForPrimaryIndexAccess(scan, variables, recordType,
+                    metaRecordType, dataset, context);
+            return true;
+        } else if (fid.equals(AsterixBuiltinFunctions.FEED_COLLECT)) {
+            if (unnest.getPositionalVariable() != null) {
+                throw new AlgebricksException("No positional variables are allowed over feeds.");
+            }
+            String dataverse = ConstantExpressionUtil.getStringArgument(f, 0);
+            String sourceFeedName = ConstantExpressionUtil.getStringArgument(f, 1);
+            String getTargetFeed = ConstantExpressionUtil.getStringArgument(f, 2);
+            String subscriptionLocation = ConstantExpressionUtil.getStringArgument(f, 3);
+            String targetDataset = ConstantExpressionUtil.getStringArgument(f, 4);
+            String outputType = ConstantExpressionUtil.getStringArgument(f, 5);
+            AqlMetadataProvider metadataProvider = (AqlMetadataProvider) context.getMetadataProvider();
+            AqlSourceId asid = new AqlSourceId(dataverse, getTargetFeed);
+            String policyName = metadataProvider.getConfig().get(FeedActivityDetails.FEED_POLICY_NAME);
+            FeedPolicyEntity policy = metadataProvider.findFeedPolicy(dataverse, policyName);
+            if (policy == null) {
+                policy = BuiltinFeedPolicies.getFeedPolicy(policyName);
                 if (policy == null) {
-                    policy = BuiltinFeedPolicies.getFeedPolicy(policyName);
-                    if (policy == null) {
-                        throw new AlgebricksException("Unknown feed policy:" + policyName);
-                    }
-                }
-
-                ArrayList<LogicalVariable> feedDataScanOutputVariables = new ArrayList<LogicalVariable>();
-
-                String csLocations = metadataProvider.getConfig().get(FeedActivityDetails.COLLECT_LOCATIONS);
-                List<LogicalVariable> pkVars = new ArrayList<>();
-                FeedDataSource ds = createFeedDataSource(asid, targetDataset, sourceFeedName, subscriptionLocation,
-                        metadataProvider, policy, outputType, csLocations, unnest.getVariable(), context, pkVars);
-                // The order for feeds is <Record-Meta-PK>
-                feedDataScanOutputVariables.add(unnest.getVariable());
-                // Does it produce meta?
-                if (ds.hasMeta()) {
-                    feedDataScanOutputVariables.add(context.newVar());
-                }
-                // Does it produce pk?
-                if (ds.isChange()) {
-                    feedDataScanOutputVariables.addAll(pkVars);
+                    throw new AlgebricksException("Unknown feed policy:" + policyName);
                 }
-
-                DataSourceScanOperator scan = new DataSourceScanOperator(feedDataScanOutputVariables, ds);
-                List<Mutable<ILogicalOperator>> scanInpList = scan.getInputs();
-                scanInpList.addAll(unnest.getInputs());
-                opRef.setValue(scan);
-                context.computeAndSetTypeEnvironmentForOperator(scan);
-                return true;
-            } else if (algebraExtensionManager != null) {
-                return algebraExtensionManager.unnestToDataScan(opRef, context, unnest, unnestExpr, f);
             }
+            ArrayList<LogicalVariable> feedDataScanOutputVariables = new ArrayList<>();
+            String csLocations = metadataProvider.getConfig().get(FeedActivityDetails.COLLECT_LOCATIONS);
+            List<LogicalVariable> pkVars = new ArrayList<>();
+            FeedDataSource ds = createFeedDataSource(asid, targetDataset, sourceFeedName, subscriptionLocation,
+                    metadataProvider, policy, outputType, csLocations, unnest.getVariable(), context, pkVars);
+            // The order for feeds is <Record-Meta-PK>
+            feedDataScanOutputVariables.add(unnest.getVariable());
+            // Does it produce meta?
+            if (ds.hasMeta()) {
+                feedDataScanOutputVariables.add(context.newVar());
+            }
+            // Does it produce pk?
+            if (ds.isChange()) {
+                feedDataScanOutputVariables.addAll(pkVars);
+            }
+            DataSourceScanOperator scan = new DataSourceScanOperator(feedDataScanOutputVariables, ds);
+            List<Mutable<ILogicalOperator>> scanInpList = scan.getInputs();
+            scanInpList.addAll(unnest.getInputs());
+            opRef.setValue(scan);
+            context.computeAndSetTypeEnvironmentForOperator(scan);
+            return true;
         }
-
         return false;
     }
 

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/109e4851/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/APIFramework.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/APIFramework.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/APIFramework.java
index eb1764c..31ef778 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/APIFramework.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/APIFramework.java
@@ -21,13 +21,11 @@ package org.apache.asterix.api.common;
 import java.io.IOException;
 import java.io.PrintWriter;
 import java.rmi.RemoteException;
-import java.util.ArrayList;
 import java.util.List;
 
 import org.apache.asterix.algebra.base.ILangExpressionToPlanTranslator;
 import org.apache.asterix.algebra.base.ILangExpressionToPlanTranslatorFactory;
 import org.apache.asterix.api.common.Job.SubmissionMode;
-import org.apache.asterix.app.cc.CompilerExtensionManager;
 import org.apache.asterix.app.result.ResultUtil;
 import org.apache.asterix.common.config.AsterixCompilerProperties;
 import org.apache.asterix.common.config.AsterixExternalProperties;
@@ -35,6 +33,7 @@ import org.apache.asterix.common.config.OptimizationConfUtil;
 import org.apache.asterix.common.exceptions.ACIDException;
 import org.apache.asterix.common.exceptions.AsterixException;
 import org.apache.asterix.compiler.provider.ILangCompilationProvider;
+import org.apache.asterix.compiler.provider.IRuleSetFactory;
 import org.apache.asterix.dataflow.data.common.AqlExpressionTypeComputer;
 import org.apache.asterix.dataflow.data.common.AqlMergeAggregationExpressionFactory;
 import org.apache.asterix.dataflow.data.common.AqlMissableTypeComputer;
@@ -50,7 +49,6 @@ import org.apache.asterix.lang.common.rewrites.LangRewritingContext;
 import org.apache.asterix.lang.common.statement.FunctionDecl;
 import org.apache.asterix.lang.common.statement.Query;
 import org.apache.asterix.metadata.declared.AqlMetadataProvider;
-import org.apache.asterix.optimizer.base.RuleCollections;
 import org.apache.asterix.runtime.job.listener.JobEventListenerFactory;
 import org.apache.asterix.runtime.util.AsterixAppContextInfo;
 import org.apache.asterix.transaction.management.service.transaction.JobIdFactory;
@@ -63,8 +61,6 @@ import org.apache.hyracks.algebricks.common.utils.Pair;
 import org.apache.hyracks.algebricks.compiler.api.HeuristicCompilerFactoryBuilder;
 import org.apache.hyracks.algebricks.compiler.api.ICompiler;
 import org.apache.hyracks.algebricks.compiler.api.ICompilerFactory;
-import org.apache.hyracks.algebricks.compiler.rewriter.rulecontrollers.SequentialFixpointRuleController;
-import org.apache.hyracks.algebricks.compiler.rewriter.rulecontrollers.SequentialOnceRuleController;
 import org.apache.hyracks.algebricks.core.algebra.base.ILogicalPlan;
 import org.apache.hyracks.algebricks.core.algebra.base.IOptimizationContext;
 import org.apache.hyracks.algebricks.core.algebra.expressions.IConflictingTypeResolver;
@@ -77,9 +73,7 @@ import org.apache.hyracks.algebricks.core.algebra.prettyprint.AlgebricksAppendab
 import org.apache.hyracks.algebricks.core.algebra.prettyprint.LogicalOperatorPrettyPrintVisitor;
 import org.apache.hyracks.algebricks.core.algebra.prettyprint.PlanPlotter;
 import org.apache.hyracks.algebricks.core.algebra.prettyprint.PlanPrettyPrinter;
-import org.apache.hyracks.algebricks.core.rewriter.base.AbstractRuleController;
 import org.apache.hyracks.algebricks.core.rewriter.base.AlgebricksOptimizationContext;
-import org.apache.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
 import org.apache.hyracks.algebricks.core.rewriter.base.IOptimizationContextFactory;
 import org.apache.hyracks.algebricks.core.rewriter.base.PhysicalOptimizationConfig;
 import org.apache.hyracks.api.client.IHyracksClientConnection;
@@ -96,56 +90,13 @@ public class APIFramework {
     private final IRewriterFactory rewriterFactory;
     private final IAstPrintVisitorFactory astPrintVisitorFactory;
     private final ILangExpressionToPlanTranslatorFactory translatorFactory;
-    private final CompilerExtensionManager cExtensionManager;
+    private final IRuleSetFactory ruleSetFactory;
 
-    public APIFramework(ILangCompilationProvider compilationProvider, CompilerExtensionManager cExtensionManager) {
+    public APIFramework(ILangCompilationProvider compilationProvider) {
         this.rewriterFactory = compilationProvider.getRewriterFactory();
         this.astPrintVisitorFactory = compilationProvider.getAstPrintVisitorFactory();
         this.translatorFactory = compilationProvider.getExpressionToPlanTranslatorFactory();
-        this.cExtensionManager = cExtensionManager;
-    }
-
-    private static List<Pair<AbstractRuleController, List<IAlgebraicRewriteRule>>>
-            buildDefaultLogicalRewrites(CompilerExtensionManager ccExtensionManager) {
-        List<Pair<AbstractRuleController, List<IAlgebraicRewriteRule>>> defaultLogicalRewrites = new ArrayList<>();
-        SequentialFixpointRuleController seqCtrlNoDfs = new SequentialFixpointRuleController(false);
-        SequentialFixpointRuleController seqCtrlFullDfs = new SequentialFixpointRuleController(true);
-        SequentialOnceRuleController seqOnceCtrl = new SequentialOnceRuleController(true);
-        defaultLogicalRewrites.add(new Pair<>(seqOnceCtrl, RuleCollections.buildInitialTranslationRuleCollection()));
-        defaultLogicalRewrites.add(new Pair<>(seqOnceCtrl, RuleCollections.buildTypeInferenceRuleCollection()));
-        defaultLogicalRewrites.add(new Pair<>(seqOnceCtrl, RuleCollections.buildAutogenerateIDRuleCollection()));
-        defaultLogicalRewrites
-                .add(new Pair<>(seqCtrlFullDfs, RuleCollections.buildNormalizationRuleCollection(ccExtensionManager)));
-        defaultLogicalRewrites
-                .add(new Pair<>(seqCtrlNoDfs, RuleCollections.buildCondPushDownAndJoinInferenceRuleCollection()));
-        defaultLogicalRewrites.add(new Pair<>(seqCtrlFullDfs, RuleCollections.buildLoadFieldsRuleCollection()));
-        // fj
-        defaultLogicalRewrites.add(new Pair<>(seqCtrlFullDfs, RuleCollections.buildFuzzyJoinRuleCollection()));
-        //
-        defaultLogicalRewrites
-                .add(new Pair<>(seqCtrlFullDfs, RuleCollections.buildNormalizationRuleCollection(ccExtensionManager)));
-        defaultLogicalRewrites
-                .add(new Pair<>(seqCtrlNoDfs, RuleCollections.buildCondPushDownAndJoinInferenceRuleCollection()));
-        defaultLogicalRewrites.add(new Pair<>(seqCtrlFullDfs, RuleCollections.buildLoadFieldsRuleCollection()));
-        defaultLogicalRewrites.add(new Pair<>(seqOnceCtrl, RuleCollections.buildDataExchangeRuleCollection()));
-        defaultLogicalRewrites.add(new Pair<>(seqCtrlNoDfs, RuleCollections.buildConsolidationRuleCollection()));
-        defaultLogicalRewrites.add(new Pair<>(seqCtrlNoDfs, RuleCollections.buildAccessMethodRuleCollection()));
-        defaultLogicalRewrites.add(new Pair<>(seqCtrlNoDfs, RuleCollections.buildPlanCleanupRuleCollection()));
-
-        //put TXnRuleCollection!
-        return defaultLogicalRewrites;
-    }
-
-    private static List<Pair<AbstractRuleController, List<IAlgebraicRewriteRule>>> buildDefaultPhysicalRewrites() {
-        List<Pair<AbstractRuleController, List<IAlgebraicRewriteRule>>> defaultPhysicalRewrites = new ArrayList<>();
-        SequentialOnceRuleController seqOnceCtrl = new SequentialOnceRuleController(true);
-        SequentialOnceRuleController seqOnceTopLevel = new SequentialOnceRuleController(false);
-        defaultPhysicalRewrites
-                .add(new Pair<>(seqOnceCtrl, RuleCollections.buildPhysicalRewritesAllLevelsRuleCollection()));
-        defaultPhysicalRewrites
-                .add(new Pair<>(seqOnceTopLevel, RuleCollections.buildPhysicalRewritesTopLevelRuleCollection()));
-        defaultPhysicalRewrites.add(new Pair<>(seqOnceCtrl, RuleCollections.prepareForJobGenRuleCollection()));
-        return defaultPhysicalRewrites;
+        this.ruleSetFactory = compilationProvider.getRuleSetFactory();
     }
 
     private static class AqlOptimizationContextFactory implements IOptimizationContextFactory {
@@ -258,8 +209,8 @@ public class APIFramework {
         HeuristicCompilerFactoryBuilder builder =
                 new HeuristicCompilerFactoryBuilder(AqlOptimizationContextFactory.INSTANCE);
         builder.setPhysicalOptimizationConfig(OptimizationConfUtil.getPhysicalOptimizationConfig());
-        builder.setLogicalRewrites(buildDefaultLogicalRewrites(cExtensionManager));
-        builder.setPhysicalRewrites(buildDefaultPhysicalRewrites());
+        builder.setLogicalRewrites(ruleSetFactory.getLogicalRewrites());
+        builder.setPhysicalRewrites(ruleSetFactory.getPhysicalRewrites());
         IDataFormat format = queryMetadataProvider.getFormat();
         ICompilerFactory compilerFactory = builder.create();
         builder.setExpressionEvalSizeComputer(format.getExpressionEvalSizeComputer());

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/109e4851/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/java/AsterixJavaClient.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/java/AsterixJavaClient.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/java/AsterixJavaClient.java
index b269ecf..fd9c6cd 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/java/AsterixJavaClient.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/java/AsterixJavaClient.java
@@ -56,7 +56,7 @@ public class AsterixJavaClient {
         this.queryText = queryText;
         this.writer = writer;
         this.compilationProvider = compilationProvider;
-        this.apiFramework = new APIFramework(compilationProvider, null);
+        this.apiFramework = new APIFramework(compilationProvider);
         this.statementExecutorFactory = statementExecutorFactory;
         parserFactory = compilationProvider.getParserFactory();
     }

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/109e4851/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/cc/CompilerExtensionManager.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/cc/CompilerExtensionManager.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/cc/CompilerExtensionManager.java
index af10394..4734841 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/cc/CompilerExtensionManager.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/cc/CompilerExtensionManager.java
@@ -24,7 +24,6 @@ import java.util.Map;
 
 import org.apache.asterix.algebra.base.ILangExtension;
 import org.apache.asterix.algebra.base.ILangExtension.Language;
-import org.apache.asterix.algebra.extension.ExtensionFunctionIdentifier;
 import org.apache.asterix.algebra.extension.IAlgebraExtensionManager;
 import org.apache.asterix.app.translator.DefaultStatementExecutorFactory;
 import org.apache.asterix.common.api.ExtensionId;
@@ -35,15 +34,7 @@ import org.apache.asterix.compiler.provider.AqlCompilationProvider;
 import org.apache.asterix.compiler.provider.ILangCompilationProvider;
 import org.apache.asterix.compiler.provider.SqlppCompilationProvider;
 import org.apache.asterix.translator.IStatementExecutorFactory;
-import org.apache.commons.lang3.mutable.Mutable;
-import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
 import org.apache.hyracks.algebricks.common.utils.Pair;
-import org.apache.hyracks.algebricks.core.algebra.base.ILogicalExpression;
-import org.apache.hyracks.algebricks.core.algebra.base.ILogicalOperator;
-import org.apache.hyracks.algebricks.core.algebra.base.IOptimizationContext;
-import org.apache.hyracks.algebricks.core.algebra.expressions.AbstractFunctionCallExpression;
-import org.apache.hyracks.algebricks.core.algebra.functions.FunctionIdentifier;
-import org.apache.hyracks.algebricks.core.algebra.operators.logical.UnnestOperator;
 import org.apache.hyracks.api.exceptions.HyracksDataException;
 
 /**
@@ -77,7 +68,7 @@ public class CompilerExtensionManager implements IAlgebraExtensionManager {
         Pair<ExtensionId, ILangCompilationProvider> aqlcp = null;
         Pair<ExtensionId, ILangCompilationProvider> sqlppcp = null;
         IStatementExecutorExtension see = null;
-        defaultQueryTranslatorFactory = new DefaultStatementExecutorFactory(this);
+        defaultQueryTranslatorFactory = new DefaultStatementExecutorFactory();
 
         if (list != null) {
             for (AsterixExtension extensionConf : list) {
@@ -139,18 +130,4 @@ public class CompilerExtensionManager implements IAlgebraExtensionManager {
     public ILangCompilationProvider getSqlppCompilationProvider() {
         return sqlppCompilationProvider;
     }
-
-    // TODO(amoudi/yingyi): this is not a good way to extend rewrite rules. introduce re-write rule provider
-    @Override
-    public boolean unnestToDataScan(Mutable<ILogicalOperator> opRef, IOptimizationContext context,
-            UnnestOperator unnestOp, ILogicalExpression unnestExpr, AbstractFunctionCallExpression functionCallExpr)
-            throws AlgebricksException {
-        FunctionIdentifier functionId = functionCallExpr.getFunctionIdentifier();
-        if (functionId instanceof ExtensionFunctionIdentifier) {
-            ILangExtension extension =
-                    (ILangExtension) extensions.get(((ExtensionFunctionIdentifier) functionId).getExtensionId());
-            return extension.unnestToDataScan(opRef, context, unnestOp, unnestExpr, functionCallExpr);
-        }
-        return false;
-    }
 }

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/109e4851/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/external/FeedWorkCollection.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/external/FeedWorkCollection.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/external/FeedWorkCollection.java
index 62f11df..414585a 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/external/FeedWorkCollection.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/external/FeedWorkCollection.java
@@ -70,7 +70,7 @@ public class FeedWorkCollection {
 
         private static class SubscribeFeedWorkRunnable implements Runnable {
 
-            private static final DefaultStatementExecutorFactory qtFactory = new DefaultStatementExecutorFactory(null);
+            private static final DefaultStatementExecutorFactory qtFactory = new DefaultStatementExecutorFactory();
             private final FeedConnectionRequest request;
             private final String[] locations;
 

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/109e4851/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/DefaultStatementExecutorFactory.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/DefaultStatementExecutorFactory.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/DefaultStatementExecutorFactory.java
index 1b10f3a..6cdf329 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/DefaultStatementExecutorFactory.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/DefaultStatementExecutorFactory.java
@@ -20,7 +20,6 @@ package org.apache.asterix.app.translator;
 
 import java.util.List;
 
-import org.apache.asterix.app.cc.CompilerExtensionManager;
 import org.apache.asterix.compiler.provider.ILangCompilationProvider;
 import org.apache.asterix.lang.common.base.Statement;
 import org.apache.asterix.translator.IStatementExecutorFactory;
@@ -28,16 +27,10 @@ import org.apache.asterix.translator.SessionConfig;
 
 public class DefaultStatementExecutorFactory implements IStatementExecutorFactory {
 
-    protected final CompilerExtensionManager cExtensionManager;
-
-    public DefaultStatementExecutorFactory(CompilerExtensionManager cExtensionManager) {
-        this.cExtensionManager = cExtensionManager;
-    }
-
     @Override
     public QueryTranslator create(List<Statement> aqlStatements, SessionConfig conf,
             ILangCompilationProvider compilationProvider) {
-        return new QueryTranslator(aqlStatements, conf, compilationProvider, cExtensionManager);
+        return new QueryTranslator(aqlStatements, conf, compilationProvider);
     }
 
 }

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/109e4851/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
index d5b1415..0b6d806 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
@@ -47,7 +47,6 @@ import org.apache.asterix.active.IActiveEntityEventsListener;
 import org.apache.asterix.algebra.extension.IExtensionStatement;
 import org.apache.asterix.api.common.APIFramework;
 import org.apache.asterix.api.http.servlet.APIServlet;
-import org.apache.asterix.app.cc.CompilerExtensionManager;
 import org.apache.asterix.app.external.ExternalIndexingOperations;
 import org.apache.asterix.app.external.FeedJoint;
 import org.apache.asterix.app.external.FeedOperations;
@@ -230,12 +229,12 @@ public class QueryTranslator extends AbstractLangTranslator implements IStatemen
     protected final APIFramework apiFramework;
     protected final IRewriterFactory rewriterFactory;
 
-    public QueryTranslator(List<Statement> aqlStatements, SessionConfig conf,
-            ILangCompilationProvider compliationProvider, CompilerExtensionManager ccExtensionManager) {
-        this.statements = aqlStatements;
+    public QueryTranslator(List<Statement> statements, SessionConfig conf,
+            ILangCompilationProvider compliationProvider) {
+        this.statements = statements;
         this.sessionConfig = conf;
-        this.declaredFunctions = getDeclaredFunctions(aqlStatements);
-        this.apiFramework = new APIFramework(compliationProvider, ccExtensionManager);
+        this.declaredFunctions = getDeclaredFunctions(statements);
+        this.apiFramework = new APIFramework(compliationProvider);
         this.rewriterFactory = compliationProvider.getRewriterFactory();
         activeDefaultDataverse = MetadataBuiltinEntities.DEFAULT_DATAVERSE;
     }
@@ -3123,7 +3122,7 @@ public class QueryTranslator extends AbstractLangTranslator implements IStatemen
         return (dataverse != null) ? dataverse : activeDefaultDataverse.getDataverseName();
     }
 
-    protected String getActiveDataverse(Identifier dataverse) {
+    public String getActiveDataverse(Identifier dataverse) {
         return getActiveDataverseName(dataverse != null ? dataverse.getValue() : null);
     }
 

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/109e4851/asterixdb/asterix-app/src/main/java/org/apache/asterix/drivers/AsterixCLI.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/drivers/AsterixCLI.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/drivers/AsterixCLI.java
index 486b4ab..9b9ab4a 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/drivers/AsterixCLI.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/drivers/AsterixCLI.java
@@ -58,7 +58,7 @@ public class AsterixCLI {
             for (String queryFile : options.args) {
                 Reader in = new FileReader(queryFile);
                 AsterixJavaClient ajc = new AsterixJavaClient(integrationUtil.getHyracksClientConnection(), in,
-                        compilationProvider, new DefaultStatementExecutorFactory(null));
+                        compilationProvider, new DefaultStatementExecutorFactory());
                 try {
                     ajc.compile(true, false, false, false, false, true, false);
                 } finally {

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/109e4851/asterixdb/asterix-app/src/main/java/org/apache/asterix/drivers/AsterixClientDriver.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/drivers/AsterixClientDriver.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/drivers/AsterixClientDriver.java
index e0a60c6..f05b67d 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/drivers/AsterixClientDriver.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/drivers/AsterixClientDriver.java
@@ -64,7 +64,7 @@ public class AsterixClientDriver {
         ILangCompilationProvider compilationProvider = new AqlCompilationProvider();
         FileReader reader = new FileReader(filename);
         AsterixJavaClient q = new AsterixJavaClient(hcc, reader, compilationProvider,
-                new DefaultStatementExecutorFactory(null));
+                new DefaultStatementExecutorFactory());
         q.compile(optimize, true, true, true, onlyPhysical, createBinaryRuntime, createBinaryRuntime);
         return q;
     }

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/109e4851/asterixdb/asterix-app/src/main/java/org/apache/asterix/drivers/AsterixWebServer.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/drivers/AsterixWebServer.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/drivers/AsterixWebServer.java
index 8bcceca..f0eac39 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/drivers/AsterixWebServer.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/drivers/AsterixWebServer.java
@@ -34,7 +34,7 @@ public class AsterixWebServer {
         server.setHandler(context);
 
         context.addServlet(new ServletHolder(new APIServlet(new AqlCompilationProvider(),
-                new SqlppCompilationProvider(), new DefaultStatementExecutorFactory(null))), "/*");
+                new SqlppCompilationProvider(), new DefaultStatementExecutorFactory())), "/*");
         server.start();
         server.join();
     }

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/109e4851/asterixdb/asterix-app/src/test/java/org/apache/asterix/aql/translator/QueryTranslatorTest.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/aql/translator/QueryTranslatorTest.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/aql/translator/QueryTranslatorTest.java
index 56d4191..53b8602 100644
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/aql/translator/QueryTranslatorTest.java
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/aql/translator/QueryTranslatorTest.java
@@ -69,7 +69,7 @@ public class QueryTranslatorTest {
         when(mockCluster.getMasterNode()).thenReturn(mockMasterNode);
         when(mockMasterNode.getClientIp()).thenReturn("127.0.0.1");
 
-        IStatementExecutor aqlTranslator = new DefaultStatementExecutorFactory(null).create(statements, mockSessionConfig,
+        IStatementExecutor aqlTranslator = new DefaultStatementExecutorFactory().create(statements, mockSessionConfig,
                 new AqlCompilationProvider());
         List<String> parameters = new ArrayList<String>();
         parameters.add("examples/pregelix-example-jar-with-dependencies.jar");

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/109e4851/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dml/DmlTest.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dml/DmlTest.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dml/DmlTest.java
index 34dab28..d5501c3 100644
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dml/DmlTest.java
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dml/DmlTest.java
@@ -61,7 +61,7 @@ public class DmlTest {
         Reader loadReader = new BufferedReader(
                 new InputStreamReader(new FileInputStream(LOAD_FOR_ENLIST_FILE), "UTF-8"));
         AsterixJavaClient asterixLoad = new AsterixJavaClient(integrationUtil.getHyracksClientConnection(), loadReader,
-                ERR, new AqlCompilationProvider(), new DefaultStatementExecutorFactory(null));
+                ERR, new AqlCompilationProvider(), new DefaultStatementExecutorFactory());
         try {
             asterixLoad.compile(true, false, false, false, false, true, false);
         } catch (AsterixException e) {

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/109e4851/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/optimizer/OptimizerTest.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/optimizer/OptimizerTest.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/optimizer/OptimizerTest.java
index 2716859..8db2bc0 100644
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/optimizer/OptimizerTest.java
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/optimizer/OptimizerTest.java
@@ -30,7 +30,6 @@ import java.util.logging.Logger;
 
 import org.apache.asterix.api.common.AsterixHyracksIntegrationUtil;
 import org.apache.asterix.api.java.AsterixJavaClient;
-import org.apache.asterix.app.cc.CompilerExtensionManager;
 import org.apache.asterix.app.translator.DefaultStatementExecutorFactory;
 import org.apache.asterix.common.config.GlobalConfig;
 import org.apache.asterix.common.exceptions.AsterixException;
@@ -39,10 +38,10 @@ import org.apache.asterix.compiler.provider.ILangCompilationProvider;
 import org.apache.asterix.compiler.provider.SqlppCompilationProvider;
 import org.apache.asterix.external.util.ExternalDataConstants;
 import org.apache.asterix.external.util.IdentitiyResolverFactory;
-import org.apache.asterix.runtime.util.AsterixAppContextInfo;
 import org.apache.asterix.test.base.AsterixTestHelper;
 import org.apache.asterix.test.common.TestHelper;
 import org.apache.asterix.test.runtime.HDFSCluster;
+import org.apache.asterix.translator.IStatementExecutorFactory;
 import org.apache.hyracks.api.client.IHyracksClientConnection;
 import org.junit.AfterClass;
 import org.junit.Assume;
@@ -76,6 +75,7 @@ public class OptimizerTest {
     private static final ILangCompilationProvider aqlCompilationProvider = new AqlCompilationProvider();
     private static final ILangCompilationProvider sqlppCompilationProvider = new SqlppCompilationProvider();
     protected static ILangCompilationProvider extensionLangCompilationProvider = null;
+    protected static IStatementExecutorFactory statementExecutorFactory = new DefaultStatementExecutorFactory();
 
     protected static AsterixHyracksIntegrationUtil integrationUtil = new AsterixHyracksIntegrationUtil();
 
@@ -178,9 +178,7 @@ public class OptimizerTest {
                 provider = extensionLangCompilationProvider;
             }
             IHyracksClientConnection hcc = integrationUtil.getHyracksClientConnection();
-            AsterixJavaClient asterix = new AsterixJavaClient(hcc, query, plan, provider,
-                    new DefaultStatementExecutorFactory(
-                            (CompilerExtensionManager) AsterixAppContextInfo.INSTANCE.getExtensionManager()));
+            AsterixJavaClient asterix = new AsterixJavaClient(hcc, query, plan, provider, statementExecutorFactory);
             try {
                 asterix.compile(true, false, false, true, true, false, false);
             } catch (AsterixException e) {


Mime
View raw message