asterixdb-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ima...@apache.org
Subject [47/51] [partial] incubator-asterixdb git commit: Change folder structure for Java repackage
Date Tue, 25 Aug 2015 16:44:35 GMT
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/34d81630/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/FuzzyEqRule.java
----------------------------------------------------------------------
diff --git a/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/FuzzyEqRule.java b/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/FuzzyEqRule.java
deleted file mode 100644
index 3a06168..0000000
--- a/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/FuzzyEqRule.java
+++ /dev/null
@@ -1,124 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.asterix.optimizer.rules;
-
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.commons.lang3.mutable.Mutable;
-import org.apache.commons.lang3.mutable.MutableObject;
-
-import edu.uci.ics.asterix.aql.util.FunctionUtils;
-import edu.uci.ics.asterix.metadata.declared.AqlMetadataProvider;
-import edu.uci.ics.asterix.om.base.IAObject;
-import edu.uci.ics.asterix.om.constants.AsterixConstantValue;
-import edu.uci.ics.asterix.om.functions.AsterixBuiltinFunctions;
-import edu.uci.ics.asterix.optimizer.base.FuzzyUtils;
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalExpressionTag;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.AbstractFunctionCallExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.ConstantExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.ScalarFunctionCallExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.functions.AlgebricksBuiltinFunctions;
-import edu.uci.ics.hyracks.algebricks.core.algebra.functions.FunctionIdentifier;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractBinaryJoinOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SelectOperator;
-import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
-
-public class FuzzyEqRule implements IAlgebraicRewriteRule {
-
-    @Override
-    public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context)
-            throws AlgebricksException {
-        AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue();
-
-        // current operator is INNERJOIN or LEFTOUTERJOIN or SELECT
-        Mutable<ILogicalExpression> expRef;
-        if (op.getOperatorTag() == LogicalOperatorTag.INNERJOIN
-                || op.getOperatorTag() == LogicalOperatorTag.LEFTOUTERJOIN) {
-            AbstractBinaryJoinOperator joinOp = (AbstractBinaryJoinOperator) op;
-            expRef = joinOp.getCondition();
-        } else if (op.getOperatorTag() == LogicalOperatorTag.SELECT) {
-            SelectOperator selectOp = (SelectOperator) op;
-            expRef = selectOp.getCondition();
-        } else {
-            return false;
-        }
-
-        AqlMetadataProvider metadataProvider = ((AqlMetadataProvider) context.getMetadataProvider());
-
-        IVariableTypeEnvironment env = context.getOutputTypeEnvironment(op);
-        if (expandFuzzyEq(expRef, context, env, metadataProvider)) {
-            context.computeAndSetTypeEnvironmentForOperator(op);
-            return true;
-        }
-        return false;
-    }
-
-    private boolean expandFuzzyEq(Mutable<ILogicalExpression> expRef, IOptimizationContext context,
-            IVariableTypeEnvironment env, AqlMetadataProvider metadataProvider) throws AlgebricksException {
-        ILogicalExpression exp = expRef.getValue();
-
-        if (exp.getExpressionTag() != LogicalExpressionTag.FUNCTION_CALL) {
-            return false;
-        }
-
-        boolean expanded = false;
-        AbstractFunctionCallExpression funcExp = (AbstractFunctionCallExpression) exp;
-        FunctionIdentifier fi = funcExp.getFunctionIdentifier();
-        if (fi.equals(AsterixBuiltinFunctions.FUZZY_EQ)) {
-            List<Mutable<ILogicalExpression>> inputExps = funcExp.getArguments();
-
-            String simFuncName = FuzzyUtils.getSimFunction(metadataProvider);
-            ArrayList<Mutable<ILogicalExpression>> similarityArgs = new ArrayList<Mutable<ILogicalExpression>>();
-            for (int i = 0; i < inputExps.size(); ++i) {
-                Mutable<ILogicalExpression> inputExpRef = inputExps.get(i);
-                similarityArgs.add(inputExpRef);
-            }
-
-            FunctionIdentifier simFunctionIdentifier = FuzzyUtils.getFunctionIdentifier(simFuncName);
-            ScalarFunctionCallExpression similarityExp = new ScalarFunctionCallExpression(
-                    FunctionUtils.getFunctionInfo(simFunctionIdentifier), similarityArgs);
-            // Add annotations from the original fuzzy-eq function.
-            similarityExp.getAnnotations().putAll(funcExp.getAnnotations());
-            ArrayList<Mutable<ILogicalExpression>> cmpArgs = new ArrayList<Mutable<ILogicalExpression>>();
-            cmpArgs.add(new MutableObject<ILogicalExpression>(similarityExp));
-            IAObject simThreshold = FuzzyUtils.getSimThreshold(metadataProvider, simFuncName);
-            cmpArgs.add(new MutableObject<ILogicalExpression>(new ConstantExpression(new AsterixConstantValue(
-                    simThreshold))));
-            ScalarFunctionCallExpression cmpExpr = FuzzyUtils.getComparisonExpr(simFuncName, cmpArgs);
-            expRef.setValue(cmpExpr);
-            return true;
-        } else if (fi.equals(AlgebricksBuiltinFunctions.AND) || fi.equals(AlgebricksBuiltinFunctions.OR)) {
-            for (int i = 0; i < 2; i++) {
-                if (expandFuzzyEq(funcExp.getArguments().get(i), context, env, metadataProvider)) {
-                    expanded = true;
-                }
-            }
-        }
-        return expanded;
-    }
-
-    @Override
-    public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException {
-        return false;
-    }
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/34d81630/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/FuzzyJoinRule.java
----------------------------------------------------------------------
diff --git a/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/FuzzyJoinRule.java b/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/FuzzyJoinRule.java
deleted file mode 100644
index 9bcfafa..0000000
--- a/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/FuzzyJoinRule.java
+++ /dev/null
@@ -1,404 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.asterix.optimizer.rules;
-
-import java.io.StringReader;
-import java.util.Collection;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Locale;
-
-import org.apache.commons.lang3.mutable.Mutable;
-import org.apache.commons.lang3.mutable.MutableObject;
-
-import edu.uci.ics.asterix.algebra.base.LogicalOperatorDeepCopyVisitor;
-import edu.uci.ics.asterix.aql.base.Clause;
-import edu.uci.ics.asterix.aql.expression.Identifier;
-import edu.uci.ics.asterix.aqlplus.parser.AQLPlusParser;
-import edu.uci.ics.asterix.aqlplus.parser.ParseException;
-import edu.uci.ics.asterix.common.exceptions.AsterixException;
-import edu.uci.ics.asterix.metadata.declared.AqlMetadataProvider;
-import edu.uci.ics.asterix.om.functions.AsterixBuiltinFunctions;
-import edu.uci.ics.asterix.om.types.IAType;
-import edu.uci.ics.asterix.om.types.TypeHelper;
-import edu.uci.ics.asterix.optimizer.base.FuzzyUtils;
-import edu.uci.ics.asterix.translator.AqlPlusExpressionToPlanTranslator;
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.Counter;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalExpressionTag;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.AbstractFunctionCallExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.ConstantExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IndexedNLJoinExpressionAnnotation;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.VariableReferenceExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.functions.AlgebricksBuiltinFunctions;
-import edu.uci.ics.hyracks.algebricks.core.algebra.functions.FunctionIdentifier;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractBinaryJoinOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.LeftOuterJoinOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SelectOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.visitors.VariableUtilities;
-import edu.uci.ics.hyracks.algebricks.core.algebra.util.OperatorPropertiesUtil;
-import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
-
-public class FuzzyJoinRule implements IAlgebraicRewriteRule {
-
-    private static HashSet<FunctionIdentifier> simFuncs = new HashSet<FunctionIdentifier>();
-    static {
-        simFuncs.add(AsterixBuiltinFunctions.SIMILARITY_JACCARD_CHECK);
-    }
-
-    private static final String AQLPLUS = ""
-            //
-            // -- - Stage 3 - --
-            //
-            + "((#RIGHT), "
-            + "  (join((#LEFT), "
-            //
-            // -- -- - Stage 2 - --
-            //
-            + "    ("
-            + "    join( "
-            + "      ( "
-            + "      #LEFT_1 "
-            + "      let $tokensUnrankedLeft := %s($$LEFT_1) "
-            + "      let $lenLeft := len($tokensUnrankedLeft) "
-            + "      let $tokensLeft := "
-            + "        for $token in $tokensUnrankedLeft "
-            + "        for $tokenRanked at $i in "
-            //
-            // -- -- -- - Stage 1 - --
-            //
-            // + "          #LEFT_2 "
-            // + "          let $id := $$LEFTPK_2 "
-            // + "          for $token in %s($$LEFT_2) "
-            + "          #RIGHT_2 "
-            + "          let $id := $$RIGHTPK_2 "
-            + "          for $token in %s($$RIGHT_2) "
-            + "          /*+ hash */ "
-            + "          group by $tokenGroupped := $token with $id "
-            + "          /*+ inmem 34 198608 */ "
-            + "          order by count($id), $tokenGroupped "
-            + "          return $tokenGroupped "
-            //
-            // -- -- -- -
-            //
-            + "        where $token = /*+ bcast */ $tokenRanked "
-            + "        order by $i "
-            + "        return $i "
-            + "      for $prefixTokenLeft in subset-collection($tokensLeft, 0, prefix-len-%s(len($tokensLeft), %ff)) "
-            + "      ),( "
-            + "      #RIGHT_1 "
-            + "      let $tokensUnrankedRight := %s($$RIGHT_1) "
-            + "      let $lenRight := len($tokensUnrankedRight) "
-            + "      let $tokensRight := "
-            + "        for $token in $tokensUnrankedRight "
-            + "        for $tokenRanked at $i in "
-            //
-            // -- -- -- - Stage 1 - --
-            //
-            // + "          #LEFT_3 "
-            // + "          let $id := $$LEFTPK_3 "
-            // + "          for $token in %s($$LEFT_3) "
-            + "          #RIGHT_3 "
-            + "          let $id := $$RIGHTPK_3 "
-            + "          for $token in %s($$RIGHT_3) "
-            + "          /*+ hash */ "
-            + "          group by $tokenGroupped := $token with $id "
-            + "          /*+ inmem 34 198608 */ "
-            + "          order by count($id), $tokenGroupped "
-            + "          return $tokenGroupped "
-            //
-            // -- -- -- -
-            //
-            + "        where $token = /*+ bcast */ $tokenRanked "
-            + "        order by $i "
-            + "        return $i "
-            + "      for $prefixTokenRight in subset-collection($tokensRight, 0, prefix-len-%s(len($tokensRight), %ff)) "
-            + "      ), $prefixTokenLeft = $prefixTokenRight) "
-            + "    let $sim := similarity-%s-prefix($lenLeft, $tokensLeft, $lenRight, $tokensRight, $prefixTokenLeft, %ff) "
-            + "    where $sim >= %ff " + "    /*+ hash*/ "
-            + "    group by $idLeft := $$LEFTPK_1, $idRight := $$RIGHTPK_1 with $sim "
-            //
-            // -- -- -
-            //
-            + "    ), $$LEFTPK = $idLeft)),  $$RIGHTPK = $idRight)";
-
-    private Collection<LogicalVariable> liveVars = new HashSet<LogicalVariable>();
-
-    @Override
-    public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context)
-            throws AlgebricksException {
-        AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue();
-        // current opperator is join
-        if (op.getOperatorTag() != LogicalOperatorTag.INNERJOIN
-                && op.getOperatorTag() != LogicalOperatorTag.LEFTOUTERJOIN) {
-            return false;
-        }
-
-        // Find GET_ITEM function.
-        AbstractBinaryJoinOperator joinOp = (AbstractBinaryJoinOperator) op;
-        Mutable<ILogicalExpression> expRef = joinOp.getCondition();
-        Mutable<ILogicalExpression> getItemExprRef = getSimilarityExpression(expRef);
-        if (getItemExprRef == null) {
-            return false;
-        }
-        // Check if the GET_ITEM function is on one of the supported similarity-check functions.
-        AbstractFunctionCallExpression getItemFuncExpr = (AbstractFunctionCallExpression) getItemExprRef.getValue();
-        Mutable<ILogicalExpression> argRef = getItemFuncExpr.getArguments().get(0);
-        AbstractFunctionCallExpression simFuncExpr = (AbstractFunctionCallExpression) argRef.getValue();
-        if (!simFuncs.contains(simFuncExpr.getFunctionIdentifier())) {
-            return false;
-        }
-        // Skip this rule based on annotations.
-        if (simFuncExpr.getAnnotations().containsKey(IndexedNLJoinExpressionAnnotation.INSTANCE)) {
-            return false;
-        }
-
-        List<Mutable<ILogicalOperator>> inputOps = joinOp.getInputs();
-        ILogicalOperator leftInputOp = inputOps.get(0).getValue();
-        ILogicalOperator rightInputOp = inputOps.get(1).getValue();
-
-        List<Mutable<ILogicalExpression>> inputExps = simFuncExpr.getArguments();
-
-        ILogicalExpression inputExp0 = inputExps.get(0).getValue();
-        ILogicalExpression inputExp1 = inputExps.get(1).getValue();
-
-        // left and right expressions are variables
-        if (inputExp0.getExpressionTag() != LogicalExpressionTag.VARIABLE
-                || inputExp1.getExpressionTag() != LogicalExpressionTag.VARIABLE) {
-            return false;
-        }
-
-        LogicalVariable inputVar0 = ((VariableReferenceExpression) inputExp0).getVariableReference();
-        LogicalVariable inputVar1 = ((VariableReferenceExpression) inputExp1).getVariableReference();
-
-        LogicalVariable leftInputVar;
-        LogicalVariable rightInputVar;
-
-        liveVars.clear();
-        VariableUtilities.getLiveVariables(leftInputOp, liveVars);
-        if (liveVars.contains(inputVar0)) {
-            leftInputVar = inputVar0;
-            rightInputVar = inputVar1;
-        } else {
-            leftInputVar = inputVar1;
-            rightInputVar = inputVar0;
-        }
-
-        List<LogicalVariable> leftInputPKs = context.findPrimaryKey(leftInputVar);
-        List<LogicalVariable> rightInputPKs = context.findPrimaryKey(rightInputVar);
-        // Bail if primary keys could not be inferred.
-        if (leftInputPKs == null || rightInputPKs == null) {
-            return false;
-        }
-        // primary key has only one variable
-        if (leftInputPKs.size() != 1 || rightInputPKs.size() != 1) {
-            return false;
-        }
-        IAType leftType = (IAType) context.getOutputTypeEnvironment(leftInputOp).getVarType(leftInputVar);
-        IAType rightType = (IAType) context.getOutputTypeEnvironment(rightInputOp).getVarType(rightInputVar);
-        // left-hand side and right-hand side of "~=" has the same type
-        IAType left2 = TypeHelper.getNonOptionalType(leftType);
-        IAType right2 = TypeHelper.getNonOptionalType(rightType);
-        if (!left2.deepEqual(right2)) {
-            return false;
-        }
-        //
-        // -- - FIRE - --
-        //
-        AqlMetadataProvider metadataProvider = ((AqlMetadataProvider) context.getMetadataProvider());
-        FunctionIdentifier funcId = FuzzyUtils.getTokenizer(leftType.getTypeTag());
-        String tokenizer;
-        if (funcId == null) {
-            tokenizer = "";
-        } else {
-            tokenizer = funcId.getName();
-        }
-
-        float simThreshold = FuzzyUtils.getSimThreshold(metadataProvider);
-        String simFunction = FuzzyUtils.getSimFunction(metadataProvider);
-
-        // finalize AQL+ query
-        String prepareJoin;
-        switch (joinOp.getJoinKind()) {
-            case INNER: {
-                prepareJoin = "join" + AQLPLUS;
-                break;
-            }
-            case LEFT_OUTER: {
-                // TODO To make it work for Left Outer Joins, we should permute
-                // the #LEFT and #RIGHT at the top of the AQL+ query. But, when
-                // doing this, the
-                // fuzzyjoin/user-vis-int-vis-user-lot-aqlplus_1.aql (the one
-                // doing 3-way fuzzy joins) gives a different result. But even
-                // if we don't change the FuzzyJoinRule, permuting the for
-                // clauses in fuzzyjoin/user-vis-int-vis-user-lot-aqlplus_1.aql
-                // leads to different results, which suggests there is some
-                // other sort of bug.
-                return false;
-                // prepareJoin = "loj" + AQLPLUS;
-                // break;
-            }
-            default: {
-                throw new IllegalStateException();
-            }
-        }
-        String aqlPlus = String.format(Locale.US, prepareJoin, tokenizer, tokenizer, simFunction, simThreshold,
-                tokenizer, tokenizer, simFunction, simThreshold, simFunction, simThreshold, simThreshold);
-
-        LogicalVariable leftPKVar = leftInputPKs.get(0);
-        LogicalVariable rightPKVar = rightInputPKs.get(0);
-
-        Counter counter = new Counter(context.getVarCounter());
-
-        AQLPlusParser parser = new AQLPlusParser(new StringReader(aqlPlus));
-        parser.initScope();
-        parser.setVarCounter(counter);
-        List<Clause> clauses;
-        try {
-            clauses = parser.Clauses();
-        } catch (ParseException e) {
-            throw new AlgebricksException(e);
-        }
-        // The translator will compile metadata internally. Run this compilation
-        // under the same transaction id as the "outer" compilation.
-        AqlPlusExpressionToPlanTranslator translator = new AqlPlusExpressionToPlanTranslator(
-                metadataProvider.getJobId(), metadataProvider, counter, null, null);
-
-        LogicalOperatorDeepCopyVisitor deepCopyVisitor = new LogicalOperatorDeepCopyVisitor(counter);
-
-        translator.addOperatorToMetaScope(new Identifier("#LEFT"), leftInputOp);
-        translator.addVariableToMetaScope(new Identifier("$$LEFT"), leftInputVar);
-        translator.addVariableToMetaScope(new Identifier("$$LEFTPK"), leftPKVar);
-
-        translator.addOperatorToMetaScope(new Identifier("#RIGHT"), rightInputOp);
-        translator.addVariableToMetaScope(new Identifier("$$RIGHT"), rightInputVar);
-        translator.addVariableToMetaScope(new Identifier("$$RIGHTPK"), rightPKVar);
-
-        translator.addOperatorToMetaScope(new Identifier("#LEFT_1"), deepCopyVisitor.deepCopy(leftInputOp, null));
-        translator.addVariableToMetaScope(new Identifier("$$LEFT_1"), deepCopyVisitor.varCopy(leftInputVar));
-        translator.addVariableToMetaScope(new Identifier("$$LEFTPK_1"), deepCopyVisitor.varCopy(leftPKVar));
-        deepCopyVisitor.updatePrimaryKeys(context);
-        deepCopyVisitor.reset();
-
-        // translator.addOperatorToMetaScope(new Identifier("#LEFT_2"),
-        // deepCopyVisitor.deepCopy(leftInputOp, null));
-        // translator.addVariableToMetaScope(new Identifier("$$LEFT_2"),
-        // deepCopyVisitor.varCopy(leftInputVar));
-        // translator.addVariableToMetaScope(new Identifier("$$LEFTPK_2"),
-        // deepCopyVisitor.varCopy(leftPKVar));
-        // deepCopyVisitor.updatePrimaryKeys(context);
-        // deepCopyVisitor.reset();
-        //
-        // translator.addOperatorToMetaScope(new Identifier("#LEFT_3"),
-        // deepCopyVisitor.deepCopy(leftInputOp, null));
-        // translator.addVariableToMetaScope(new Identifier("$$LEFT_3"),
-        // deepCopyVisitor.varCopy(leftInputVar));
-        // translator.addVariableToMetaScope(new Identifier("$$LEFTPK_3"),
-        // deepCopyVisitor.varCopy(leftPKVar));
-        // deepCopyVisitor.updatePrimaryKeys(context);
-        // deepCopyVisitor.reset();
-
-        translator.addOperatorToMetaScope(new Identifier("#RIGHT_1"), deepCopyVisitor.deepCopy(rightInputOp, null));
-        translator.addVariableToMetaScope(new Identifier("$$RIGHT_1"), deepCopyVisitor.varCopy(rightInputVar));
-        translator.addVariableToMetaScope(new Identifier("$$RIGHTPK_1"), deepCopyVisitor.varCopy(rightPKVar));
-        deepCopyVisitor.updatePrimaryKeys(context);
-        deepCopyVisitor.reset();
-
-        // TODO pick side to run Stage 1, currently always picks RIGHT side
-        translator.addOperatorToMetaScope(new Identifier("#RIGHT_2"), deepCopyVisitor.deepCopy(rightInputOp, null));
-        translator.addVariableToMetaScope(new Identifier("$$RIGHT_2"), deepCopyVisitor.varCopy(rightInputVar));
-        translator.addVariableToMetaScope(new Identifier("$$RIGHTPK_2"), deepCopyVisitor.varCopy(rightPKVar));
-        deepCopyVisitor.updatePrimaryKeys(context);
-        deepCopyVisitor.reset();
-
-        translator.addOperatorToMetaScope(new Identifier("#RIGHT_3"), deepCopyVisitor.deepCopy(rightInputOp, null));
-        translator.addVariableToMetaScope(new Identifier("$$RIGHT_3"), deepCopyVisitor.varCopy(rightInputVar));
-        translator.addVariableToMetaScope(new Identifier("$$RIGHTPK_3"), deepCopyVisitor.varCopy(rightPKVar));
-        deepCopyVisitor.updatePrimaryKeys(context);
-        deepCopyVisitor.reset();
-
-        ILogicalPlan plan;
-        try {
-            plan = translator.translate(clauses);
-        } catch (AsterixException e) {
-            throw new AlgebricksException(e);
-        }
-        context.setVarCounter(counter.get());
-
-        ILogicalOperator outputOp = plan.getRoots().get(0).getValue();
-
-        SelectOperator extraSelect = null;
-        if (getItemExprRef != expRef) {
-            // more than one join condition
-            getItemExprRef.setValue(ConstantExpression.TRUE);
-            switch (joinOp.getJoinKind()) {
-                case INNER: {
-                    extraSelect = new SelectOperator(expRef, false, null);
-                    extraSelect.getInputs().add(new MutableObject<ILogicalOperator>(outputOp));
-                    outputOp = extraSelect;
-                    break;
-                }
-                case LEFT_OUTER: {
-                    if (((AbstractLogicalOperator) outputOp).getOperatorTag() != LogicalOperatorTag.LEFTOUTERJOIN) {
-                        throw new IllegalStateException();
-                    }
-                    LeftOuterJoinOperator topJoin = (LeftOuterJoinOperator) outputOp;
-                    topJoin.getCondition().setValue(expRef.getValue());
-                    break;
-                }
-                default: {
-                    throw new IllegalStateException();
-                }
-            }
-        }
-        opRef.setValue(outputOp);
-        OperatorPropertiesUtil.typeOpRec(opRef, context);
-        return true;
-    }
-
-    /**
-     * Look for GET_ITEM function call.
-     */
-    private Mutable<ILogicalExpression> getSimilarityExpression(Mutable<ILogicalExpression> expRef) {
-        ILogicalExpression exp = expRef.getValue();
-        if (exp.getExpressionTag() == LogicalExpressionTag.FUNCTION_CALL) {
-            AbstractFunctionCallExpression funcExpr = (AbstractFunctionCallExpression) exp;
-            if (funcExpr.getFunctionIdentifier().equals(AsterixBuiltinFunctions.GET_ITEM)) {
-                return expRef;
-            }
-            if (funcExpr.getFunctionIdentifier().equals(AlgebricksBuiltinFunctions.AND)) {
-                for (int i = 0; i < 2; i++) {
-                    Mutable<ILogicalExpression> expRefRet = getSimilarityExpression(funcExpr.getArguments().get(i));
-                    if (expRefRet != null) {
-                        return expRefRet;
-                    }
-                }
-            }
-        }
-        return null;
-    }
-
-    @Override
-    public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException {
-        return false;
-    }
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/34d81630/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/IfElseToSwitchCaseFunctionRule.java
----------------------------------------------------------------------
diff --git a/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/IfElseToSwitchCaseFunctionRule.java b/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/IfElseToSwitchCaseFunctionRule.java
deleted file mode 100644
index cd18517..0000000
--- a/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/IfElseToSwitchCaseFunctionRule.java
+++ /dev/null
@@ -1,126 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.asterix.optimizer.rules;
-
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.commons.lang3.mutable.Mutable;
-import org.apache.commons.lang3.mutable.MutableObject;
-
-import edu.uci.ics.asterix.aql.util.FunctionUtils;
-import edu.uci.ics.asterix.om.functions.AsterixBuiltinFunctions;
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalExpressionTag;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.AbstractFunctionCallExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.ConstantExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.ScalarFunctionCallExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.VariableReferenceExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AssignOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SelectOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SubplanOperator;
-import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
-
-public class IfElseToSwitchCaseFunctionRule implements IAlgebraicRewriteRule {
-
-    @Override
-    public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException {
-        return false;
-    }
-
-    @Override
-    public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context)
-            throws AlgebricksException {
-        AbstractLogicalOperator op1 = (AbstractLogicalOperator) opRef.getValue();
-        if (op1.getOperatorTag() != LogicalOperatorTag.ASSIGN)
-            return false;
-
-        AssignOperator assignOp = (AssignOperator) op1;
-        List<Mutable<ILogicalExpression>> assignExprs = assignOp.getExpressions();
-        if (assignExprs.size() > 1)
-            return false;
-        ILogicalExpression expr = assignExprs.get(0).getValue();
-        if (expr.getExpressionTag() == LogicalExpressionTag.FUNCTION_CALL) {
-            AbstractFunctionCallExpression funcExpr = (AbstractFunctionCallExpression) expr;
-            if (!funcExpr.getFunctionIdentifier().equals(AsterixBuiltinFunctions.CONCAT_NON_NULL))
-                return false;
-        }
-
-        AbstractLogicalOperator op2 = (AbstractLogicalOperator) op1.getInputs().get(0).getValue();
-        if (op2.getOperatorTag() != LogicalOperatorTag.SUBPLAN)
-            return false;
-
-        SubplanOperator subplan = (SubplanOperator) op2;
-        List<ILogicalPlan> subPlans = subplan.getNestedPlans();
-        List<Mutable<ILogicalExpression>> arguments = new ArrayList<Mutable<ILogicalExpression>>();
-        for (ILogicalPlan plan : subPlans) {
-            List<Mutable<ILogicalOperator>> roots = plan.getRoots();
-
-            AbstractLogicalOperator nestedRoot = (AbstractLogicalOperator) roots.get(0).getValue();
-            if (nestedRoot.getOperatorTag() != LogicalOperatorTag.SELECT)
-                return false;
-            SelectOperator selectOp = (SelectOperator) nestedRoot;
-
-            AbstractLogicalOperator nestedNextOp = (AbstractLogicalOperator) nestedRoot.getInputs().get(0).getValue();
-            if (nestedNextOp.getOperatorTag() != LogicalOperatorTag.ASSIGN)
-                return false;
-            AssignOperator assignRoot = (AssignOperator) nestedNextOp;
-            Mutable<ILogicalExpression> actionExprRef = assignRoot.getExpressions().get(0);
-
-            arguments.add(selectOp.getCondition());
-            arguments.add(actionExprRef);
-            AbstractLogicalOperator nestedBottomOp = (AbstractLogicalOperator) assignRoot.getInputs().get(0).getValue();
-
-            if (nestedBottomOp.getOperatorTag() != LogicalOperatorTag.NESTEDTUPLESOURCE)
-                return false;
-        }
-
-        AbstractLogicalOperator op3 = (AbstractLogicalOperator) op2.getInputs().get(0).getValue();
-        if (op3.getOperatorTag() != LogicalOperatorTag.ASSIGN)
-            return false;
-
-        AssignOperator bottomAssign = (AssignOperator) op3;
-        LogicalVariable conditionVar = bottomAssign.getVariables().get(0);
-        Mutable<ILogicalExpression> switchCondition = new MutableObject<ILogicalExpression>(
-                new VariableReferenceExpression(conditionVar));
-        List<Mutable<ILogicalExpression>> argumentRefs = new ArrayList<Mutable<ILogicalExpression>>();
-        argumentRefs.add(switchCondition);
-        argumentRefs.addAll(arguments);
-
-        /** replace the branch conditions */
-        for (int i = 0; i < arguments.size(); i += 2) {
-            if (arguments.get(i).getValue().equals(switchCondition.getValue())) {
-                arguments.get(i).setValue(ConstantExpression.TRUE);
-            } else {
-                arguments.get(i).setValue(ConstantExpression.FALSE);
-            }
-        }
-
-        ILogicalExpression callExpr = new ScalarFunctionCallExpression(
-                FunctionUtils.getFunctionInfo(AsterixBuiltinFunctions.SWITCH_CASE), argumentRefs);
-
-        assignOp.getInputs().get(0).setValue(op3);
-        assignOp.getExpressions().get(0).setValue(callExpr);
-        context.computeAndSetTypeEnvironmentForOperator(assignOp);
-        return true;
-    }
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/34d81630/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/InlineUnnestFunctionRule.java
----------------------------------------------------------------------
diff --git a/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/InlineUnnestFunctionRule.java b/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/InlineUnnestFunctionRule.java
deleted file mode 100644
index 418e114..0000000
--- a/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/InlineUnnestFunctionRule.java
+++ /dev/null
@@ -1,174 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package edu.uci.ics.asterix.optimizer.rules;
-
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.commons.lang3.mutable.Mutable;
-import org.apache.commons.lang3.mutable.MutableObject;
-
-import edu.uci.ics.asterix.om.functions.AsterixBuiltinFunctions;
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.common.utils.Pair;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalExpressionTag;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.AbstractFunctionCallExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.VariableReferenceExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AssignOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.UnnestOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.visitors.VariableUtilities;
-import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
-
-/**
- * This rule is to inline unnest functions that are hold by variables.
- * This rule is to fix issue 201.
- */
-public class InlineUnnestFunctionRule implements IAlgebraicRewriteRule {
-
-    @Override
-    public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException {
-        return false;
-    }
-
-    @Override
-    public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context)
-            throws AlgebricksException {
-        AbstractLogicalOperator op1 = (AbstractLogicalOperator) opRef.getValue();
-        if (context.checkIfInDontApplySet(this, op1))
-            return false;
-        context.addToDontApplySet(this, op1);
-        if (op1.getOperatorTag() != LogicalOperatorTag.UNNEST)
-            return false;
-        UnnestOperator unnestOperator = (UnnestOperator) op1;
-        AbstractFunctionCallExpression expr = (AbstractFunctionCallExpression) unnestOperator.getExpressionRef()
-                .getValue();
-        //we only inline for the scan-collection function
-        if (expr.getFunctionIdentifier() != AsterixBuiltinFunctions.SCAN_COLLECTION)
-            return false;
-
-        // inline all variables from an unnesting function call
-        AbstractFunctionCallExpression funcExpr = (AbstractFunctionCallExpression) expr;
-        List<Mutable<ILogicalExpression>> args = funcExpr.getArguments();
-        for (int i = 0; i < args.size(); i++) {
-            ILogicalExpression argExpr = args.get(i).getValue();
-            if (argExpr.getExpressionTag() == LogicalExpressionTag.VARIABLE) {
-                VariableReferenceExpression varExpr = (VariableReferenceExpression) argExpr;
-                inlineVariable(varExpr.getVariableReference(), unnestOperator);
-            }
-        }
-        return true;
-    }
-
-    /**
-     * This method is to inline one variable
-     * 
-     * @param usedVar
-     *            A variable that is used by the scan-collection function in the unnest operator
-     * @param unnestOp
-     *            The unnest operator.
-     * @throws AlgebricksException
-     */
-    private void inlineVariable(LogicalVariable usedVar, UnnestOperator unnestOp) throws AlgebricksException {
-        AbstractFunctionCallExpression expr = (AbstractFunctionCallExpression) unnestOp.getExpressionRef().getValue();
-        List<Pair<AbstractFunctionCallExpression, Integer>> parentAndIndexList = new ArrayList<Pair<AbstractFunctionCallExpression, Integer>>();
-        getParentFunctionExpression(usedVar, expr, parentAndIndexList);
-        ILogicalExpression usedVarOrginExpr = findUsedVarOrigin(usedVar, unnestOp, (AbstractLogicalOperator) unnestOp
-                .getInputs().get(0).getValue());
-        if (usedVarOrginExpr != null) {
-            for (Pair<AbstractFunctionCallExpression, Integer> parentAndIndex : parentAndIndexList) {
-                //we only rewrite the top scan-collection function
-                if (parentAndIndex.first.getFunctionIdentifier() == AsterixBuiltinFunctions.SCAN_COLLECTION
-                        && parentAndIndex.first == expr) {
-                    unnestOp.getExpressionRef().setValue(usedVarOrginExpr);
-                }
-            }
-        }
-    }
-
-    private void getParentFunctionExpression(LogicalVariable usedVar, ILogicalExpression expr,
-            List<Pair<AbstractFunctionCallExpression, Integer>> parentAndIndexList) {
-        AbstractFunctionCallExpression funcExpr = (AbstractFunctionCallExpression) expr;
-        List<Mutable<ILogicalExpression>> args = funcExpr.getArguments();
-        for (int i = 0; i < args.size(); i++) {
-            ILogicalExpression argExpr = args.get(i).getValue();
-            if (argExpr.getExpressionTag() == LogicalExpressionTag.VARIABLE) {
-                VariableReferenceExpression varExpr = (VariableReferenceExpression) argExpr;
-                if (varExpr.getVariableReference().equals(usedVar))
-                    parentAndIndexList.add(new Pair<AbstractFunctionCallExpression, Integer>(funcExpr, i));
-            }
-            if (argExpr.getExpressionTag() == LogicalExpressionTag.FUNCTION_CALL) {
-                getParentFunctionExpression(usedVar, argExpr, parentAndIndexList);
-            }
-        }
-    }
-
-    private ILogicalExpression findUsedVarOrigin(LogicalVariable usedVar, AbstractLogicalOperator parentOp,
-            AbstractLogicalOperator currentOp) throws AlgebricksException {
-        ILogicalExpression ret = null;
-        if (currentOp.getOperatorTag() == LogicalOperatorTag.ASSIGN) {
-            List<LogicalVariable> producedVars = new ArrayList<LogicalVariable>();
-            VariableUtilities.getProducedVariables(currentOp, producedVars);
-            if (producedVars.contains(usedVar)) {
-                AssignOperator assignOp = (AssignOperator) currentOp;
-                int index = assignOp.getVariables().indexOf(usedVar);
-                ILogicalExpression returnedExpr = assignOp.getExpressions().get(index).getValue();
-                if (returnedExpr.getExpressionTag() == LogicalExpressionTag.FUNCTION_CALL) {
-                    AbstractFunctionCallExpression funcExpr = (AbstractFunctionCallExpression) returnedExpr;
-                    if (AsterixBuiltinFunctions.isBuiltinUnnestingFunction(funcExpr.getFunctionIdentifier())) {
-                        // we only inline for unnest functions
-                        removeUnecessaryAssign(parentOp, currentOp, assignOp, index);
-                        ret = returnedExpr;
-                    }
-                } else if (returnedExpr.getExpressionTag() == LogicalExpressionTag.VARIABLE) {
-                    //recusively inline
-                    VariableReferenceExpression varExpr = (VariableReferenceExpression) returnedExpr;
-                    LogicalVariable var = varExpr.getVariableReference();
-                    ILogicalExpression finalExpr = findUsedVarOrigin(var, currentOp,
-                            (AbstractLogicalOperator) currentOp.getInputs().get(0).getValue());
-                    if (finalExpr != null) {
-                        removeUnecessaryAssign(parentOp, currentOp, assignOp, index);
-                        ret = finalExpr;
-                    }
-                }
-            }
-        } else {
-            for (Mutable<ILogicalOperator> child : currentOp.getInputs()) {
-                ILogicalExpression expr = findUsedVarOrigin(usedVar, currentOp,
-                        (AbstractLogicalOperator) child.getValue());
-                if (expr != null) {
-                    ret = expr;
-                }
-            }
-        }
-        return ret;
-    }
-
-    private void removeUnecessaryAssign(AbstractLogicalOperator parentOp, AbstractLogicalOperator currentOp,
-            AssignOperator assignOp, int index) {
-        assignOp.getVariables().remove(index);
-        assignOp.getExpressions().remove(index);
-        if (assignOp.getVariables().size() == 0) {
-            int opIndex = parentOp.getInputs().indexOf(new MutableObject<ILogicalOperator>(currentOp));
-            parentOp.getInputs().get(opIndex).setValue(assignOp.getInputs().get(0).getValue());
-        }
-    }
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/34d81630/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/IntroduceAutogenerateIDRule.java
----------------------------------------------------------------------
diff --git a/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/IntroduceAutogenerateIDRule.java b/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/IntroduceAutogenerateIDRule.java
deleted file mode 100644
index b64c0e7..0000000
--- a/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/IntroduceAutogenerateIDRule.java
+++ /dev/null
@@ -1,168 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.asterix.optimizer.rules;
-
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.commons.lang3.mutable.Mutable;
-import org.apache.commons.lang3.mutable.MutableObject;
-
-import edu.uci.ics.asterix.aql.util.FunctionUtils;
-import edu.uci.ics.asterix.metadata.declared.AqlDataSource;
-import edu.uci.ics.asterix.metadata.declared.AqlDataSource.AqlDataSourceType;
-import edu.uci.ics.asterix.metadata.declared.DatasetDataSource;
-import edu.uci.ics.asterix.metadata.entities.InternalDatasetDetails;
-import edu.uci.ics.asterix.om.base.AString;
-import edu.uci.ics.asterix.om.constants.AsterixConstantValue;
-import edu.uci.ics.asterix.om.functions.AsterixBuiltinFunctions;
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.AbstractFunctionCallExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.ConstantExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.ScalarFunctionCallExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.VariableReferenceExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AssignOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.DataSourceScanOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.InsertDeleteOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.InsertDeleteOperator.Kind;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ProjectOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.visitors.VariableUtilities;
-import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
-
-public class IntroduceAutogenerateIDRule implements IAlgebraicRewriteRule {
-
-    @Override
-    public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException {
-        return false;
-    }
-
-    @Override
-    public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context)
-            throws AlgebricksException {
-
-        // match: [insert to internal dataset with autogenerated id] - assign - project
-        // produce: insert - assign - assign* - project
-        // **
-        // OR [insert to internal dataset with autogenerated id] - assign - [datasource scan]
-        // produce insert - assign - assign* - datasource scan
-
-        AbstractLogicalOperator currentOp = (AbstractLogicalOperator) opRef.getValue();
-        if (currentOp.getOperatorTag() != LogicalOperatorTag.INSERT_DELETE) {
-            return false;
-        }
-
-        InsertDeleteOperator insertOp = (InsertDeleteOperator) currentOp;
-        if (insertOp.getOperation() != Kind.INSERT) {
-            return false;
-        }
-
-        DatasetDataSource dds = (DatasetDataSource) insertOp.getDataSource();
-        boolean autogenerated = ((InternalDatasetDetails) dds.getDataset().getDatasetDetails()).isAutogenerated();
-        if (!autogenerated) {
-            return false;
-        }
-
-        if (((AqlDataSource) insertOp.getDataSource()).getDatasourceType() != AqlDataSourceType.INTERNAL_DATASET) {
-            return false;
-        }
-
-        AbstractLogicalOperator parentOp = (AbstractLogicalOperator) currentOp.getInputs().get(0).getValue();
-        if (parentOp.getOperatorTag() != LogicalOperatorTag.ASSIGN) {
-            return false;
-        }
-        AssignOperator assignOp = (AssignOperator) parentOp;
-        LogicalVariable inputRecord;
-
-        //bug here. will not work for internal datasets with filters since the pattern becomes [project-assign-assign-insert] <-this should be fixed->
-        AbstractLogicalOperator grandparentOp = (AbstractLogicalOperator) parentOp.getInputs().get(0).getValue();
-        if (grandparentOp.getOperatorTag() == LogicalOperatorTag.PROJECT) {
-            ProjectOperator projectOp = (ProjectOperator) grandparentOp;
-            inputRecord = projectOp.getVariables().get(0);
-        } else if (grandparentOp.getOperatorTag() == LogicalOperatorTag.DATASOURCESCAN) {
-            DataSourceScanOperator dssOp = (DataSourceScanOperator) grandparentOp;
-            inputRecord = dssOp.getVariables().get(0);
-        } else {
-            return false;
-        }
-
-        List<String> pkFieldName = ((InternalDatasetDetails) dds.getDataset().getDatasetDetails()).getPrimaryKey().get(
-                0);
-        ILogicalExpression rec0 = new VariableReferenceExpression(inputRecord);
-        ILogicalExpression rec1 = createPrimaryKeyRecordExpression(pkFieldName);
-        ILogicalExpression mergedRec = createRecordMergeFunction(rec0, rec1);
-        ILogicalExpression nonNullMergedRec = createNotNullFunction(mergedRec);
-
-        LogicalVariable v = context.newVar();
-        AssignOperator newAssign = new AssignOperator(v, new MutableObject<ILogicalExpression>(nonNullMergedRec));
-        newAssign.getInputs().add(new MutableObject<ILogicalOperator>(grandparentOp));
-        assignOp.getInputs().set(0, new MutableObject<ILogicalOperator>(newAssign));
-        VariableUtilities.substituteVariables(assignOp, inputRecord, v, context);
-        VariableUtilities.substituteVariables(insertOp, inputRecord, v, context);
-        context.computeAndSetTypeEnvironmentForOperator(newAssign);
-        context.computeAndSetTypeEnvironmentForOperator(assignOp);
-        context.computeAndSetTypeEnvironmentForOperator(insertOp);
-        return true;
-    }
-
-    private ILogicalExpression createNotNullFunction(ILogicalExpression mergedRec) {
-        List<Mutable<ILogicalExpression>> args = new ArrayList<>();
-        args.add(new MutableObject<ILogicalExpression>(mergedRec));
-        AbstractFunctionCallExpression notNullFn = new ScalarFunctionCallExpression(
-                FunctionUtils.getFunctionInfo(AsterixBuiltinFunctions.NOT_NULL), args);
-        return notNullFn;
-    }
-
-    private AbstractFunctionCallExpression createPrimaryKeyRecordExpression(List<String> pkFieldName) {
-        //Create lowest level of nested uuid
-        AbstractFunctionCallExpression uuidFn = new ScalarFunctionCallExpression(
-                FunctionUtils.getFunctionInfo(AsterixBuiltinFunctions.CREATE_UUID));
-        List<Mutable<ILogicalExpression>> openRecordConsArgs = new ArrayList<>();
-        Mutable<ILogicalExpression> pkFieldNameExpression = new MutableObject<ILogicalExpression>(
-                new ConstantExpression(new AsterixConstantValue(new AString(pkFieldName.get(pkFieldName.size() - 1)))));
-        openRecordConsArgs.add(pkFieldNameExpression);
-        Mutable<ILogicalExpression> pkFieldValueExpression = new MutableObject<ILogicalExpression>(uuidFn);
-        openRecordConsArgs.add(pkFieldValueExpression);
-        AbstractFunctionCallExpression openRecFn = new ScalarFunctionCallExpression(
-                FunctionUtils.getFunctionInfo(AsterixBuiltinFunctions.OPEN_RECORD_CONSTRUCTOR), openRecordConsArgs);
-
-        //Create higher levels
-        for (int i = pkFieldName.size() - 2; i > -1; i--) {
-            AString fieldName = new AString(pkFieldName.get(i));
-            openRecordConsArgs = new ArrayList<>();
-            openRecordConsArgs.add(new MutableObject<ILogicalExpression>(new ConstantExpression(
-                    new AsterixConstantValue(fieldName))));
-            openRecordConsArgs.add(new MutableObject<ILogicalExpression>(openRecFn));
-            openRecFn = new ScalarFunctionCallExpression(
-                    FunctionUtils.getFunctionInfo(AsterixBuiltinFunctions.OPEN_RECORD_CONSTRUCTOR), openRecordConsArgs);
-        }
-
-        return openRecFn;
-    }
-
-    private AbstractFunctionCallExpression createRecordMergeFunction(ILogicalExpression rec0, ILogicalExpression rec1) {
-        List<Mutable<ILogicalExpression>> recordMergeFnArgs = new ArrayList<>();
-        recordMergeFnArgs.add(new MutableObject<>(rec0));
-        recordMergeFnArgs.add(new MutableObject<>(rec1));
-        AbstractFunctionCallExpression recordMergeFn = new ScalarFunctionCallExpression(
-                FunctionUtils.getFunctionInfo(AsterixBuiltinFunctions.RECORD_MERGE), recordMergeFnArgs);
-        return recordMergeFn;
-    }
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/34d81630/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/IntroduceDynamicTypeCastForExternalFunctionRule.java
----------------------------------------------------------------------
diff --git a/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/IntroduceDynamicTypeCastForExternalFunctionRule.java b/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/IntroduceDynamicTypeCastForExternalFunctionRule.java
deleted file mode 100644
index 545a336..0000000
--- a/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/IntroduceDynamicTypeCastForExternalFunctionRule.java
+++ /dev/null
@@ -1,141 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package edu.uci.ics.asterix.optimizer.rules;
-
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.commons.lang3.mutable.Mutable;
-
-import edu.uci.ics.asterix.metadata.functions.AsterixExternalScalarFunctionInfo;
-import edu.uci.ics.asterix.om.functions.AsterixBuiltinFunctions;
-import edu.uci.ics.asterix.om.types.ARecordType;
-import edu.uci.ics.asterix.om.types.AUnionType;
-import edu.uci.ics.asterix.om.types.IAType;
-import edu.uci.ics.asterix.om.util.NonTaggedFormatUtil;
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalExpressionTag;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.ScalarFunctionCallExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.functions.FunctionIdentifier;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AssignOperator;
-import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
-
-/**
- * This rule provides the same type-casting handling as the IntroduceDynamicTypeCastRule does.
- * The only difference is that this rule is intended for external functions (User-Defined Functions).
- * Refer to IntroduceDynamicTypeCastRule for the detail.
- */
-public class IntroduceDynamicTypeCastForExternalFunctionRule implements IAlgebraicRewriteRule {
-
-    @Override
-    public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException {
-        return false;
-    }
-
-    @Override
-    public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context)
-            throws AlgebricksException {
-        /**
-         * pattern match: distribute_result - project - assign (external function call) - assign (open_record_constructor)
-         * resulting plan: distribute_result - project - assign (external function call) - assign (cast-record) - assign(open_record_constructor)
-         */
-        AbstractLogicalOperator op1 = (AbstractLogicalOperator) opRef.getValue();
-        if (op1.getOperatorTag() != LogicalOperatorTag.DISTRIBUTE_RESULT)
-            return false;
-        AbstractLogicalOperator op2 = (AbstractLogicalOperator) op1.getInputs().get(0).getValue();
-        if (op2.getOperatorTag() != LogicalOperatorTag.PROJECT)
-            return false;
-        AbstractLogicalOperator op3 = (AbstractLogicalOperator) op2.getInputs().get(0).getValue();
-        if (op3.getOperatorTag() != LogicalOperatorTag.ASSIGN)
-            return false;
-        AbstractLogicalOperator op4 = (AbstractLogicalOperator) op3.getInputs().get(0).getValue();
-        if (op4.getOperatorTag() != LogicalOperatorTag.ASSIGN)
-            return false;
-
-        // Op1 : assign (external function call), Op2 : assign (open_record_constructor)
-        AssignOperator assignOp1 = (AssignOperator) op3;
-        AssignOperator assignOp2 = (AssignOperator) op4;
-
-        // Checks whether open-record-constructor is called to create a record in the first assign operator - assignOp2
-        FunctionIdentifier fid = null;
-        ILogicalExpression assignExpr = assignOp2.getExpressions().get(0).getValue();
-        if (assignExpr.getExpressionTag() == LogicalExpressionTag.FUNCTION_CALL) {
-            ScalarFunctionCallExpression funcExpr = (ScalarFunctionCallExpression) assignOp2.getExpressions().get(0)
-                    .getValue();
-            fid = funcExpr.getFunctionIdentifier();
-
-            if (fid != AsterixBuiltinFunctions.OPEN_RECORD_CONSTRUCTOR) {
-                return false;
-            }
-        } else {
-            return false;
-        }
-
-        // Checks whether an external function is called in the second assign operator - assignOp1
-        assignExpr = assignOp1.getExpressions().get(0).getValue();
-        ScalarFunctionCallExpression funcExpr = null;
-        if (assignExpr.getExpressionTag() == LogicalExpressionTag.FUNCTION_CALL) {
-            funcExpr = (ScalarFunctionCallExpression) assignOp1.getExpressions().get(0).getValue();
-            fid = funcExpr.getFunctionIdentifier();
-
-            // Checks whether this is an internal function call. Then, we return false.
-            if (AsterixBuiltinFunctions.getBuiltinFunctionIdentifier(fid) != null) {
-                return false;
-            }
-
-        } else {
-            return false;
-        }
-
-        AsterixExternalScalarFunctionInfo finfo = (AsterixExternalScalarFunctionInfo) funcExpr.getFunctionInfo();
-        ARecordType requiredRecordType = (ARecordType) finfo.getArgumenTypes().get(0);
-
-        List<LogicalVariable> recordVar = new ArrayList<LogicalVariable>();
-        recordVar.addAll(assignOp2.getVariables());
-
-        IVariableTypeEnvironment env = assignOp2.computeOutputTypeEnvironment(context);
-        IAType inputRecordType = (IAType) env.getVarType(recordVar.get(0));
-
-        /** the input record type can be an union type -- for the case when it comes from a subplan or left-outer join */
-        boolean checkNull = false;
-        while (NonTaggedFormatUtil.isOptional(inputRecordType)) {
-            /** while-loop for the case there is a nested multi-level union */
-            inputRecordType = ((AUnionType) inputRecordType).getNullableType();
-            checkNull = true;
-        }
-
-        /** see whether the input record type needs to be casted */
-        boolean cast = !IntroduceDynamicTypeCastRule.compatible(requiredRecordType, inputRecordType);
-
-        if (checkNull) {
-            recordVar.set(0, IntroduceDynamicTypeCastRule.addWrapperFunction(requiredRecordType, recordVar.get(0),
-                    assignOp1, context, AsterixBuiltinFunctions.NOT_NULL));
-        }
-        if (cast) {
-            IntroduceDynamicTypeCastRule.addWrapperFunction(requiredRecordType, recordVar.get(0), assignOp1, context,
-                    AsterixBuiltinFunctions.CAST_RECORD);
-        }
-        return cast || checkNull;
-    }
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/34d81630/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/IntroduceDynamicTypeCastRule.java
----------------------------------------------------------------------
diff --git a/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/IntroduceDynamicTypeCastRule.java b/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/IntroduceDynamicTypeCastRule.java
deleted file mode 100644
index 64c34ed..0000000
--- a/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/IntroduceDynamicTypeCastRule.java
+++ /dev/null
@@ -1,288 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package edu.uci.ics.asterix.optimizer.rules;
-
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.commons.lang3.mutable.Mutable;
-import org.apache.commons.lang3.mutable.MutableObject;
-
-import edu.uci.ics.asterix.aql.util.FunctionUtils;
-import edu.uci.ics.asterix.metadata.declared.AqlDataSource;
-import edu.uci.ics.asterix.om.functions.AsterixBuiltinFunctions;
-import edu.uci.ics.asterix.om.typecomputer.base.TypeComputerUtilities;
-import edu.uci.ics.asterix.om.types.ARecordType;
-import edu.uci.ics.asterix.om.types.ATypeTag;
-import edu.uci.ics.asterix.om.types.AUnionType;
-import edu.uci.ics.asterix.om.types.IAType;
-import edu.uci.ics.asterix.om.util.NonTaggedFormatUtil;
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.AbstractFunctionCallExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.ScalarFunctionCallExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.VariableReferenceExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.functions.FunctionIdentifier;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AssignOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.InsertDeleteOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.visitors.VariableUtilities;
-import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
-
-/**
- * Dynamically cast a variable from its type to a specified required type, in a
- * recursive way. It enables: 1. bag-based fields in a record, 2. bidirectional
- * cast of a open field and a matched closed field, and 3. put in null fields
- * when necessary.
- * Here is an example: A record { "hobby": {{"music", "coding"}}, "id": "001",
- * "name": "Person Three"} which conforms to closed type ( id: string, name:
- * string, hobby: {{string}}? ) can be cast to an open type (id: string ), or
- * vice versa.
- * However, if the input record is a variable, then we don't know its exact
- * field layout at compile time. For example, records conforming to the same
- * type can have different field orderings and different open parts. That's why
- * we need dynamic type casting.
- * Note that as we can see in the example, the ordering of fields of a record is
- * not required. Since the open/closed part of a record has completely different
- * underlying memory/storage layout, a cast-record function will change the
- * layout as specified at runtime.
- * Implementation wise, this rule checks the target dataset type and the input
- * record type, and if the types are different, then it plugs in an assign with
- * a cast-record function, and projects away the original (uncast) field.
- */
-public class IntroduceDynamicTypeCastRule implements IAlgebraicRewriteRule {
-
-    @Override
-    public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException {
-        return false;
-    }
-
-    @Override
-    public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context)
-            throws AlgebricksException {
-        // Depending on the operator type, we need to extract the following pieces of information.
-        AbstractLogicalOperator op;
-        ARecordType requiredRecordType;
-        LogicalVariable recordVar;
-
-        // We identify INSERT and DISTRIBUTE_RESULT operators.
-        AbstractLogicalOperator op1 = (AbstractLogicalOperator) opRef.getValue();
-        switch (op1.getOperatorTag()) {
-            case SINK: {
-                /**
-                 * pattern match: sink insert assign
-                 * resulting plan: sink-insert-project-assign
-                 */
-
-                AbstractLogicalOperator op2 = (AbstractLogicalOperator) op1.getInputs().get(0).getValue();
-                if (op2.getOperatorTag() == LogicalOperatorTag.INSERT_DELETE) {
-                    InsertDeleteOperator insertDeleteOp = (InsertDeleteOperator) op2;
-                    if (insertDeleteOp.getOperation() == InsertDeleteOperator.Kind.DELETE)
-                        return false;
-
-                    // Remember this is the operator we need to modify
-                    op = insertDeleteOp;
-
-                    // Derive the required ARecordType based on the schema of the AqlDataSource
-                    InsertDeleteOperator insertDeleteOperator = (InsertDeleteOperator) op2;
-                    AqlDataSource dataSource = (AqlDataSource) insertDeleteOperator.getDataSource();
-                    IAType[] schemaTypes = (IAType[]) dataSource.getSchemaTypes();
-                    requiredRecordType = (ARecordType) schemaTypes[schemaTypes.length - 1];
-
-                    // Derive the Variable which we will potentially wrap with cast/null functions
-                    ILogicalExpression expr = insertDeleteOperator.getPayloadExpression().getValue();
-                    List<LogicalVariable> payloadVars = new ArrayList<LogicalVariable>();
-                    expr.getUsedVariables(payloadVars);
-                    recordVar = payloadVars.get(0);
-                } else {
-                    return false;
-                }
-
-                break;
-            }
-            case DISTRIBUTE_RESULT: {
-                // First, see if there was an output-record-type specified
-                requiredRecordType = (ARecordType) op1.getAnnotations().get("output-record-type");
-                if (requiredRecordType == null) {
-                    return false;
-                }
-
-                // Remember this is the operator we need to modify
-                op = op1;
-
-                // The Variable we want is the (hopefully singular, hopefully record-typed) live variable
-                // of the singular input operator of the DISTRIBUTE_RESULT
-                if (op.getInputs().size() > 1) {
-                    // Hopefully not possible?
-                    throw new AlgebricksException(
-                            "output-record-type defined for expression with multiple input operators");
-                }
-                AbstractLogicalOperator input = (AbstractLogicalOperator) op.getInputs().get(0).getValue();
-                List<LogicalVariable> liveVars = new ArrayList<LogicalVariable>();
-                VariableUtilities.getLiveVariables(input, liveVars);
-                if (liveVars.size() > 1) {
-                    throw new AlgebricksException(
-                            "Expression with multiple fields cannot be cast to output-record-type!");
-                }
-                recordVar = liveVars.get(0);
-
-                break;
-            }
-            default: {
-                return false;
-            }
-        }
-
-        // Derive the statically-computed type of the record
-        IVariableTypeEnvironment env = op.computeOutputTypeEnvironment(context);
-        IAType inputRecordType = (IAType) env.getVarType(recordVar);
-
-        /** the input record type can be an union type -- for the case when it comes from a subplan or left-outer join */
-        boolean checkNull = false;
-        while (NonTaggedFormatUtil.isOptional(inputRecordType)) {
-            /** while-loop for the case there is a nested multi-level union */
-            inputRecordType = ((AUnionType) inputRecordType).getNullableType();
-            checkNull = true;
-        }
-
-        /** see whether the input record type needs to be casted */
-        boolean cast = !compatible(requiredRecordType, inputRecordType);
-
-        if (checkNull) {
-            recordVar = addWrapperFunction(requiredRecordType, recordVar, op, context, AsterixBuiltinFunctions.NOT_NULL);
-        }
-        if (cast) {
-            addWrapperFunction(requiredRecordType, recordVar, op, context, AsterixBuiltinFunctions.CAST_RECORD);
-        }
-        return cast || checkNull;
-    }
-
-    /**
-     * Inject a function to wrap a variable when necessary
-     *
-     * @param requiredRecordType
-     *            the required record type
-     * @param recordVar
-     *            the record variable
-     * @param parent
-     *            the current parent operator to be rewritten
-     * @param context
-     *            the optimization context
-     * @param fd
-     *            the function to be injected
-     * @return true if cast is injected; false otherwise.
-     * @throws AlgebricksException
-     */
-    public static LogicalVariable addWrapperFunction(ARecordType requiredRecordType, LogicalVariable recordVar,
-            ILogicalOperator parent, IOptimizationContext context, FunctionIdentifier fd) throws AlgebricksException {
-        List<Mutable<ILogicalOperator>> opRefs = parent.getInputs();
-        for (int index = 0; index < opRefs.size(); index++) {
-            Mutable<ILogicalOperator> opRef = opRefs.get(index);
-            ILogicalOperator op = opRef.getValue();
-
-            /** get produced vars */
-            List<LogicalVariable> producedVars = new ArrayList<LogicalVariable>();
-            VariableUtilities.getProducedVariables(op, producedVars);
-            IVariableTypeEnvironment env = op.computeOutputTypeEnvironment(context);
-            for (int i = 0; i < producedVars.size(); i++) {
-                LogicalVariable var = producedVars.get(i);
-                if (var.equals(recordVar)) {
-                    /** insert an assign operator to call the function on-top-of the variable */
-                    IAType actualType = (IAType) env.getVarType(var);
-                    AbstractFunctionCallExpression cast = new ScalarFunctionCallExpression(
-                            FunctionUtils.getFunctionInfo(fd));
-                    cast.getArguments()
-                            .add(new MutableObject<ILogicalExpression>(new VariableReferenceExpression(var)));
-                    /** enforce the required record type */
-                    TypeComputerUtilities.setRequiredAndInputTypes(cast, requiredRecordType, actualType);
-                    LogicalVariable newAssignVar = context.newVar();
-                    AssignOperator newAssignOperator = new AssignOperator(newAssignVar,
-                            new MutableObject<ILogicalExpression>(cast));
-                    newAssignOperator.getInputs().add(new MutableObject<ILogicalOperator>(op));
-                    opRef.setValue(newAssignOperator);
-                    context.computeAndSetTypeEnvironmentForOperator(newAssignOperator);
-                    newAssignOperator.computeOutputTypeEnvironment(context);
-                    VariableUtilities.substituteVariables(parent, recordVar, newAssignVar, context);
-                    return newAssignVar;
-                }
-            }
-            /** recursive descend to the operator who produced the recordVar */
-            LogicalVariable replacedVar = addWrapperFunction(requiredRecordType, recordVar, op, context, fd);
-            if (replacedVar != null) {
-                /** substitute the recordVar by the replacedVar for operators who uses recordVar */
-                VariableUtilities.substituteVariables(parent, recordVar, replacedVar, context);
-                return replacedVar;
-            }
-        }
-        return null;
-    }
-
-    /**
-     * Check whether the required record type and the input type is compatible
-     *
-     * @param reqType
-     * @param inputType
-     * @return true if compatible; false otherwise
-     * @throws AlgebricksException
-     */
-    public static boolean compatible(ARecordType reqType, IAType inputType) throws AlgebricksException {
-        if (inputType.getTypeTag() == ATypeTag.ANY) {
-            return false;
-        }
-        if (inputType.getTypeTag() != ATypeTag.RECORD) {
-            throw new AlgebricksException("The input type " + inputType + " is not a valid record type!");
-        }
-        ARecordType inputRecType = (ARecordType) inputType;
-        if (reqType.isOpen() != inputRecType.isOpen()) {
-            return false;
-        }
-
-        IAType[] reqTypes = reqType.getFieldTypes();
-        String[] reqFieldNames = reqType.getFieldNames();
-        IAType[] inputTypes = inputRecType.getFieldTypes();
-        String[] inputFieldNames = ((ARecordType) inputType).getFieldNames();
-
-        if (reqTypes.length != inputTypes.length) {
-            return false;
-        }
-        for (int i = 0; i < reqTypes.length; i++) {
-            if (!reqFieldNames[i].equals(inputFieldNames[i])) {
-                return false;
-            }
-            IAType reqTypeInside = reqTypes[i];
-            if (NonTaggedFormatUtil.isOptional(reqTypes[i])) {
-                reqTypeInside = ((AUnionType) reqTypes[i]).getNullableType();
-            }
-            IAType inputTypeInside = inputTypes[i];
-            if (NonTaggedFormatUtil.isOptional(inputTypes[i])) {
-                if (!NonTaggedFormatUtil.isOptional(reqTypes[i])) {
-                    /** if the required type is not optional, the two types are incompatible */
-                    return false;
-                }
-                inputTypeInside = ((AUnionType) inputTypes[i]).getNullableType();
-            }
-            if (inputTypeInside.getTypeTag() != ATypeTag.NULL && !reqTypeInside.equals(inputTypeInside)) {
-                return false;
-            }
-        }
-        return true;
-    }
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/34d81630/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/IntroduceEnforcedListTypeRule.java
----------------------------------------------------------------------
diff --git a/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/IntroduceEnforcedListTypeRule.java b/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/IntroduceEnforcedListTypeRule.java
deleted file mode 100644
index 543a132..0000000
--- a/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/IntroduceEnforcedListTypeRule.java
+++ /dev/null
@@ -1,96 +0,0 @@
-/*
- * Copyright 2009-2013 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package edu.uci.ics.asterix.optimizer.rules;
-
-import java.util.Collections;
-import java.util.List;
-
-import org.apache.commons.lang3.mutable.Mutable;
-
-import edu.uci.ics.asterix.om.typecomputer.base.TypeComputerUtilities;
-import edu.uci.ics.asterix.om.types.IAType;
-import edu.uci.ics.asterix.optimizer.rules.typecast.StaticTypeCastUtil;
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
-import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalExpressionTag;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.AbstractFunctionCallExpression;
-import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractAssignOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractUnnestOperator;
-import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
-
-/**
- * This class is to enforce types for function expressions which contain list constructor function calls.
- * The List constructor is very special because a nested list is of type List<ANY>.
- * However, the bottom-up type inference (InferTypeRule in algebricks) did not infer that so we need this method to enforce the type.
- * We do not want to break the generality of algebricks so this method is called in an ASTERIX rule: @ IntroduceEnforcedListTypeRule} .
- */
-public class IntroduceEnforcedListTypeRule implements IAlgebraicRewriteRule {
-
-    @Override
-    public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException {
-        return false;
-    }
-
-    @Override
-    public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context)
-            throws AlgebricksException {
-        if (context.checkIfInDontApplySet(this, opRef.getValue()))
-            return false;
-        AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue();
-        context.addToDontApplySet(this, opRef.getValue());
-
-        /**
-         * rewrite list constructor types for list constructor functions
-         */
-        List<Mutable<ILogicalExpression>> expressions;
-        switch (op.getOperatorTag()) {
-            case ASSIGN:
-                AbstractAssignOperator assignOp = (AbstractAssignOperator) op;
-                expressions = assignOp.getExpressions();
-                break;
-            case UNNEST:
-                AbstractUnnestOperator unnestOp = (AbstractUnnestOperator) op;
-                expressions = Collections.singletonList(unnestOp.getExpressionRef());
-                break;
-            default:
-                return false;
-        }
-        IVariableTypeEnvironment env = op.computeOutputTypeEnvironment(context);
-        return rewriteExpressions(expressions, env);
-    }
-
-    private boolean rewriteExpressions(List<Mutable<ILogicalExpression>> expressions, IVariableTypeEnvironment env)
-            throws AlgebricksException {
-        boolean changed = false;
-        for (Mutable<ILogicalExpression> exprRef : expressions) {
-            ILogicalExpression expr = exprRef.getValue();
-            if (expr.getExpressionTag() == LogicalExpressionTag.FUNCTION_CALL) {
-                AbstractFunctionCallExpression argFuncExpr = (AbstractFunctionCallExpression) expr;
-                IAType exprType = (IAType) env.getType(argFuncExpr);
-                if (StaticTypeCastUtil.rewriteListExpr(argFuncExpr, exprType, exprType, env)) {
-                    TypeComputerUtilities.resetRequiredAndInputTypes(argFuncExpr);
-                    changed = true;
-                }
-            }
-        }
-        return changed;
-    }
-
-}


Mime
View raw message