asterixdb-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ima...@apache.org
Subject [31/51] [partial] incubator-asterixdb git commit: Change folder structure for Java repackage
Date Tue, 25 Aug 2015 16:44:19 GMT
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/34d81630/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/IntroduceSecondaryIndexInsertDeleteRule.java
----------------------------------------------------------------------
diff --git a/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/IntroduceSecondaryIndexInsertDeleteRule.java b/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/IntroduceSecondaryIndexInsertDeleteRule.java
new file mode 100644
index 0000000..42b60d5
--- /dev/null
+++ b/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/IntroduceSecondaryIndexInsertDeleteRule.java
@@ -0,0 +1,598 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.asterix.optimizer.rules;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.List;
+import java.util.Stack;
+
+import org.apache.commons.lang3.ArrayUtils;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.lang3.mutable.Mutable;
+import org.apache.commons.lang3.mutable.MutableObject;
+
+import edu.uci.ics.asterix.aql.util.FunctionUtils;
+import edu.uci.ics.asterix.common.config.DatasetConfig.DatasetType;
+import edu.uci.ics.asterix.common.config.DatasetConfig.IndexType;
+import edu.uci.ics.asterix.common.exceptions.AsterixException;
+import edu.uci.ics.asterix.metadata.declared.AqlDataSource;
+import edu.uci.ics.asterix.metadata.declared.AqlIndex;
+import edu.uci.ics.asterix.metadata.declared.AqlMetadataProvider;
+import edu.uci.ics.asterix.metadata.declared.DatasetDataSource;
+import edu.uci.ics.asterix.metadata.entities.Dataset;
+import edu.uci.ics.asterix.metadata.entities.Index;
+import edu.uci.ics.asterix.metadata.entities.InternalDatasetDetails;
+import edu.uci.ics.asterix.om.base.AInt32;
+import edu.uci.ics.asterix.om.base.AOrderedList;
+import edu.uci.ics.asterix.om.base.AString;
+import edu.uci.ics.asterix.om.constants.AsterixConstantValue;
+import edu.uci.ics.asterix.om.functions.AsterixBuiltinFunctions;
+import edu.uci.ics.asterix.om.typecomputer.base.TypeComputerUtilities;
+import edu.uci.ics.asterix.om.types.AOrderedListType;
+import edu.uci.ics.asterix.om.types.ARecordType;
+import edu.uci.ics.asterix.om.types.ATypeTag;
+import edu.uci.ics.asterix.om.types.AUnionType;
+import edu.uci.ics.asterix.om.types.BuiltinType;
+import edu.uci.ics.asterix.om.types.IAType;
+import edu.uci.ics.asterix.om.util.NonTaggedFormatUtil;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.common.utils.Pair;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalExpressionTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.AbstractFunctionCallExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.ConstantExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.ScalarFunctionCallExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.VariableReferenceExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.functions.FunctionIdentifier;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator.ExecutionMode;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AssignOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IndexInsertDeleteOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.InsertDeleteOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ProjectOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ReplicateOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.TokenizeOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.visitors.VariableUtilities;
+import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
+
+public class IntroduceSecondaryIndexInsertDeleteRule implements IAlgebraicRewriteRule {
+
+    @Override
+    public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException {
+        return false;
+    }
+
+    @Override
+    public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context)
+            throws AlgebricksException {
+        AbstractLogicalOperator op0 = (AbstractLogicalOperator) opRef.getValue();
+        if (op0.getOperatorTag() != LogicalOperatorTag.SINK) {
+            return false;
+        }
+        AbstractLogicalOperator op1 = (AbstractLogicalOperator) op0.getInputs().get(0).getValue();
+        if (op1.getOperatorTag() != LogicalOperatorTag.INSERT_DELETE) {
+            return false;
+        }
+
+        FunctionIdentifier fid = null;
+        /** find the record variable */
+        InsertDeleteOperator insertOp = (InsertDeleteOperator) op1;
+        ILogicalExpression recordExpr = insertOp.getPayloadExpression().getValue();
+        List<LogicalVariable> recordVar = new ArrayList<LogicalVariable>();
+        /** assume the payload is always a single variable expression */
+        recordExpr.getUsedVariables(recordVar);
+
+        /**
+         * op2 is the assign operator which extract primary keys from the record
+         * variable
+         */
+        AbstractLogicalOperator op2 = (AbstractLogicalOperator) op1.getInputs().get(0).getValue();
+
+        if (recordVar.size() == 0) {
+            /**
+             * For the case primary key-assignment expressions are constant
+             * expressions, find assign op that creates record to be
+             * inserted/deleted.
+             */
+            while (fid != AsterixBuiltinFunctions.OPEN_RECORD_CONSTRUCTOR) {
+                if (op2.getInputs().size() == 0) {
+                    return false;
+                }
+                op2 = (AbstractLogicalOperator) op2.getInputs().get(0).getValue();
+                if (op2.getOperatorTag() != LogicalOperatorTag.ASSIGN) {
+                    continue;
+                }
+                AssignOperator assignOp = (AssignOperator) op2;
+                ILogicalExpression assignExpr = assignOp.getExpressions().get(0).getValue();
+                if (assignExpr.getExpressionTag() == LogicalExpressionTag.FUNCTION_CALL) {
+                    ScalarFunctionCallExpression funcExpr = (ScalarFunctionCallExpression) assignOp.getExpressions()
+                            .get(0).getValue();
+                    fid = funcExpr.getFunctionIdentifier();
+                }
+            }
+            AssignOperator assignOp2 = (AssignOperator) op2;
+            recordVar.addAll(assignOp2.getVariables());
+        }
+        AqlDataSource datasetSource = (AqlDataSource) insertOp.getDataSource();
+        AqlMetadataProvider mp = (AqlMetadataProvider) context.getMetadataProvider();
+        String dataverseName = datasetSource.getId().getDataverseName();
+        String datasetName = datasetSource.getId().getDatasourceName();
+        Dataset dataset = mp.findDataset(dataverseName, datasetName);
+        if (dataset == null) {
+            throw new AlgebricksException("Unknown dataset " + datasetName + " in dataverse " + dataverseName);
+        }
+        if (dataset.getDatasetType() == DatasetType.EXTERNAL) {
+            return false;
+        }
+
+        // Create operators for secondary index insert/delete.
+        String itemTypeName = dataset.getItemTypeName();
+        IAType itemType = mp.findType(dataset.getDataverseName(), itemTypeName);
+        if (itemType.getTypeTag() != ATypeTag.RECORD) {
+            throw new AlgebricksException("Only record types can be indexed.");
+        }
+        ARecordType recType = (ARecordType) itemType;
+        List<Index> indexes = mp.getDatasetIndexes(dataset.getDataverseName(), dataset.getDatasetName());
+        ILogicalOperator currentTop = op1;
+        boolean hasSecondaryIndex = false;
+
+        // Put an n-gram or a keyword index in the later stage of index-update,
+        // since TokenizeOperator needs to be involved.
+        Collections.sort(indexes, new Comparator<Index>() {
+            @Override
+            public int compare(Index o1, Index o2) {
+                return o1.getIndexType().ordinal() - o2.getIndexType().ordinal();
+            }
+
+        });
+
+        // Check whether multiple keyword or n-gram indexes exist
+        int secondaryIndexTotalCnt = 0;
+        for (Index index : indexes) {
+            if (index.isSecondaryIndex())
+                secondaryIndexTotalCnt++;
+        }
+
+        // Initialize inputs to the SINK operator
+        if (secondaryIndexTotalCnt > 0) {
+            op0.getInputs().clear();
+        }
+
+        // Replicate Operator is applied only when doing the bulk-load.
+        AbstractLogicalOperator replicateOp = null;
+
+        if (secondaryIndexTotalCnt > 1 && insertOp.isBulkload()) {
+            // Split the logical plan into "each secondary index update branch"
+            // to replicate each <PK,RECORD> pair.
+            replicateOp = new ReplicateOperator(secondaryIndexTotalCnt);
+            replicateOp.getInputs().add(new MutableObject<ILogicalOperator>(currentTop));
+            replicateOp.setExecutionMode(ExecutionMode.PARTITIONED);
+            context.computeAndSetTypeEnvironmentForOperator(replicateOp);
+            currentTop = replicateOp;
+        }
+
+        // Prepare filtering field information
+        List<String> additionalFilteringField = ((InternalDatasetDetails) dataset.getDatasetDetails()).getFilterField();
+        List<LogicalVariable> additionalFilteringVars = null;
+        List<Mutable<ILogicalExpression>> additionalFilteringAssignExpressions = null;
+        List<Mutable<ILogicalExpression>> additionalFilteringExpressions = null;
+        AssignOperator additionalFilteringAssign = null;
+
+        if (additionalFilteringField != null) {
+            additionalFilteringVars = new ArrayList<LogicalVariable>();
+            additionalFilteringAssignExpressions = new ArrayList<Mutable<ILogicalExpression>>();
+            additionalFilteringExpressions = new ArrayList<Mutable<ILogicalExpression>>();
+            prepareVarAndExpression(additionalFilteringField, recType.getFieldNames(), recordVar.get(0),
+                    additionalFilteringAssignExpressions, additionalFilteringVars, context);
+            additionalFilteringAssign = new AssignOperator(additionalFilteringVars,
+                    additionalFilteringAssignExpressions);
+            for (LogicalVariable var : additionalFilteringVars) {
+                additionalFilteringExpressions.add(new MutableObject<ILogicalExpression>(
+                        new VariableReferenceExpression(var)));
+            }
+        }
+
+        // Iterate each secondary index and applying Index Update operations.
+        for (Index index : indexes) {
+            List<LogicalVariable> projectVars = new ArrayList<LogicalVariable>();
+            VariableUtilities.getUsedVariables(op1, projectVars);
+            if (!index.isSecondaryIndex()) {
+                continue;
+            }
+            LogicalVariable enforcedRecordVar = recordVar.get(0);
+            hasSecondaryIndex = true;
+            //if the index is enforcing field types
+            if (index.isEnforcingKeyFileds()) {
+                try {
+                    DatasetDataSource ds = (DatasetDataSource) (insertOp.getDataSource());
+                    ARecordType insertRecType = (ARecordType) ds.getSchemaTypes()[ds.getSchemaTypes().length - 1];
+                    LogicalVariable castVar = context.newVar();
+                    ARecordType enforcedType = createEnforcedType(insertRecType, index);
+                    //introduce casting to enforced type
+                    AbstractFunctionCallExpression castFunc = new ScalarFunctionCallExpression(
+                            FunctionUtils.getFunctionInfo(AsterixBuiltinFunctions.CAST_RECORD));
+
+                    castFunc.getArguments().add(
+                            new MutableObject<ILogicalExpression>(insertOp.getPayloadExpression().getValue()));
+                    TypeComputerUtilities.setRequiredAndInputTypes(castFunc, enforcedType, insertRecType);
+                    AssignOperator newAssignOperator = new AssignOperator(castVar,
+                            new MutableObject<ILogicalExpression>(castFunc));
+                    newAssignOperator.getInputs().add(new MutableObject<ILogicalOperator>(currentTop));
+                    currentTop = newAssignOperator;
+                    //project out casted record
+                    projectVars.add(castVar);
+                    enforcedRecordVar = castVar;
+                    context.computeAndSetTypeEnvironmentForOperator(newAssignOperator);
+                    context.computeAndSetTypeEnvironmentForOperator(currentTop);
+                    recType = enforcedType;
+                } catch (AsterixException e) {
+                    throw new AlgebricksException(e);
+                }
+            }
+
+            List<List<String>> secondaryKeyFields = index.getKeyFieldNames();
+            List<IAType> secondaryKeyTypes = index.getKeyFieldTypes();
+            List<LogicalVariable> secondaryKeyVars = new ArrayList<LogicalVariable>();
+            List<Mutable<ILogicalExpression>> expressions = new ArrayList<Mutable<ILogicalExpression>>();
+            List<Mutable<ILogicalExpression>> secondaryExpressions = new ArrayList<Mutable<ILogicalExpression>>();
+
+            for (List<String> secondaryKey : secondaryKeyFields) {
+                prepareVarAndExpression(secondaryKey, recType.getFieldNames(), enforcedRecordVar, expressions,
+                        secondaryKeyVars, context);
+            }
+
+            AssignOperator assign = new AssignOperator(secondaryKeyVars, expressions);
+            ProjectOperator project = new ProjectOperator(projectVars);
+
+            if (additionalFilteringAssign != null) {
+                additionalFilteringAssign.getInputs().add(new MutableObject<ILogicalOperator>(project));
+                assign.getInputs().add(new MutableObject<ILogicalOperator>(additionalFilteringAssign));
+            } else {
+                assign.getInputs().add(new MutableObject<ILogicalOperator>(project));
+            }
+
+            // Only apply replicate operator when doing bulk-load
+            if (secondaryIndexTotalCnt > 1 && insertOp.isBulkload())
+                project.getInputs().add(new MutableObject<ILogicalOperator>(replicateOp));
+            else
+                project.getInputs().add(new MutableObject<ILogicalOperator>(currentTop));
+
+            context.computeAndSetTypeEnvironmentForOperator(project);
+
+            if (additionalFilteringAssign != null) {
+                context.computeAndSetTypeEnvironmentForOperator(additionalFilteringAssign);
+            }
+
+            context.computeAndSetTypeEnvironmentForOperator(assign);
+            currentTop = assign;
+
+            // BTree, Keyword, or n-gram index case
+            if (index.getIndexType() == IndexType.BTREE
+                    || index.getIndexType() == IndexType.SINGLE_PARTITION_WORD_INVIX
+                    || index.getIndexType() == IndexType.SINGLE_PARTITION_NGRAM_INVIX
+                    || index.getIndexType() == IndexType.LENGTH_PARTITIONED_WORD_INVIX
+                    || index.getIndexType() == IndexType.LENGTH_PARTITIONED_NGRAM_INVIX) {
+                for (LogicalVariable secondaryKeyVar : secondaryKeyVars) {
+                    secondaryExpressions.add(new MutableObject<ILogicalExpression>(new VariableReferenceExpression(
+                            secondaryKeyVar)));
+                }
+                Mutable<ILogicalExpression> filterExpression = createFilterExpression(secondaryKeyVars,
+                        context.getOutputTypeEnvironment(currentTop), false);
+                AqlIndex dataSourceIndex = new AqlIndex(index, dataverseName, datasetName, mp);
+
+                // Introduce the TokenizeOperator only when doing bulk-load,
+                // and index type is keyword or n-gram.
+                if (index.getIndexType() != IndexType.BTREE && insertOp.isBulkload()) {
+
+                    // Check whether the index is length-partitioned or not.
+                    // If partitioned, [input variables to TokenizeOperator,
+                    // token, number of token] pairs will be generated and
+                    // fed into the IndexInsertDeleteOperator.
+                    // If not, [input variables, token] pairs will be generated
+                    // and fed into the IndexInsertDeleteOperator.
+                    // Input variables are passed since TokenizeOperator is not an
+                    // filtering operator.
+                    boolean isPartitioned = false;
+                    if (index.getIndexType() == IndexType.LENGTH_PARTITIONED_WORD_INVIX
+                            || index.getIndexType() == IndexType.LENGTH_PARTITIONED_NGRAM_INVIX)
+                        isPartitioned = true;
+
+                    // Create a new logical variable - token
+                    List<LogicalVariable> tokenizeKeyVars = new ArrayList<LogicalVariable>();
+                    List<Mutable<ILogicalExpression>> tokenizeKeyExprs = new ArrayList<Mutable<ILogicalExpression>>();
+                    LogicalVariable tokenVar = context.newVar();
+                    tokenizeKeyVars.add(tokenVar);
+                    tokenizeKeyExprs.add(new MutableObject<ILogicalExpression>(
+                            new VariableReferenceExpression(tokenVar)));
+
+                    // Check the field type of the secondary key.
+                    IAType secondaryKeyType = null;
+                    Pair<IAType, Boolean> keyPairType = Index.getNonNullableKeyFieldType(secondaryKeyFields.get(0),
+                            recType);
+                    secondaryKeyType = keyPairType.first;
+
+                    List<Object> varTypes = new ArrayList<Object>();
+                    varTypes.add(NonTaggedFormatUtil.getTokenType(secondaryKeyType));
+
+                    // If the index is a length-partitioned, then create
+                    // additional variable - number of token.
+                    // We use a special type for the length-partitioned index.
+                    // The type is short, and this does not contain type info.
+                    if (isPartitioned) {
+                        LogicalVariable lengthVar = context.newVar();
+                        tokenizeKeyVars.add(lengthVar);
+                        tokenizeKeyExprs.add(new MutableObject<ILogicalExpression>(new VariableReferenceExpression(
+                                lengthVar)));
+                        varTypes.add(BuiltinType.SHORTWITHOUTTYPEINFO);
+                    }
+
+                    // TokenizeOperator to tokenize [SK, PK] pairs
+                    TokenizeOperator tokenUpdate = new TokenizeOperator(dataSourceIndex,
+                            insertOp.getPrimaryKeyExpressions(), secondaryExpressions, tokenizeKeyVars,
+                            filterExpression, insertOp.getOperation(), insertOp.isBulkload(), isPartitioned, varTypes);
+                    tokenUpdate.getInputs().add(new MutableObject<ILogicalOperator>(assign));
+                    context.computeAndSetTypeEnvironmentForOperator(tokenUpdate);
+
+                    IndexInsertDeleteOperator indexUpdate = new IndexInsertDeleteOperator(dataSourceIndex,
+                            insertOp.getPrimaryKeyExpressions(), tokenizeKeyExprs, filterExpression,
+                            insertOp.getOperation(), insertOp.isBulkload());
+                    indexUpdate.setAdditionalFilteringExpressions(additionalFilteringExpressions);
+                    indexUpdate.getInputs().add(new MutableObject<ILogicalOperator>(tokenUpdate));
+
+                    context.computeAndSetTypeEnvironmentForOperator(indexUpdate);
+
+                    currentTop = indexUpdate;
+                    op0.getInputs().add(new MutableObject<ILogicalOperator>(currentTop));
+
+                } else {
+                    // When TokenizeOperator is not needed
+                    IndexInsertDeleteOperator indexUpdate = new IndexInsertDeleteOperator(dataSourceIndex,
+                            insertOp.getPrimaryKeyExpressions(), secondaryExpressions, filterExpression,
+                            insertOp.getOperation(), insertOp.isBulkload());
+                    indexUpdate.setAdditionalFilteringExpressions(additionalFilteringExpressions);
+                    indexUpdate.getInputs().add(new MutableObject<ILogicalOperator>(currentTop));
+
+                    currentTop = indexUpdate;
+                    context.computeAndSetTypeEnvironmentForOperator(indexUpdate);
+
+                    if (insertOp.isBulkload())
+                        op0.getInputs().add(new MutableObject<ILogicalOperator>(currentTop));
+
+                }
+
+            } else if (index.getIndexType() == IndexType.RTREE) {
+                Pair<IAType, Boolean> keyPairType = Index.getNonNullableOpenFieldType(secondaryKeyTypes.get(0),
+                        secondaryKeyFields.get(0), recType);
+                IAType spatialType = keyPairType.first;
+                int dimension = NonTaggedFormatUtil.getNumDimensions(spatialType.getTypeTag());
+                int numKeys = dimension * 2;
+                List<LogicalVariable> keyVarList = new ArrayList<LogicalVariable>();
+                List<Mutable<ILogicalExpression>> keyExprList = new ArrayList<Mutable<ILogicalExpression>>();
+                for (int i = 0; i < numKeys; i++) {
+                    LogicalVariable keyVar = context.newVar();
+                    keyVarList.add(keyVar);
+                    AbstractFunctionCallExpression createMBR = new ScalarFunctionCallExpression(
+                            FunctionUtils.getFunctionInfo(AsterixBuiltinFunctions.CREATE_MBR));
+                    createMBR.getArguments().add(
+                            new MutableObject<ILogicalExpression>(new VariableReferenceExpression(secondaryKeyVars
+                                    .get(0))));
+                    createMBR.getArguments().add(
+                            new MutableObject<ILogicalExpression>(new ConstantExpression(new AsterixConstantValue(
+                                    new AInt32(dimension)))));
+                    createMBR.getArguments().add(
+                            new MutableObject<ILogicalExpression>(new ConstantExpression(new AsterixConstantValue(
+                                    new AInt32(i)))));
+                    keyExprList.add(new MutableObject<ILogicalExpression>(createMBR));
+                }
+                for (LogicalVariable secondaryKeyVar : keyVarList) {
+                    secondaryExpressions.add(new MutableObject<ILogicalExpression>(new VariableReferenceExpression(
+                            secondaryKeyVar)));
+                }
+                AssignOperator assignCoordinates = new AssignOperator(keyVarList, keyExprList);
+                assignCoordinates.getInputs().add(new MutableObject<ILogicalOperator>(currentTop));
+                context.computeAndSetTypeEnvironmentForOperator(assignCoordinates);
+                // We must enforce the filter if the originating spatial type is
+                // nullable.
+                boolean forceFilter = keyPairType.second;
+                Mutable<ILogicalExpression> filterExpression = createFilterExpression(keyVarList,
+                        context.getOutputTypeEnvironment(assignCoordinates), forceFilter);
+                AqlIndex dataSourceIndex = new AqlIndex(index, dataverseName, datasetName, mp);
+                IndexInsertDeleteOperator indexUpdate = new IndexInsertDeleteOperator(dataSourceIndex,
+                        insertOp.getPrimaryKeyExpressions(), secondaryExpressions, filterExpression,
+                        insertOp.getOperation(), insertOp.isBulkload());
+                indexUpdate.setAdditionalFilteringExpressions(additionalFilteringExpressions);
+                indexUpdate.getInputs().add(new MutableObject<ILogicalOperator>(assignCoordinates));
+                currentTop = indexUpdate;
+                context.computeAndSetTypeEnvironmentForOperator(indexUpdate);
+
+                if (insertOp.isBulkload())
+                    op0.getInputs().add(new MutableObject<ILogicalOperator>(currentTop));
+
+            }
+
+        }
+        if (!hasSecondaryIndex) {
+            return false;
+        }
+
+        if (!insertOp.isBulkload()) {
+            op0.getInputs().clear();
+            op0.getInputs().add(new MutableObject<ILogicalOperator>(currentTop));
+        }
+        return true;
+    }
+
+    public static ARecordType createEnforcedType(ARecordType initialType, Index index) throws AsterixException,
+            AlgebricksException {
+        ARecordType enforcedType = initialType;
+        for (int i = 0; i < index.getKeyFieldNames().size(); i++) {
+            try {
+                Stack<Pair<ARecordType, String>> nestedTypeStack = new Stack<Pair<ARecordType, String>>();
+                List<String> splits = index.getKeyFieldNames().get(i);
+                ARecordType nestedFieldType = enforcedType;
+                boolean openRecords = false;
+                String bridgeName = nestedFieldType.getTypeName();
+                int j;
+                //Build the stack for the enforced type
+                for (j = 1; j < splits.size(); j++) {
+                    nestedTypeStack.push(new Pair<ARecordType, String>(nestedFieldType, splits.get(j - 1)));
+                    bridgeName = nestedFieldType.getTypeName();
+                    nestedFieldType = (ARecordType) enforcedType.getSubFieldType(splits.subList(0, j));
+                    if (nestedFieldType == null) {
+                        openRecords = true;
+                        break;
+                    }
+                }
+                if (openRecords == true) {
+                    //create the smallest record
+                    enforcedType = new ARecordType(splits.get(splits.size() - 2), new String[] { splits.get(splits
+                            .size() - 1) }, new IAType[] { AUnionType.createNullableType(index.getKeyFieldTypes()
+                            .get(i)) }, true);
+                    //create the open part of the nested field
+                    for (int k = splits.size() - 3; k > (j - 2); k--) {
+                        enforcedType = new ARecordType(splits.get(k), new String[] { splits.get(k + 1) },
+                                new IAType[] { AUnionType.createNullableType(enforcedType) }, true);
+                    }
+                    //Bridge the gap
+                    Pair<ARecordType, String> gapPair = nestedTypeStack.pop();
+                    ARecordType parent = gapPair.first;
+
+                    IAType[] parentFieldTypes = ArrayUtils.addAll(parent.getFieldTypes().clone(),
+                            new IAType[] { AUnionType.createNullableType(enforcedType) });
+                    enforcedType = new ARecordType(bridgeName, ArrayUtils.addAll(parent.getFieldNames(),
+                            enforcedType.getTypeName()), parentFieldTypes, true);
+
+                } else {
+                    //Schema is closed all the way to the field
+                    //enforced fields are either null or strongly typed
+                    enforcedType = new ARecordType(nestedFieldType.getTypeName(), ArrayUtils.addAll(
+                            nestedFieldType.getFieldNames(), splits.get(splits.size() - 1)), ArrayUtils.addAll(
+                            nestedFieldType.getFieldTypes(),
+                            AUnionType.createNullableType(index.getKeyFieldTypes().get(i))), nestedFieldType.isOpen());
+                }
+
+                //Create the enforcedtype for the nested fields in the schema, from the ground up
+                if (nestedTypeStack.size() > 0) {
+                    while (!nestedTypeStack.isEmpty()) {
+                        Pair<ARecordType, String> nestedTypePair = nestedTypeStack.pop();
+                        ARecordType nestedRecType = nestedTypePair.first;
+                        IAType[] nestedRecTypeFieldTypes = nestedRecType.getFieldTypes().clone();
+                        nestedRecTypeFieldTypes[nestedRecType.findFieldPosition(nestedTypePair.second)] = enforcedType;
+                        enforcedType = new ARecordType(nestedRecType.getTypeName(), nestedRecType.getFieldNames(),
+                                nestedRecTypeFieldTypes, nestedRecType.isOpen());
+                    }
+                }
+
+            } catch (AsterixException e) {
+                throw new AlgebricksException("Cannot enforce typed fields "
+                        + StringUtils.join(index.getKeyFieldNames()), e);
+            } catch (IOException e) {
+                throw new AsterixException(e);
+            }
+        }
+        return enforcedType;
+    }
+
+    @SuppressWarnings("unchecked")
+    private void prepareVarAndExpression(List<String> field, String[] fieldNames, LogicalVariable recordVar,
+            List<Mutable<ILogicalExpression>> expressions, List<LogicalVariable> vars, IOptimizationContext context)
+            throws AlgebricksException {
+        Mutable<ILogicalExpression> varRef = new MutableObject<ILogicalExpression>(new VariableReferenceExpression(
+                recordVar));
+        int pos = -1;
+        if (field.size() == 1) {
+            for (int j = 0; j < fieldNames.length; j++) {
+                if (fieldNames[j].equals(field.get(0))) {
+                    pos = j;
+                    break;
+                }
+            }
+        }
+        if (pos == -1) {
+            AbstractFunctionCallExpression func;
+            if (field.size() > 1) {
+                AOrderedList fieldList = new AOrderedList(new AOrderedListType(BuiltinType.ASTRING, null));
+                for (int i = 0; i < field.size(); i++) {
+                    fieldList.add(new AString(field.get(i)));
+                }
+                Mutable<ILogicalExpression> fieldRef = new MutableObject<ILogicalExpression>(new ConstantExpression(
+                        new AsterixConstantValue(fieldList)));
+                //Create an expression for the nested case
+                func = new ScalarFunctionCallExpression(
+                        FunctionUtils.getFunctionInfo(AsterixBuiltinFunctions.FIELD_ACCESS_NESTED), varRef, fieldRef);
+            } else {
+                Mutable<ILogicalExpression> fieldRef = new MutableObject<ILogicalExpression>(new ConstantExpression(
+                        new AsterixConstantValue(new AString(field.get(0)))));
+                //Create an expression for the open field case (By name)
+                func = new ScalarFunctionCallExpression(
+                        FunctionUtils.getFunctionInfo(AsterixBuiltinFunctions.FIELD_ACCESS_BY_NAME), varRef, fieldRef);
+            }
+            expressions.add(new MutableObject<ILogicalExpression>(func));
+            LogicalVariable newVar = context.newVar();
+            vars.add(newVar);
+        } else {
+            // Assumes the indexed field is in the closed portion of the type.
+            Mutable<ILogicalExpression> indexRef = new MutableObject<ILogicalExpression>(new ConstantExpression(
+                    new AsterixConstantValue(new AInt32(pos))));
+            AbstractFunctionCallExpression func = new ScalarFunctionCallExpression(
+                    FunctionUtils.getFunctionInfo(AsterixBuiltinFunctions.FIELD_ACCESS_BY_INDEX), varRef, indexRef);
+            expressions.add(new MutableObject<ILogicalExpression>(func));
+            LogicalVariable newVar = context.newVar();
+            vars.add(newVar);
+        }
+    }
+
+    @SuppressWarnings("unchecked")
+    private Mutable<ILogicalExpression> createFilterExpression(List<LogicalVariable> secondaryKeyVars,
+            IVariableTypeEnvironment typeEnv, boolean forceFilter) throws AlgebricksException {
+        List<Mutable<ILogicalExpression>> filterExpressions = new ArrayList<Mutable<ILogicalExpression>>();
+        // Add 'is not null' to all nullable secondary index keys as a filtering
+        // condition.
+        for (LogicalVariable secondaryKeyVar : secondaryKeyVars) {
+            IAType secondaryKeyType = (IAType) typeEnv.getVarType(secondaryKeyVar);
+            if (!NonTaggedFormatUtil.isOptional(secondaryKeyType) && !forceFilter) {
+                continue;
+            }
+            ScalarFunctionCallExpression isNullFuncExpr = new ScalarFunctionCallExpression(
+                    FunctionUtils.getFunctionInfo(AsterixBuiltinFunctions.IS_NULL),
+                    new MutableObject<ILogicalExpression>(new VariableReferenceExpression(secondaryKeyVar)));
+            ScalarFunctionCallExpression notFuncExpr = new ScalarFunctionCallExpression(
+                    FunctionUtils.getFunctionInfo(AsterixBuiltinFunctions.NOT), new MutableObject<ILogicalExpression>(
+                            isNullFuncExpr));
+            filterExpressions.add(new MutableObject<ILogicalExpression>(notFuncExpr));
+        }
+        // No nullable secondary keys.
+        if (filterExpressions.isEmpty()) {
+            return null;
+        }
+        Mutable<ILogicalExpression> filterExpression = null;
+        if (filterExpressions.size() > 1) {
+            // Create a conjunctive condition.
+            filterExpression = new MutableObject<ILogicalExpression>(new ScalarFunctionCallExpression(
+                    FunctionUtils.getFunctionInfo(AsterixBuiltinFunctions.AND), filterExpressions));
+        } else {
+            filterExpression = filterExpressions.get(0);
+        }
+        return filterExpression;
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/34d81630/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/IntroduceStaticTypeCastForInsertRule.java
----------------------------------------------------------------------
diff --git a/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/IntroduceStaticTypeCastForInsertRule.java b/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/IntroduceStaticTypeCastForInsertRule.java
new file mode 100644
index 0000000..6419fc1
--- /dev/null
+++ b/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/IntroduceStaticTypeCastForInsertRule.java
@@ -0,0 +1,157 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package edu.uci.ics.asterix.optimizer.rules;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.commons.lang3.mutable.Mutable;
+
+import edu.uci.ics.asterix.metadata.declared.AqlDataSource;
+import edu.uci.ics.asterix.om.typecomputer.base.TypeComputerUtilities;
+import edu.uci.ics.asterix.om.types.IAType;
+import edu.uci.ics.asterix.optimizer.rules.typecast.StaticTypeCastUtil;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalExpressionTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.AbstractFunctionCallExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AssignOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.InsertDeleteOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.visitors.VariableUtilities;
+import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
+
+/**
+ * Statically cast a constant from its type to a specified required type, in a
+ * recursive way. It enables: 1. bag-based fields in a record, 2. bidirectional
+ * cast of an open field and a matched closed field, and 3. put in null fields
+ * when necessary. It should be fired before the constant folding rule.
+ * This rule is not responsible for type casting between primitive types.
+ * Here is an example: A record { "hobby": {{"music", "coding"}}, "id": "001",
+ * "name": "Person Three"} which confirms to closed type ( id: string, name:
+ * string, hobby: {{string}}? ) can be cast to an open type (id: string ), or
+ * vice versa.
+ * Implementation wise: first, we match the record's type and its target dataset
+ * type to see if it is "cast-able"; second, if the types are cast-able, we
+ * embed the required type into the original producer expression. If the types
+ * are not cast-able, we throw a compile time exception.
+ * Then, at runtime (not in this rule), the corresponding record/list
+ * constructors know what to do by checking the required output type.
+ * TODO: right now record/list constructor of the cast result is not done in the
+ * ConstantFoldingRule and has to go to the runtime, because the
+ * ConstantFoldingRule uses ARecordSerializerDeserializer which seems to have
+ * some problem.
+ */
+public class IntroduceStaticTypeCastForInsertRule implements IAlgebraicRewriteRule {
+
+    @Override
+    public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException {
+        return false;
+    }
+
+    @Override
+    public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context)
+            throws AlgebricksException {
+        /**
+         * pattern match: sink/insert/assign record type is propagated from
+         * insert data source to the record-constructor expression
+         */
+        if (context.checkIfInDontApplySet(this, opRef.getValue()))
+            return false;
+        context.addToDontApplySet(this, opRef.getValue());
+
+        AbstractLogicalOperator op1 = (AbstractLogicalOperator) opRef.getValue();
+        List<LogicalVariable> producedVariables = new ArrayList<LogicalVariable>();
+        LogicalVariable oldRecordVariable;
+
+        if (op1.getOperatorTag() != LogicalOperatorTag.SINK)
+            return false;
+        AbstractLogicalOperator op2 = (AbstractLogicalOperator) op1.getInputs().get(0).getValue();
+        if (op2.getOperatorTag() != LogicalOperatorTag.INSERT_DELETE)
+            return false;
+        InsertDeleteOperator insertDeleteOp = (InsertDeleteOperator) op2;
+        if (insertDeleteOp.getOperation() == InsertDeleteOperator.Kind.DELETE)
+            return false;
+        /**
+         * get required record type
+         */
+        InsertDeleteOperator insertDeleteOperator = (InsertDeleteOperator) op2;
+        AqlDataSource dataSource = (AqlDataSource) insertDeleteOperator.getDataSource();
+        IAType[] schemaTypes = (IAType[]) dataSource.getSchemaTypes();
+        IAType requiredRecordType = schemaTypes[schemaTypes.length - 1];
+
+        List<LogicalVariable> usedVariables = new ArrayList<LogicalVariable>();
+        insertDeleteOperator.getPayloadExpression().getValue().getUsedVariables(usedVariables);
+
+        // the used variable should contain the record that will be inserted
+        // but it will not fail in many cases even if the used variable set is
+        // empty
+        if (usedVariables.size() == 0)
+            return false;
+
+        oldRecordVariable = usedVariables.get(0);
+        LogicalVariable inputRecordVar = usedVariables.get(0);
+        IVariableTypeEnvironment env = insertDeleteOperator.computeOutputTypeEnvironment(context);
+        IAType inputRecordType = (IAType) env.getVarType(inputRecordVar);
+
+        AbstractLogicalOperator currentOperator = (AbstractLogicalOperator) op2.getInputs().get(0).getValue();
+        /**
+         * find the assign operator for the "input record" to the insert_delete
+         * operator
+         */
+        do {
+            context.addToDontApplySet(this, currentOperator);
+            if (currentOperator.getOperatorTag() == LogicalOperatorTag.ASSIGN) {
+                AssignOperator assignOp = (AssignOperator) currentOperator;
+                producedVariables.clear();
+                VariableUtilities.getProducedVariables(currentOperator, producedVariables);
+                int position = producedVariables.indexOf(oldRecordVariable);
+
+                /**
+                 * set the top-down propagated type
+                 */
+                if (position >= 0) {
+                    AssignOperator originalAssign = (AssignOperator) currentOperator;
+                    List<Mutable<ILogicalExpression>> expressionRefs = originalAssign.getExpressions();
+                    ILogicalExpression expr = expressionRefs.get(position).getValue();
+                    if (expr.getExpressionTag() == LogicalExpressionTag.FUNCTION_CALL) {
+                        AbstractFunctionCallExpression funcExpr = (AbstractFunctionCallExpression) expr;
+                        // that expression has been rewritten, and it will not
+                        // fail but just return false
+                        if (TypeComputerUtilities.getRequiredType(funcExpr) != null) {
+                            context.computeAndSetTypeEnvironmentForOperator(assignOp);
+                            return false;
+                        }
+                        IVariableTypeEnvironment assignEnv = assignOp.computeOutputTypeEnvironment(context);
+                        StaticTypeCastUtil.rewriteFuncExpr(funcExpr, requiredRecordType, inputRecordType, assignEnv);
+                    }
+                    context.computeAndSetTypeEnvironmentForOperator(originalAssign);
+                }
+            }
+            if (currentOperator.getInputs().size() > 0)
+                currentOperator = (AbstractLogicalOperator) currentOperator.getInputs().get(0).getValue();
+            else
+                break;
+        } while (currentOperator != null);
+        return true;
+    }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/34d81630/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/IntroduceTransactionCommitByAssignOpRule.java
----------------------------------------------------------------------
diff --git a/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/IntroduceTransactionCommitByAssignOpRule.java b/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/IntroduceTransactionCommitByAssignOpRule.java
new file mode 100644
index 0000000..3984fe8
--- /dev/null
+++ b/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/IntroduceTransactionCommitByAssignOpRule.java
@@ -0,0 +1,76 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.asterix.optimizer.rules;
+
+import org.apache.commons.lang3.mutable.Mutable;
+import org.apache.commons.lang3.mutable.MutableObject;
+
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.VariableReferenceExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AssignOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SelectOperator;
+import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
+
+public class IntroduceTransactionCommitByAssignOpRule implements IAlgebraicRewriteRule {
+
+    @Override
+    public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException {
+        return false;
+    }
+
+    @Override
+    public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context)
+            throws AlgebricksException {
+
+        AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue();
+        if (op.getOperatorTag() != LogicalOperatorTag.SELECT) {
+            return false;
+        }
+        SelectOperator selectOperator = (SelectOperator) op;
+
+        Mutable<ILogicalOperator> childOfSelect = selectOperator.getInputs().get(0);
+
+        //[Direction] SelectOp(cond1)<--ChildOps... ==> SelectOp(booleanValue of cond1)<--NewAssignOp(cond1)<--ChildOps...
+        //#. Create an assign-operator with a new local variable and the condition of the select-operator.
+        //#. Set the input(child operator) of the new assign-operator to input(child operator) of the select-operator.
+        //   (Later, the newly created assign-operator will apply the condition on behalf of the select-operator,
+        //    and set the variable of the assign-operator to a boolean value according to the condition evaluation.)
+        //#. Give the select-operator the result boolean value created by the newly created child assign-operator.
+
+        //create an assignOp with a variable and the condition of the select-operator.
+        LogicalVariable v = context.newVar();
+        AssignOperator assignOperator = new AssignOperator(v, new MutableObject<ILogicalExpression>(selectOperator
+                .getCondition().getValue()));
+
+        //set the input of the new assign-operator to the input of the select-operator.
+        assignOperator.getInputs().add(childOfSelect);
+
+        //set the result value of the assign-operator to the condition of the select-operator
+        selectOperator.getCondition().setValue(new VariableReferenceExpression(v));//scalarFunctionCallExpression);
+        selectOperator.getInputs().set(0, new MutableObject<ILogicalOperator>(assignOperator));
+
+        context.computeAndSetTypeEnvironmentForOperator(assignOperator);
+
+        //Once this rule is fired, don't apply again.
+        context.addToDontApplySet(this, selectOperator);
+        return true;
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/34d81630/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/IntroduceUnionRule.java
----------------------------------------------------------------------
diff --git a/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/IntroduceUnionRule.java b/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/IntroduceUnionRule.java
new file mode 100644
index 0000000..050a2cd
--- /dev/null
+++ b/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/IntroduceUnionRule.java
@@ -0,0 +1,123 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package edu.uci.ics.asterix.optimizer.rules;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.commons.lang3.mutable.Mutable;
+
+import edu.uci.ics.asterix.om.functions.AsterixBuiltinFunctions;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.common.utils.Triple;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalExpressionTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.AbstractFunctionCallExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractBinaryJoinOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AssignOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.UnionAllOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.visitors.VariableUtilities;
+import edu.uci.ics.hyracks.algebricks.core.algebra.util.OperatorPropertiesUtil;
+import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
+
+/**
+ * @author kereno, ecarm002, ildar.absalyamov
+ *         Generates a union operator and puts it instead of "assign <- [function-call: asterix:union]"
+ *         Before rule:
+ *         ============
+ *         assign [var] <- [asterix:union(left_branch, right_branch)]
+ *         join (TRUE)
+ *         left_branch
+ *         right_branch
+ *         After rule:
+ *         ============
+ *         union (left_branch, right_branch, result_var)
+ *         left_branch
+ *         right_branch
+ */
+public class IntroduceUnionRule implements IAlgebraicRewriteRule {
+
+    @Override
+    public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) {
+        return false;
+    }
+
+    @Override
+    public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context)
+            throws AlgebricksException {
+
+        if (!opRef.getValue().getOperatorTag().equals(LogicalOperatorTag.ASSIGN)) {
+            return false;
+        }
+
+        AssignOperator assignUnion = (AssignOperator) opRef.getValue();
+
+        if (assignUnion.getExpressions().get(0).getValue().getExpressionTag() != LogicalExpressionTag.FUNCTION_CALL)
+            return false;
+
+        AbstractFunctionCallExpression u = (AbstractFunctionCallExpression) assignUnion.getExpressions().get(0)
+                .getValue();
+        if (!AsterixBuiltinFunctions.UNION.equals(u.getFunctionIdentifier())) {
+            return false;
+        }
+
+        //Retrieving the logical variables for the union from the two aggregates which are inputs to the join
+        Mutable<ILogicalOperator> join = assignUnion.getInputs().get(0);
+
+        LogicalOperatorTag tag1 = join.getValue().getOperatorTag();
+        if (tag1 != LogicalOperatorTag.INNERJOIN && tag1 != LogicalOperatorTag.LEFTOUTERJOIN) {
+            return false;
+        }
+        AbstractBinaryJoinOperator join1 = (AbstractBinaryJoinOperator) join.getValue();
+        ILogicalExpression cond1 = join1.getCondition().getValue();
+        // don't try to push a product down
+        if (!OperatorPropertiesUtil.isAlwaysTrueCond(cond1)) {
+            return false;
+        }
+
+        List<Mutable<ILogicalOperator>> joinInputs = join.getValue().getInputs();
+
+        Mutable<ILogicalOperator> left_branch = joinInputs.get(0);
+        Mutable<ILogicalOperator> right_branch = joinInputs.get(1);
+
+        List<LogicalVariable> input1Var = new ArrayList<LogicalVariable>();
+        VariableUtilities.getProducedVariables(left_branch.getValue(), input1Var);
+
+        List<LogicalVariable> input2Var = new ArrayList<LogicalVariable>();
+        VariableUtilities.getProducedVariables(right_branch.getValue(), input2Var);
+
+        List<Triple<LogicalVariable, LogicalVariable, LogicalVariable>> varMap = new ArrayList<Triple<LogicalVariable, LogicalVariable, LogicalVariable>>(
+                1);
+        Triple<LogicalVariable, LogicalVariable, LogicalVariable> triple = new Triple<LogicalVariable, LogicalVariable, LogicalVariable>(
+                input1Var.get(0), input2Var.get(0), assignUnion.getVariables().get(0));
+        varMap.add(triple);
+        UnionAllOperator unionOp = new UnionAllOperator(varMap);
+
+        unionOp.getInputs().add(left_branch);
+        unionOp.getInputs().add(right_branch);
+
+        context.computeAndSetTypeEnvironmentForOperator(unionOp);
+
+        opRef.setValue(unionOp);
+
+        return true;
+
+    }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/34d81630/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/IntroduceUnnestForCollectionToSequenceRule.java
----------------------------------------------------------------------
diff --git a/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/IntroduceUnnestForCollectionToSequenceRule.java b/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/IntroduceUnnestForCollectionToSequenceRule.java
new file mode 100644
index 0000000..5943911
--- /dev/null
+++ b/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/IntroduceUnnestForCollectionToSequenceRule.java
@@ -0,0 +1,94 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.asterix.optimizer.rules;
+
+import java.util.List;
+
+import org.apache.commons.lang3.mutable.Mutable;
+import org.apache.commons.lang3.mutable.MutableObject;
+
+import edu.uci.ics.asterix.aql.util.FunctionUtils;
+import edu.uci.ics.asterix.om.functions.AsterixBuiltinFunctions;
+import edu.uci.ics.asterix.om.types.IAType;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalExpressionTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.AbstractFunctionCallExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.UnnestingFunctionCallExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AssignOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.UnnestOperator;
+import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
+
+/**
+ * This rule introduces a unnest operator for the collection-to-sequence function (if the input to the function is a collection).
+ *
+ * @author yingyib
+ */
+public class IntroduceUnnestForCollectionToSequenceRule implements IAlgebraicRewriteRule {
+
+    @Override
+    public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException {
+        return false;
+    }
+
+    @Override
+    public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context)
+            throws AlgebricksException {
+        AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue();
+        if (op.getOperatorTag() != LogicalOperatorTag.ASSIGN) {
+            return false;
+        }
+        AssignOperator assign = (AssignOperator) op;
+        List<Mutable<ILogicalExpression>> exprs = assign.getExpressions();
+        if (exprs.size() != 1) {
+            return false;
+        }
+        ILogicalExpression expr = exprs.get(0).getValue();
+        if (expr.getExpressionTag() != LogicalExpressionTag.FUNCTION_CALL) {
+            return false;
+        }
+        AbstractFunctionCallExpression func = (AbstractFunctionCallExpression) expr;
+        if (func.getFunctionIdentifier() != AsterixBuiltinFunctions.COLLECTION_TO_SEQUENCE) {
+            return false;
+        }
+
+        IVariableTypeEnvironment env = assign.computeInputTypeEnvironment(context);
+        ILogicalExpression argExpr = func.getArguments().get(0).getValue();
+        IAType outerExprType = (IAType) env.getType(expr);
+        IAType innerExprType = (IAType) env.getType(argExpr);
+        if (outerExprType.equals(innerExprType)) {
+            /** nothing is changed with the collection-to-sequence function, remove the collection-sequence function call */
+            assign.getExpressions().set(0, new MutableObject<ILogicalExpression>(argExpr));
+            return true;
+        }
+        /** change the assign operator to an unnest operator */
+        LogicalVariable var = assign.getVariables().get(0);
+        @SuppressWarnings("unchecked")
+        UnnestOperator unnest = new UnnestOperator(var, new MutableObject<ILogicalExpression>(
+                new UnnestingFunctionCallExpression(
+                        FunctionUtils.getFunctionInfo(AsterixBuiltinFunctions.SCAN_COLLECTION),
+                        new MutableObject<ILogicalExpression>(argExpr))));
+        unnest.getInputs().addAll(assign.getInputs());
+        opRef.setValue(unnest);
+        context.computeAndSetTypeEnvironmentForOperator(unnest);
+        return true;
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/34d81630/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/LoadRecordFieldsRule.java
----------------------------------------------------------------------
diff --git a/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/LoadRecordFieldsRule.java b/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/LoadRecordFieldsRule.java
new file mode 100644
index 0000000..54e470c
--- /dev/null
+++ b/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/LoadRecordFieldsRule.java
@@ -0,0 +1,368 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.asterix.optimizer.rules;
+
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.LinkedList;
+import java.util.List;
+
+import org.apache.commons.lang3.mutable.Mutable;
+import org.apache.commons.lang3.mutable.MutableObject;
+
+import edu.uci.ics.asterix.algebra.base.AsterixOperatorAnnotations;
+import edu.uci.ics.asterix.common.exceptions.AsterixRuntimeException;
+import edu.uci.ics.asterix.om.base.AString;
+import edu.uci.ics.asterix.om.constants.AsterixConstantValue;
+import edu.uci.ics.asterix.om.functions.AsterixBuiltinFunctions;
+import edu.uci.ics.asterix.optimizer.base.AnalysisUtil;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalExpressionTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.AbstractFunctionCallExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.AbstractLogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.ConstantExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.VariableReferenceExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.functions.AlgebricksBuiltinFunctions;
+import edu.uci.ics.hyracks.algebricks.core.algebra.functions.FunctionIdentifier;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractOperatorWithNestedPlans;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AssignOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.NestedTupleSourceOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SelectOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.visitors.VariableUtilities;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.FunctionalDependency;
+import edu.uci.ics.hyracks.algebricks.core.algebra.util.OperatorPropertiesUtil;
+import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalExpressionReferenceTransform;
+import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
+
+public class LoadRecordFieldsRule implements IAlgebraicRewriteRule {
+
+    private ExtractFieldLoadExpressionVisitor exprVisitor = new ExtractFieldLoadExpressionVisitor();
+
+    @Override
+    public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) {
+        return false;
+    }
+
+    @Override
+    public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context)
+            throws AlgebricksException {
+        AbstractLogicalOperator op1 = (AbstractLogicalOperator) opRef.getValue();
+        if (context.checkIfInDontApplySet(this, op1)) {
+            return false;
+        }
+
+        if (op1.getOperatorTag() == LogicalOperatorTag.ASSIGN) {
+            AssignOperator a1 = (AssignOperator) op1;
+            ILogicalExpression expr = getFirstExpr(a1);
+            if (AnalysisUtil.isAccessToFieldRecord(expr)) {
+                boolean res = findAndEliminateRedundantFieldAccess(a1);
+                context.addToDontApplySet(this, op1);
+                return res;
+            }
+        }
+        exprVisitor.setTopOp(op1);
+        exprVisitor.setContext(context);
+        boolean res = op1.acceptExpressionTransform(exprVisitor);
+        if (!res) {
+            context.addToDontApplySet(this, op1);
+        }
+        if (res && op1.getOperatorTag() == LogicalOperatorTag.SELECT) {
+            // checking if we can annotate a Selection as using just one field
+            // access
+            SelectOperator sigma = (SelectOperator) op1;
+            LinkedList<LogicalVariable> vars = new LinkedList<LogicalVariable>();
+            VariableUtilities.getUsedVariables(sigma, vars);
+            if (vars.size() == 1) {
+                // we can annotate Selection
+                AssignOperator assign1 = (AssignOperator) op1.getInputs().get(0).getValue();
+                AbstractLogicalExpression expr1 = (AbstractLogicalExpression) getFirstExpr(assign1);
+                if (expr1.getExpressionTag() == LogicalExpressionTag.FUNCTION_CALL) {
+                    AbstractFunctionCallExpression f = (AbstractFunctionCallExpression) expr1;
+                    // f should be a call to a field/data access kind of
+                    // function
+                    sigma.getAnnotations().put(AsterixOperatorAnnotations.FIELD_ACCESS, f.getArguments().get(0));
+                }
+            }
+        }
+
+        // TODO: avoid having to recompute type env. here
+        if (res) {
+            OperatorPropertiesUtil.typeOpRec(opRef, context);
+        }
+        return res;
+    }
+
+    private static boolean pushFieldLoads(Mutable<ILogicalExpression> exprRef, AbstractLogicalOperator topOp,
+            IOptimizationContext context) throws AlgebricksException {
+        ILogicalExpression expr = exprRef.getValue();
+        if (expr == null) {
+            return false;
+        }
+        switch (expr.getExpressionTag()) {
+            case FUNCTION_CALL: {
+                AbstractFunctionCallExpression f = (AbstractFunctionCallExpression) expr;
+                FunctionIdentifier fi = f.getFunctionIdentifier();
+                if (AlgebricksBuiltinFunctions.isComparisonFunction(fi)) {
+                    boolean b1 = pushFieldLoads(f.getArguments().get(0), topOp, context);
+                    boolean b2 = pushFieldLoads(f.getArguments().get(1), topOp, context);
+                    return b1 || b2;
+                }
+                if (fi.equals(AsterixBuiltinFunctions.FIELD_ACCESS_BY_NAME)) {
+                    if (AnalysisUtil.numberOfVarsInExpr(f) == 0) {
+                        return false;
+                    }
+                    // create an assign
+                    LogicalVariable v = context.newVar();
+                    AssignOperator a2 = new AssignOperator(v, new MutableObject<ILogicalExpression>(f));
+                    pushFieldAssign(a2, topOp, context);
+                    context.computeAndSetTypeEnvironmentForOperator(a2);
+                    ILogicalExpression arg = f.getArguments().get(0).getValue();
+                    if (arg.getExpressionTag() == LogicalExpressionTag.VARIABLE) {
+                        VariableReferenceExpression ref = (VariableReferenceExpression) arg;
+                        LogicalVariable var = ref.getVariableReference();
+                        List<LogicalVariable> keys = context.findPrimaryKey(var);
+                        if (keys != null) {
+                            List<LogicalVariable> tail = new ArrayList<LogicalVariable>();
+                            tail.add(v);
+                            FunctionalDependency pk = new FunctionalDependency(keys, tail);
+                            context.addPrimaryKey(pk);
+                        }
+                    }
+                    exprRef.setValue(new VariableReferenceExpression(v));
+                    return true;
+                } else {
+                    boolean pushed = false;
+                    for (Mutable<ILogicalExpression> argRef : f.getArguments()) {
+                        if (pushFieldLoads(argRef, topOp, context)) {
+                            pushed = true;
+                        }
+                    }
+                    return pushed;
+                }
+            }
+            case CONSTANT:
+            case VARIABLE: {
+                return false;
+            }
+            default: {
+                assert false;
+                throw new IllegalArgumentException();
+            }
+        }
+    }
+
+    private static void pushFieldAssign(AssignOperator a2, AbstractLogicalOperator topOp, IOptimizationContext context)
+            throws AlgebricksException {
+        if (topOp.getInputs().size() == 1 && !topOp.hasNestedPlans()) {
+            Mutable<ILogicalOperator> topChild = topOp.getInputs().get(0);
+            // plugAccessAboveOp(a2, topChild, context);
+            List<Mutable<ILogicalOperator>> a2InptList = a2.getInputs();
+            a2InptList.clear();
+            a2InptList.add(topChild);
+            // and link it as child in the op. tree
+            topOp.getInputs().set(0, new MutableObject<ILogicalOperator>(a2));
+            findAndEliminateRedundantFieldAccess(a2);
+        } else { // e.g., a join
+            LinkedList<LogicalVariable> usedInAccess = new LinkedList<LogicalVariable>();
+            VariableUtilities.getUsedVariables(a2, usedInAccess);
+
+            LinkedList<LogicalVariable> produced2 = new LinkedList<LogicalVariable>();
+            VariableUtilities.getProducedVariables(topOp, produced2);
+
+            if (OperatorPropertiesUtil.disjoint(produced2, usedInAccess)) {
+                for (Mutable<ILogicalOperator> inp : topOp.getInputs()) {
+                    HashSet<LogicalVariable> v2 = new HashSet<LogicalVariable>();
+                    VariableUtilities.getLiveVariables(inp.getValue(), v2);
+                    if (!OperatorPropertiesUtil.disjoint(usedInAccess, v2)) {
+                        pushAccessAboveOpRef(a2, inp, context);
+                        return;
+                    }
+                }
+                if (topOp.hasNestedPlans()) {
+                    AbstractOperatorWithNestedPlans nestedOp = (AbstractOperatorWithNestedPlans) topOp;
+                    for (ILogicalPlan plan : nestedOp.getNestedPlans()) {
+                        for (Mutable<ILogicalOperator> root : plan.getRoots()) {
+                            HashSet<LogicalVariable> v2 = new HashSet<LogicalVariable>();
+                            VariableUtilities.getLiveVariables(root.getValue(), v2);
+                            if (!OperatorPropertiesUtil.disjoint(usedInAccess, v2)) {
+                                pushAccessAboveOpRef(a2, root, context);
+                                return;
+                            }
+                        }
+                    }
+                }
+                throw new AsterixRuntimeException("Field access " + getFirstExpr(a2)
+                        + " does not correspond to any input of operator " + topOp);
+            }
+        }
+    }
+
+    /**
+     * Pushes one field-access assignment above toPushThroughChildRef
+     * 
+     * @param toPush
+     * @param toPushThroughChildRef
+     */
+    private static void pushAccessAboveOpRef(AssignOperator toPush, Mutable<ILogicalOperator> toPushThroughChildRef,
+            IOptimizationContext context) throws AlgebricksException {
+        List<Mutable<ILogicalOperator>> tpInpList = toPush.getInputs();
+        tpInpList.clear();
+        tpInpList.add(new MutableObject<ILogicalOperator>(toPushThroughChildRef.getValue()));
+        toPushThroughChildRef.setValue(toPush);
+        findAndEliminateRedundantFieldAccess(toPush);
+    }
+
+    /**
+     * Rewrite
+     * assign $x := field-access($y, "field")
+     * assign $y := record-constructor { "field": Expr, ... }
+     * into
+     * assign $x := Expr
+     * assign $y := record-constructor { "field": Expr, ... }
+     * 
+     * @param toPush
+     */
+    private static boolean findAndEliminateRedundantFieldAccess(AssignOperator assign) throws AlgebricksException {
+        ILogicalExpression expr = getFirstExpr(assign);
+        AbstractFunctionCallExpression f = (AbstractFunctionCallExpression) expr;
+        ILogicalExpression arg0 = f.getArguments().get(0).getValue();
+        if (arg0.getExpressionTag() != LogicalExpressionTag.VARIABLE) {
+            return false;
+        }
+        VariableReferenceExpression vre = (VariableReferenceExpression) arg0;
+        LogicalVariable recordVar = vre.getVariableReference();
+        ILogicalExpression arg1 = f.getArguments().get(1).getValue();
+        if (arg1.getExpressionTag() != LogicalExpressionTag.CONSTANT) {
+            return false;
+        }
+        ConstantExpression ce = (ConstantExpression) arg1;
+        if (f.getFunctionIdentifier().equals(AsterixBuiltinFunctions.FIELD_ACCESS_BY_NAME)) {
+            String fldName = ((AString) ((AsterixConstantValue) ce.getValue()).getObject()).getStringValue();
+            ILogicalExpression fldExpr = findFieldExpression(assign, recordVar, fldName);
+
+            if (fldExpr != null) {
+                // check the liveness of the new expression
+                List<LogicalVariable> usedVariables = new ArrayList<LogicalVariable>();
+                fldExpr.getUsedVariables(usedVariables);
+                List<LogicalVariable> liveInputVars = new ArrayList<LogicalVariable>();
+                VariableUtilities.getLiveVariables(assign, liveInputVars);
+                usedVariables.removeAll(liveInputVars);
+                if (usedVariables.size() == 0) {
+                    assign.getExpressions().get(0).setValue(fldExpr);
+                    return true;
+                } else {
+                    return false;
+                }
+            } else {
+                return false;
+            }
+        } else if (f.getFunctionIdentifier().equals(AsterixBuiltinFunctions.FIELD_ACCESS_BY_INDEX)) {
+            // int fldIdx = ((IntegerLiteral) ce.getValue()).getValue();
+            // TODO
+            return false;
+        } else if (f.getFunctionIdentifier().equals(AsterixBuiltinFunctions.FIELD_ACCESS_NESTED)) {
+            return false;
+        } else {
+            throw new IllegalStateException();
+        }
+    }
+
+    private static ILogicalExpression findFieldExpression(AbstractLogicalOperator op, LogicalVariable recordVar,
+            String fldName) {
+        for (Mutable<ILogicalOperator> child : op.getInputs()) {
+            AbstractLogicalOperator opChild = (AbstractLogicalOperator) child.getValue();
+            if (opChild.getOperatorTag() == LogicalOperatorTag.ASSIGN) {
+                AssignOperator op2 = (AssignOperator) opChild;
+                int i = 0;
+                for (LogicalVariable var : op2.getVariables()) {
+                    if (var == recordVar) {
+                        AbstractLogicalExpression constr = (AbstractLogicalExpression) op2.getExpressions().get(i)
+                                .getValue();
+                        if (constr.getExpressionTag() != LogicalExpressionTag.FUNCTION_CALL) {
+                            return null;
+                        }
+                        AbstractFunctionCallExpression fce = (AbstractFunctionCallExpression) constr;
+                        if (!fce.getFunctionIdentifier().equals(AsterixBuiltinFunctions.OPEN_RECORD_CONSTRUCTOR)
+                                && !fce.getFunctionIdentifier().equals(
+                                        AsterixBuiltinFunctions.CLOSED_RECORD_CONSTRUCTOR)) {
+                            return null;
+                        }
+                        Iterator<Mutable<ILogicalExpression>> fldIter = fce.getArguments().iterator();
+                        while (fldIter.hasNext()) {
+                            ILogicalExpression fldExpr = fldIter.next().getValue();
+                            if (fldExpr.getExpressionTag() == LogicalExpressionTag.CONSTANT) {
+                                ConstantExpression ce = (ConstantExpression) fldExpr;
+                                String f2 = ((AString) ((AsterixConstantValue) ce.getValue()).getObject())
+                                        .getStringValue();
+                                if (fldName.equals(f2)) {
+                                    return fldIter.next().getValue();
+                                }
+                            }
+                            fldIter.next();
+                        }
+                        return null;
+                    }
+                    i++;
+                }
+            } else if (opChild.getOperatorTag() == LogicalOperatorTag.NESTEDTUPLESOURCE) {
+                NestedTupleSourceOperator nts = (NestedTupleSourceOperator) opChild;
+                AbstractLogicalOperator opBelowNestedPlan = (AbstractLogicalOperator) nts.getDataSourceReference()
+                        .getValue().getInputs().get(0).getValue();
+                ILogicalExpression expr1 = findFieldExpression(opBelowNestedPlan, recordVar, fldName);
+                if (expr1 != null) {
+                    return expr1;
+                }
+            }
+            ILogicalExpression expr2 = findFieldExpression(opChild, recordVar, fldName);
+            if (expr2 != null) {
+                return expr2;
+            }
+        }
+        return null;
+    }
+
+    private final class ExtractFieldLoadExpressionVisitor implements ILogicalExpressionReferenceTransform {
+
+        private AbstractLogicalOperator topOp;
+        private IOptimizationContext context;
+
+        public void setTopOp(AbstractLogicalOperator topOp) {
+            this.topOp = topOp;
+        }
+
+        public void setContext(IOptimizationContext context) {
+            this.context = context;
+        }
+
+        @Override
+        public boolean transform(Mutable<ILogicalExpression> exprRef) throws AlgebricksException {
+            return pushFieldLoads(exprRef, topOp, context);
+        }
+
+    }
+
+    private static ILogicalExpression getFirstExpr(AssignOperator assign) {
+        return assign.getExpressions().get(0).getValue();
+    }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/34d81630/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/NestGroupByRule.java
----------------------------------------------------------------------
diff --git a/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/NestGroupByRule.java b/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/NestGroupByRule.java
new file mode 100644
index 0000000..3ae35bc
--- /dev/null
+++ b/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/NestGroupByRule.java
@@ -0,0 +1,191 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.asterix.optimizer.rules;
+
+import java.util.HashSet;
+import java.util.Set;
+
+import org.apache.commons.lang3.mutable.Mutable;
+
+import edu.uci.ics.asterix.om.functions.AsterixBuiltinFunctions;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalExpressionTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.AbstractFunctionCallExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.VariableReferenceExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AggregateOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.GroupByOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.NestedTupleSourceOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SubplanOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.UnnestOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.visitors.VariableUtilities;
+import edu.uci.ics.hyracks.algebricks.core.algebra.util.OperatorPropertiesUtil;
+import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
+
+public class NestGroupByRule implements IAlgebraicRewriteRule {
+
+    @Override
+    public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException {
+        return false;
+    }
+
+    @Override
+    public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context)
+            throws AlgebricksException {
+        AbstractLogicalOperator op1 = (AbstractLogicalOperator) opRef.getValue();
+        if (op1.getOperatorTag() != LogicalOperatorTag.SUBPLAN) {
+            return false;
+        }
+        SubplanOperator subplan = (SubplanOperator) op1;
+        if (subplan.getNestedPlans().size() != 1) {
+            return false;
+        }
+        ILogicalPlan p = subplan.getNestedPlans().get(0);
+        if (p.getRoots().size() != 1) {
+            return false;
+        }
+
+        Set<LogicalVariable> free = new HashSet<LogicalVariable>();
+        OperatorPropertiesUtil.getFreeVariablesInSubplans(subplan, free);
+        if (free.size() != 1) {
+            return false;
+        }
+        LogicalVariable fVar = null;
+        for (LogicalVariable v : free) {
+            fVar = v;
+            break;
+        }
+
+        AbstractLogicalOperator op2 = (AbstractLogicalOperator) op1.getInputs().get(0).getValue();
+        if (op2.getOperatorTag() != LogicalOperatorTag.GROUP) {
+            return false;
+        }
+        GroupByOperator gby = (GroupByOperator) op2;
+        if (gby.getNestedPlans().size() != 1) {
+            return false;
+        }
+        ILogicalPlan p2 = gby.getNestedPlans().get(0);
+        if (p2.getRoots().size() != 1) {
+            return false;
+        }
+        Mutable<ILogicalOperator> r2 = p2.getRoots().get(0);
+        AbstractLogicalOperator opr2 = (AbstractLogicalOperator) r2.getValue();
+        if (opr2.getOperatorTag() != LogicalOperatorTag.AGGREGATE) {
+            return false;
+        }
+        AggregateOperator aggOuter = (AggregateOperator) opr2;
+        int posInAggList = aggOuter.getVariables().indexOf(fVar);
+        if (posInAggList < 0) {
+            return false;
+        }
+        AbstractLogicalOperator outerAggSon = (AbstractLogicalOperator) aggOuter.getInputs().get(0).getValue();
+        if (outerAggSon.getOperatorTag() != LogicalOperatorTag.NESTEDTUPLESOURCE) {
+            return false;
+        }
+        ILogicalExpression eAgg = aggOuter.getExpressions().get(posInAggList).getValue();
+        if (eAgg.getExpressionTag() != LogicalExpressionTag.FUNCTION_CALL) {
+            return false;
+        }
+        AbstractFunctionCallExpression listifyCall = (AbstractFunctionCallExpression) eAgg;
+        if (listifyCall.getFunctionIdentifier() != AsterixBuiltinFunctions.LISTIFY) {
+            return false;
+        }
+        ILogicalExpression argListify = listifyCall.getArguments().get(0).getValue();
+        if (argListify.getExpressionTag() != LogicalExpressionTag.VARIABLE) {
+            return false;
+        }
+
+        Mutable<ILogicalOperator> r = p.getRoots().get(0);
+        AbstractLogicalOperator opInS = (AbstractLogicalOperator) r.getValue();
+        if (opInS.getOperatorTag() != LogicalOperatorTag.AGGREGATE) {
+            return false;
+        }
+        AggregateOperator aggInner = (AggregateOperator) opInS;
+        do {
+            opInS = (AbstractLogicalOperator) opInS.getInputs().get(0).getValue();
+        } while (opInS.getOperatorTag() == LogicalOperatorTag.ASSIGN);
+        if (opInS.getOperatorTag() != LogicalOperatorTag.GROUP) {
+            return false;
+        }
+        AbstractLogicalOperator unnestParent = opInS;
+        AbstractLogicalOperator opUnder = (AbstractLogicalOperator) opInS.getInputs().get(0).getValue();
+        // skip Assigns
+        while (opUnder.getOperatorTag() == LogicalOperatorTag.ASSIGN) {
+            unnestParent = opUnder;
+            opUnder = (AbstractLogicalOperator) opUnder.getInputs().get(0).getValue();
+        }
+        if (opUnder.getOperatorTag() != LogicalOperatorTag.UNNEST) {
+            return false;
+        }
+        UnnestOperator unnest = (UnnestOperator) opUnder;
+        AbstractLogicalOperator unnestSon = (AbstractLogicalOperator) unnest.getInputs().get(0).getValue();
+        if (unnestSon.getOperatorTag() != LogicalOperatorTag.NESTEDTUPLESOURCE) {
+            return false;
+        }
+        NestedTupleSourceOperator innerNts = (NestedTupleSourceOperator) unnestSon;
+
+        ILogicalExpression eUnnest = unnest.getExpressionRef().getValue();
+        if (eUnnest.getExpressionTag() != LogicalExpressionTag.FUNCTION_CALL) {
+            return false;
+        }
+        AbstractFunctionCallExpression uf = (AbstractFunctionCallExpression) eUnnest;
+        if (uf.getFunctionIdentifier() != AsterixBuiltinFunctions.SCAN_COLLECTION) {
+            return false;
+        }
+        ILogicalExpression scanArg = uf.getArguments().get(0).getValue();
+        if (scanArg.getExpressionTag() != LogicalExpressionTag.VARIABLE) {
+            return false;
+        }
+        if (((VariableReferenceExpression) scanArg).getVariableReference() != fVar) {
+            return false;
+        }
+        LogicalVariable uVar = unnest.getVariable();
+        GroupByOperator innerGby = (GroupByOperator) opInS;
+        Set<LogicalVariable> freeInInnerGby = new HashSet<LogicalVariable>();
+        OperatorPropertiesUtil.getFreeVariablesInSubplans(innerGby, freeInInnerGby);
+        for (LogicalVariable v : freeInInnerGby) {
+            if (v != uVar) {
+                return false;
+            }
+        }
+
+        unnestParent.getInputs().get(0).setValue(innerNts);
+        LogicalVariable listifiedVar = ((VariableReferenceExpression) argListify).getVariableReference();
+        substInSubplan(aggInner, uVar, listifiedVar, context);
+        gby.getNestedPlans().add(p);
+        innerNts.getDataSourceReference().setValue(gby);
+        opRef.setValue(gby);
+        OperatorPropertiesUtil.typePlan(p, context);
+        OperatorPropertiesUtil.typePlan(p2, context);
+        context.computeAndSetTypeEnvironmentForOperator(gby);
+        return true;
+
+    }
+
+    private void substInSubplan(AggregateOperator aggInner, LogicalVariable v1, LogicalVariable v2,
+            IOptimizationContext context) throws AlgebricksException {
+        ILogicalOperator op = aggInner;
+        while (op.getInputs().size() == 1) {
+            VariableUtilities.substituteVariables(op, v1, v2, context);
+            op = op.getInputs().get(0).getValue();
+        }
+    }
+}



Mime
View raw message