asterixdb-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ima...@apache.org
Subject [49/58] [abbrv] [partial] incubator-asterixdb git commit: Added support of typed indexes over open fields & indexes over nested fields
Date Fri, 24 Apr 2015 18:43:15 GMT
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/04b2b77a/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/am/AccessMethodUtils.java
----------------------------------------------------------------------
diff --git a/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/am/AccessMethodUtils.java b/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/am/AccessMethodUtils.java
index a10a937..73377b9 100644
--- a/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/am/AccessMethodUtils.java
+++ b/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/am/AccessMethodUtils.java
@@ -59,9 +59,9 @@ import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.UnnestingFunction
 import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.VariableReferenceExpression;
 import edu.uci.ics.hyracks.algebricks.core.algebra.functions.AlgebricksBuiltinFunctions;
 import edu.uci.ics.hyracks.algebricks.core.algebra.functions.IFunctionInfo;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractDataSourceOperator;
 import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
 import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator.ExecutionMode;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.DataSourceScanOperator;
 import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ExternalDataLookupOperator;
 import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.GroupByOperator;
 import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.OrderOperator;
@@ -77,11 +77,11 @@ import edu.uci.ics.hyracks.algebricks.core.algebra.util.OperatorPropertiesUtil;
 public class AccessMethodUtils {
 
     public static void appendPrimaryIndexTypes(Dataset dataset, IAType itemType, List<Object> target)
-            throws IOException {
+            throws IOException, AlgebricksException {
         ARecordType recordType = (ARecordType) itemType;
-        List<String> partitioningKeys = DatasetUtils.getPartitioningKeys(dataset);
-        for (String partitioningKey : partitioningKeys) {
-            target.add(recordType.getFieldType(partitioningKey));
+        List<List<String>> partitioningKeys = DatasetUtils.getPartitioningKeys(dataset);
+        for (List<String> partitioningKey : partitioningKeys) {
+            target.add(recordType.getSubFieldType(partitioningKey));
         }
         target.add(itemType);
     }
@@ -143,7 +143,13 @@ public class AccessMethodUtils {
         } else {
             return false;
         }
-        analysisCtx.matchedFuncExprs.add(new OptimizableFuncExpr(funcExpr, fieldVar, constFilterVal));
+        OptimizableFuncExpr newOptFuncExpr = new OptimizableFuncExpr(funcExpr, fieldVar, constFilterVal);
+        for (IOptimizableFuncExpr optFuncExpr : analysisCtx.matchedFuncExprs) {
+            //avoid additional optFuncExpressions in case of a join
+            if (optFuncExpr.getFuncExpr().equals(funcExpr))
+                return true;
+        }
+        analysisCtx.matchedFuncExprs.add(newOptFuncExpr);
         return true;
     }
 
@@ -160,8 +166,14 @@ public class AccessMethodUtils {
         } else {
             return false;
         }
-        analysisCtx.matchedFuncExprs.add(new OptimizableFuncExpr(funcExpr,
-                new LogicalVariable[] { fieldVar1, fieldVar2 }, null));
+        OptimizableFuncExpr newOptFuncExpr = new OptimizableFuncExpr(funcExpr, new LogicalVariable[] { fieldVar1,
+                fieldVar2 }, null);
+        for (IOptimizableFuncExpr optFuncExpr : analysisCtx.matchedFuncExprs) {
+            //avoid additional optFuncExpressions in case of a join
+            if (optFuncExpr.getFuncExpr().equals(funcExpr))
+                return true;
+        }
+        analysisCtx.matchedFuncExprs.add(newOptFuncExpr);
         return true;
     }
 
@@ -175,8 +187,8 @@ public class AccessMethodUtils {
                 return index.getKeyFieldNames().size();
             }
             case RTREE: {
-                Pair<IAType, Boolean> keyPairType = Index.getNonNullableKeyFieldType(index.getKeyFieldNames().get(0),
-                        recordType);
+                Pair<IAType, Boolean> keyPairType = Index.getNonNullableOpenFieldType(index.getKeyFieldTypes().get(0),
+                        index.getKeyFieldNames().get(0), recordType);
                 IAType keyType = keyPairType.first;
                 int numDimensions = NonTaggedFormatUtil.getNumDimensions(keyType.getTypeTag());
                 return numDimensions * 2;
@@ -197,15 +209,16 @@ public class AccessMethodUtils {
                 case BTREE:
                 case SINGLE_PARTITION_WORD_INVIX:
                 case SINGLE_PARTITION_NGRAM_INVIX: {
-                    for (String sk : index.getKeyFieldNames()) {
-                        Pair<IAType, Boolean> keyPairType = Index.getNonNullableKeyFieldType(sk, recordType);
+                    for (int i = 0; i < index.getKeyFieldNames().size(); i++) {
+                        Pair<IAType, Boolean> keyPairType = Index.getNonNullableOpenFieldType(index.getKeyFieldTypes()
+                                .get(i), index.getKeyFieldNames().get(i), recordType);
                         dest.add(keyPairType.first);
                     }
                     break;
                 }
                 case RTREE: {
-                    Pair<IAType, Boolean> keyPairType = Index.getNonNullableKeyFieldType(index.getKeyFieldNames()
-                            .get(0), recordType);
+                    Pair<IAType, Boolean> keyPairType = Index.getNonNullableOpenFieldType(
+                            index.getKeyFieldTypes().get(0), index.getKeyFieldNames().get(0), recordType);
                     IAType keyType = keyPairType.first;
                     IAType nestedKeyType = NonTaggedFormatUtil.getNestedSpatialType(keyType.getTypeTag());
                     int numKeys = getNumSecondaryKeys(index, recordType);
@@ -229,10 +242,10 @@ public class AccessMethodUtils {
                 throw new AlgebricksException(e);
             }
         } else {
-            List<String> partitioningKeys = DatasetUtils.getPartitioningKeys(dataset);
-            for (String partitioningKey : partitioningKeys) {
+            List<List<String>> partitioningKeys = DatasetUtils.getPartitioningKeys(dataset);
+            for (List<String> partitioningKey : partitioningKeys) {
                 try {
-                    dest.add(recordType.getFieldType(partitioningKey));
+                    dest.add(recordType.getSubFieldType(partitioningKey));
                 } catch (IOException e) {
                     throw new AlgebricksException(e);
                 }
@@ -300,7 +313,7 @@ public class AccessMethodUtils {
         if (probeSubTree == null) {
             // We are optimizing a selection query. Search key is a constant.
             // Type Checking and type promotion is done here
-            ATypeTag fieldType = optFuncExpr.getTypeTag(0);
+            IAType fieldType = optFuncExpr.getFieldType(0);
             IAObject constantObj = ((AsterixConstantValue) optFuncExpr.getConstantVal(0)).getObject();
             ATypeTag constantValueTag = constantObj.getType().getTypeTag();
             // type casting applied?
@@ -310,9 +323,9 @@ public class AccessMethodUtils {
             AsterixConstantValue replacedConstantValue = null;
 
             // if the constant type and target type does not match, we do a type conversion
-            if (constantValueTag != fieldType) {
+            if (constantValueTag != fieldType.getTypeTag()) {
                 replacedConstantValue = ATypeHierarchy.getAsterixConstantValueFromNumericTypeObject(constantObj,
-                        fieldType);
+                        fieldType.getTypeTag());
                 if (replacedConstantValue != null) {
                     typeCastingApplied = true;
                 }
@@ -322,7 +335,7 @@ public class AccessMethodUtils {
                 switch (constantValueTag) {
                     case DOUBLE:
                     case FLOAT:
-                        switch (fieldType) {
+                        switch (fieldType.getTypeTag()) {
                             case INT8:
                             case INT16:
                             case INT32:
@@ -360,11 +373,16 @@ public class AccessMethodUtils {
      * Returns the first expr optimizable by this index.
      */
     public static IOptimizableFuncExpr chooseFirstOptFuncExpr(Index chosenIndex, AccessMethodAnalysisContext analysisCtx) {
-        List<Integer> indexExprs = analysisCtx.getIndexExprs(chosenIndex);
-        int firstExprIndex = indexExprs.get(0);
+        List<Pair<Integer, Integer>> indexExprs = analysisCtx.getIndexExprs(chosenIndex);
+        int firstExprIndex = indexExprs.get(0).first;
         return analysisCtx.matchedFuncExprs.get(firstExprIndex);
     }
 
+    public static int chooseFirstOptFuncVar(Index chosenIndex, AccessMethodAnalysisContext analysisCtx) {
+        List<Pair<Integer, Integer>> indexExprs = analysisCtx.getIndexExprs(chosenIndex);
+        return indexExprs.get(0).second;
+    }
+
     public static UnnestMapOperator createSecondaryIndexUnnestMap(Dataset dataset, ARecordType recordType, Index index,
             ILogicalOperator inputOp, AccessMethodJobGenParams jobGenParams, IOptimizationContext context,
             boolean outputPrimaryKeysOnly, boolean retainInput) throws AlgebricksException {
@@ -393,9 +411,10 @@ public class AccessMethodUtils {
         return secondaryIndexUnnestOp;
     }
 
-    public static UnnestMapOperator createPrimaryIndexUnnestMap(DataSourceScanOperator dataSourceScan, Dataset dataset,
-            ARecordType recordType, ILogicalOperator inputOp, IOptimizationContext context, boolean sortPrimaryKeys,
-            boolean retainInput, boolean retainNull, boolean requiresBroadcast) throws AlgebricksException {
+    public static UnnestMapOperator createPrimaryIndexUnnestMap(AbstractDataSourceOperator dataSourceOp,
+            Dataset dataset, ARecordType recordType, ILogicalOperator inputOp, IOptimizationContext context,
+            boolean sortPrimaryKeys, boolean retainInput, boolean retainNull, boolean requiresBroadcast)
+            throws AlgebricksException {
         List<LogicalVariable> primaryKeyVars = AccessMethodUtils.getPrimaryKeyVarsFromSecondaryUnnestMap(dataset,
                 inputOp);
         // Optionally add a sort on the primary-index keys before searching the primary index.
@@ -428,7 +447,7 @@ public class AccessMethodUtils {
         List<LogicalVariable> primaryIndexUnnestVars = new ArrayList<LogicalVariable>();
         List<Object> primaryIndexOutputTypes = new ArrayList<Object>();
         // Append output variables/types generated by the primary-index search (not forwarded from input).
-        primaryIndexUnnestVars.addAll(dataSourceScan.getVariables());
+        primaryIndexUnnestVars.addAll(dataSourceOp.getVariables());
         try {
             appendPrimaryIndexTypes(dataset, recordType, primaryIndexOutputTypes);
         } catch (IOException e) {
@@ -537,7 +556,7 @@ public class AccessMethodUtils {
         funcArgs.add(stringRef);
     }
 
-    public static ExternalDataLookupOperator createExternalDataLookupUnnestMap(DataSourceScanOperator dataSourceScan,
+    public static ExternalDataLookupOperator createExternalDataLookupUnnestMap(AbstractDataSourceOperator dataSourceOp,
             Dataset dataset, ARecordType recordType, ILogicalOperator inputOp, IOptimizationContext context,
             Index secondaryIndex, boolean retainInput, boolean retainNull) throws AlgebricksException {
         List<LogicalVariable> primaryKeyVars = AccessMethodUtils.getPrimaryKeyVarsFromSecondaryUnnestMap(dataset,
@@ -565,7 +584,7 @@ public class AccessMethodUtils {
         List<LogicalVariable> externalAccessByRIDVars = new ArrayList<LogicalVariable>();
         List<Object> externalAccessOutputTypes = new ArrayList<Object>();
         // Append output variables/types generated by the data scan (not forwarded from input).
-        externalAccessByRIDVars.addAll(dataSourceScan.getVariables());
+        externalAccessByRIDVars.addAll(dataSourceOp.getVariables());
         appendExternalRecTypes(dataset, recordType, externalAccessOutputTypes);
 
         IFunctionInfo externalAccessByRID = FunctionUtils.getFunctionInfo(AsterixBuiltinFunctions.EXTERNAL_LOOKUP);
@@ -573,7 +592,8 @@ public class AccessMethodUtils {
                 externalRIDAccessFuncArgs);
 
         ExternalDataLookupOperator externalLookupOp = new ExternalDataLookupOperator(externalAccessByRIDVars,
-                new MutableObject<ILogicalExpression>(externalAccessFunc), externalAccessOutputTypes, retainInput);
+                new MutableObject<ILogicalExpression>(externalAccessFunc), externalAccessOutputTypes, retainInput,
+                dataSourceOp.getDataSource());
         // Fed by the order operator or the secondaryIndexUnnestOp.
         externalLookupOp.getInputs().add(new MutableObject<ILogicalOperator>(order));
 

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/04b2b77a/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/am/BTreeAccessMethod.java
----------------------------------------------------------------------
diff --git a/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/am/BTreeAccessMethod.java b/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/am/BTreeAccessMethod.java
index b69e14a..e6a3470 100644
--- a/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/am/BTreeAccessMethod.java
+++ b/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/am/BTreeAccessMethod.java
@@ -50,10 +50,10 @@ import edu.uci.ics.hyracks.algebricks.core.algebra.functions.AlgebricksBuiltinFu
 import edu.uci.ics.hyracks.algebricks.core.algebra.functions.FunctionIdentifier;
 import edu.uci.ics.hyracks.algebricks.core.algebra.functions.IFunctionInfo;
 import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractBinaryJoinOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractDataSourceOperator;
 import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
 import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator.ExecutionMode;
 import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AssignOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.DataSourceScanOperator;
 import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ExternalDataLookupOperator;
 import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SelectOperator;
 import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.UnnestMapOperator;
@@ -214,8 +214,8 @@ public class BTreeAccessMethod implements IAccessMethod {
         Dataset dataset = indexSubTree.dataset;
         ARecordType recordType = indexSubTree.recordType;
         // we made sure indexSubTree has datasource scan
-        DataSourceScanOperator dataSourceScan = (DataSourceScanOperator) indexSubTree.dataSourceRef.getValue();
-        List<Integer> exprList = analysisCtx.indexExprs.get(chosenIndex);
+        AbstractDataSourceOperator dataSourceOp = (AbstractDataSourceOperator) indexSubTree.dataSourceRef.getValue();
+        List<Pair<Integer, Integer>> exprAndVarList = analysisCtx.indexExprsAndVars.get(chosenIndex);
         List<IOptimizableFuncExpr> matchedFuncExprs = analysisCtx.matchedFuncExprs;
         int numSecondaryKeys = analysisCtx.indexNumMatchedKeys.get(chosenIndex);
         // List of function expressions that will be replaced by the secondary-index search.
@@ -245,9 +245,9 @@ public class BTreeAccessMethod implements IAccessMethod {
         // since we have a round issues when dealing with LT(<) OR GT(>) operator.
         boolean realTypeConvertedToIntegerType = false;
 
-        for (Integer exprIndex : exprList) {
+        for (Pair<Integer, Integer> exprIndex : exprAndVarList) {
             // Position of the field of matchedFuncExprs.get(exprIndex) in the chosen index's indexed exprs.
-            IOptimizableFuncExpr optFuncExpr = matchedFuncExprs.get(exprIndex);
+            IOptimizableFuncExpr optFuncExpr = matchedFuncExprs.get(exprIndex.first);
             int keyPos = indexOf(optFuncExpr.getFieldName(0), chosenIndex.getKeyFieldNames());
             if (keyPos < 0) {
                 if (optFuncExpr.getNumLogicalVars() > 1) {
@@ -391,7 +391,7 @@ public class BTreeAccessMethod implements IAccessMethod {
             }
             if (!couldntFigureOut) {
                 // Remember to remove this funcExpr later.
-                replacedFuncExprs.add(matchedFuncExprs.get(exprIndex).getFuncExpr());
+                replacedFuncExprs.add(matchedFuncExprs.get(exprIndex.first).getFuncExpr());
             }
             if (doneWithExprs) {
                 break;
@@ -458,8 +458,8 @@ public class BTreeAccessMethod implements IAccessMethod {
             // Assign operator that sets the constant secondary-index search-key fields if necessary.
             AssignOperator assignConstantSearchKeys = new AssignOperator(assignKeyVarList, assignKeyExprList);
             // Input to this assign is the EmptyTupleSource (which the dataSourceScan also must have had as input).
-            assignConstantSearchKeys.getInputs().add(dataSourceScan.getInputs().get(0));
-            assignConstantSearchKeys.setExecutionMode(dataSourceScan.getExecutionMode());
+            assignConstantSearchKeys.getInputs().add(dataSourceOp.getInputs().get(0));
+            assignConstantSearchKeys.setExecutionMode(dataSourceOp.getExecutionMode());
             inputOp = assignConstantSearchKeys;
         } else {
             // All index search keys are variables.
@@ -475,12 +475,12 @@ public class BTreeAccessMethod implements IAccessMethod {
         if (dataset.getDatasetType() == DatasetType.EXTERNAL) {
             // External dataset
             ExternalDataLookupOperator externalDataAccessOp = AccessMethodUtils.createExternalDataLookupUnnestMap(
-                    dataSourceScan, dataset, recordType, secondaryIndexUnnestOp, context, chosenIndex, retainInput,
+                    dataSourceOp, dataset, recordType, secondaryIndexUnnestOp, context, chosenIndex, retainInput,
                     retainNull);
             indexSubTree.dataSourceRef.setValue(externalDataAccessOp);
             return externalDataAccessOp;
         } else if (!isPrimaryIndex) {
-            primaryIndexUnnestOp = AccessMethodUtils.createPrimaryIndexUnnestMap(dataSourceScan, dataset, recordType,
+            primaryIndexUnnestOp = AccessMethodUtils.createPrimaryIndexUnnestMap(dataSourceOp, dataset, recordType,
                     secondaryIndexUnnestOp, context, true, retainInput, retainNull, false);
 
             // Replace the datasource scan with the new plan rooted at
@@ -493,7 +493,7 @@ public class BTreeAccessMethod implements IAccessMethod {
             } catch (IOException e) {
                 throw new AlgebricksException(e);
             }
-            primaryIndexUnnestOp = new UnnestMapOperator(dataSourceScan.getVariables(),
+            primaryIndexUnnestOp = new UnnestMapOperator(dataSourceOp.getVariables(),
                     secondaryIndexUnnestOp.getExpressionRef(), primaryIndexOutputTypes, retainInput);
             primaryIndexUnnestOp.getInputs().add(new MutableObject<ILogicalOperator>(inputOp));
 

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/04b2b77a/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/am/IOptimizableFuncExpr.java
----------------------------------------------------------------------
diff --git a/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/am/IOptimizableFuncExpr.java b/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/am/IOptimizableFuncExpr.java
index dbe5854..ac9219d 100644
--- a/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/am/IOptimizableFuncExpr.java
+++ b/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/am/IOptimizableFuncExpr.java
@@ -14,7 +14,10 @@
  */
 package edu.uci.ics.asterix.optimizer.rules.am;
 
-import edu.uci.ics.asterix.om.types.ATypeTag;
+import java.util.List;
+
+import edu.uci.ics.asterix.om.types.IAType;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
 import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.AbstractFunctionCallExpression;
 import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IAlgebricksConstantValue;
@@ -33,9 +36,17 @@ public interface IOptimizableFuncExpr {
 
     public LogicalVariable getLogicalVar(int index);
 
-    public void setFieldName(int index, String fieldName);
+    public void setLogicalExpr(int index, ILogicalExpression logExpr);
+
+    public ILogicalExpression getLogicalExpr(int index);
+
+    public void setFieldName(int index, List<String> fieldName);
+
+    public List<String> getFieldName(int index);
+
+    public void setFieldType(int index, IAType fieldName);
 
-    public String getFieldName(int index);
+    public IAType getFieldType(int index);
 
     public void setOptimizableSubTree(int index, OptimizableOperatorSubTree subTree);
 
@@ -45,7 +56,7 @@ public interface IOptimizableFuncExpr {
 
     public int findLogicalVar(LogicalVariable var);
 
-    public int findFieldName(String fieldName);
+    public int findFieldName(List<String> fieldName);
 
     public void substituteVar(LogicalVariable original, LogicalVariable substitution);
 
@@ -53,7 +64,7 @@ public interface IOptimizableFuncExpr {
 
     public boolean containsPartialField();
 
-    public void setTypeTag(int index, ATypeTag typeTag);
+    public void setSourceVar(int index, LogicalVariable var);
 
-    public ATypeTag getTypeTag(int index);
+    public LogicalVariable getSourceVar(int index);
 }

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/04b2b77a/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/am/IntroduceJoinAccessMethodRule.java
----------------------------------------------------------------------
diff --git a/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/am/IntroduceJoinAccessMethodRule.java b/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/am/IntroduceJoinAccessMethodRule.java
index f113d97..f58e367 100644
--- a/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/am/IntroduceJoinAccessMethodRule.java
+++ b/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/am/IntroduceJoinAccessMethodRule.java
@@ -137,9 +137,10 @@ public class IntroduceJoinAccessMethodRule extends AbstractIntroduceAccessMethod
             while (amIt.hasNext()) {
                 Map.Entry<IAccessMethod, AccessMethodAnalysisContext> entry = amIt.next();
                 AccessMethodAnalysisContext amCtx = entry.getValue();
-                Iterator<Map.Entry<Index, List<Integer>>> indexIt = amCtx.indexExprs.entrySet().iterator();
+                Iterator<Map.Entry<Index, List<Pair<Integer, Integer>>>> indexIt = amCtx.indexExprsAndVars.entrySet()
+                        .iterator();
                 while (indexIt.hasNext()) {
-                    Map.Entry<Index, List<Integer>> indexEntry = indexIt.next();
+                    Map.Entry<Index, List<Pair<Integer, Integer>>> indexEntry = indexIt.next();
 
                     Index chosenIndex = indexEntry.getKey();
                     if (!chosenIndex.getDatasetName().equals(rightSubTree.dataset.getDatasetName())) {

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/04b2b77a/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/am/IntroduceLSMComponentFilterRule.java
----------------------------------------------------------------------
diff --git a/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/am/IntroduceLSMComponentFilterRule.java b/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/am/IntroduceLSMComponentFilterRule.java
index f90a72e..e441a20 100644
--- a/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/am/IntroduceLSMComponentFilterRule.java
+++ b/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/am/IntroduceLSMComponentFilterRule.java
@@ -15,6 +15,7 @@
 package edu.uci.ics.asterix.optimizer.rules.am;
 
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.List;
 
 import org.apache.commons.lang3.mutable.Mutable;
@@ -35,6 +36,7 @@ import edu.uci.ics.asterix.om.types.ARecordType;
 import edu.uci.ics.asterix.om.types.ATypeTag;
 import edu.uci.ics.asterix.om.types.IAType;
 import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.common.utils.Pair;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
@@ -79,7 +81,7 @@ public class IntroduceLSMComponentFilterRule implements IAlgebraicRewriteRule {
         }
 
         Dataset dataset = getDataset(op, context);
-        String filterFieldName = null;
+        List<String> filterFieldName = null;
         ARecordType recType = null;
         if (dataset != null && dataset.getDatasetType() == DatasetType.INTERNAL) {
             filterFieldName = DatasetUtils.getFilterField(dataset);
@@ -100,7 +102,7 @@ public class IntroduceLSMComponentFilterRule implements IAlgebraicRewriteRule {
         for (int i = 0; i < analysisCtx.matchedFuncExprs.size(); i++) {
             IOptimizableFuncExpr optFuncExpr = analysisCtx.matchedFuncExprs.get(i);
             boolean found = findMacthedExprFieldName(optFuncExpr, op, dataset, recType, datasetIndexes);
-            if (found && optFuncExpr.getFieldName(0).compareTo(filterFieldName) == 0) {
+            if (found && optFuncExpr.getFieldName(0).equals(filterFieldName)) {
                 optFuncExprs.add(optFuncExpr);
             }
         }
@@ -293,7 +295,7 @@ public class IntroduceLSMComponentFilterRule implements IAlgebraicRewriteRule {
                     if (funcVarIndex == -1) {
                         continue;
                     }
-                    String fieldName = getFieldNameFromSubAssignTree(optFuncExpr, descendantOp, varIndex, recType);
+                    List<String> fieldName = getFieldNameFromSubAssignTree(optFuncExpr, descendantOp, varIndex, recType).second;
                     if (fieldName == null) {
                         return false;
                     }
@@ -310,7 +312,7 @@ public class IntroduceLSMComponentFilterRule implements IAlgebraicRewriteRule {
                         continue;
                     }
                     // The variable value is one of the partitioning fields.
-                    String fieldName = DatasetUtils.getPartitioningKeys(dataset).get(varIndex);
+                    List<String> fieldName = DatasetUtils.getPartitioningKeys(dataset).get(varIndex);
                     if (fieldName == null) {
                         return false;
                     }
@@ -348,7 +350,7 @@ public class IntroduceLSMComponentFilterRule implements IAlgebraicRewriteRule {
                     }
 
                     int numSecondaryKeys = AccessMethodUtils.getNumSecondaryKeys(index, recType);
-                    String fieldName;
+                    List<String> fieldName;
                     if (varIndex >= numSecondaryKeys) {
                         fieldName = DatasetUtils.getPartitioningKeys(dataset).get(varIndex - numSecondaryKeys);
                     } else {
@@ -370,33 +372,73 @@ public class IntroduceLSMComponentFilterRule implements IAlgebraicRewriteRule {
         return false;
     }
 
-    private String getFieldNameFromSubAssignTree(IOptimizableFuncExpr optFuncExpr, AbstractLogicalOperator op,
-            int varIndex, ARecordType recType) {
+    private Pair<ARecordType, List<String>> getFieldNameFromSubAssignTree(IOptimizableFuncExpr optFuncExpr,
+            AbstractLogicalOperator op, int varIndex, ARecordType recType) {
         AbstractLogicalExpression expr = null;
         if (op.getOperatorTag() == LogicalOperatorTag.ASSIGN) {
             AssignOperator assignOp = (AssignOperator) op;
             expr = (AbstractLogicalExpression) assignOp.getExpressions().get(varIndex).getValue();
         }
-        if (expr.getExpressionTag() != LogicalExpressionTag.FUNCTION_CALL) {
+        if (expr == null || expr.getExpressionTag() != LogicalExpressionTag.FUNCTION_CALL) {
             return null;
         }
         AbstractFunctionCallExpression funcExpr = (AbstractFunctionCallExpression) expr;
         FunctionIdentifier funcIdent = funcExpr.getFunctionIdentifier();
-        if (funcIdent == AsterixBuiltinFunctions.FIELD_ACCESS_BY_NAME) {
-            ILogicalExpression nameArg = funcExpr.getArguments().get(1).getValue();
-            if (nameArg.getExpressionTag() != LogicalExpressionTag.CONSTANT) {
-                return null;
+        if (funcIdent == AsterixBuiltinFunctions.FIELD_ACCESS_BY_NAME
+                || funcIdent == AsterixBuiltinFunctions.FIELD_ACCESS_BY_INDEX) {
+
+            //get the variable from here. Figure out which input it came from. Go to that input!!!
+            ArrayList<LogicalVariable> usedVars = new ArrayList<LogicalVariable>();
+            expr.getUsedVariables(usedVars);
+            LogicalVariable usedVar = usedVars.get(0);
+            List<String> returnList = new ArrayList<String>();
+
+            //Find the input that it came from
+            for (int varCheck = 0; varCheck < op.getInputs().size(); varCheck++) {
+                AbstractLogicalOperator nestedOp = (AbstractLogicalOperator) op.getInputs().get(varCheck).getValue();
+                if (nestedOp.getOperatorTag() != LogicalOperatorTag.ASSIGN) {
+                    if (varCheck == op.getInputs().size() - 1) {
+
+                    }
+                } else {
+                    int nestedAssignVar = ((AssignOperator) nestedOp).getVariables().indexOf(usedVar);
+                    if (nestedAssignVar == -1) {
+                        continue;
+                    }
+                    //get the nested info from the lower input
+                    Pair<ARecordType, List<String>> lowerInfo = getFieldNameFromSubAssignTree(optFuncExpr,
+                            (AbstractLogicalOperator) op.getInputs().get(varCheck).getValue(), nestedAssignVar, recType);
+                    if (lowerInfo != null) {
+                        recType = lowerInfo.first;
+                        returnList = lowerInfo.second;
+                    }
+                }
             }
-            ConstantExpression constExpr = (ConstantExpression) nameArg;
-            return ((AString) ((AsterixConstantValue) constExpr.getValue()).getObject()).getStringValue();
-        } else if (funcIdent == AsterixBuiltinFunctions.FIELD_ACCESS_BY_INDEX) {
-            ILogicalExpression idxArg = funcExpr.getArguments().get(1).getValue();
-            if (idxArg.getExpressionTag() != LogicalExpressionTag.CONSTANT) {
-                return null;
+
+            if (funcIdent == AsterixBuiltinFunctions.FIELD_ACCESS_BY_NAME) {
+                ILogicalExpression nameArg = funcExpr.getArguments().get(1).getValue();
+                if (nameArg.getExpressionTag() != LogicalExpressionTag.CONSTANT) {
+                    return null;
+                }
+                ConstantExpression constExpr = (ConstantExpression) nameArg;
+                returnList.addAll(Arrays.asList(((AString) ((AsterixConstantValue) constExpr.getValue()).getObject())
+                        .getStringValue()));
+                return new Pair<ARecordType, List<String>>(recType, returnList);
+            } else if (funcIdent == AsterixBuiltinFunctions.FIELD_ACCESS_BY_INDEX) {
+                ILogicalExpression idxArg = funcExpr.getArguments().get(1).getValue();
+                if (idxArg.getExpressionTag() != LogicalExpressionTag.CONSTANT) {
+                    return null;
+                }
+                ConstantExpression constExpr = (ConstantExpression) idxArg;
+                int fieldIndex = ((AInt32) ((AsterixConstantValue) constExpr.getValue()).getObject()).getIntegerValue();
+                returnList.addAll(Arrays.asList(recType.getFieldNames()[fieldIndex]));
+                IAType subType = recType.getFieldTypes()[fieldIndex];
+                if (subType.getTypeTag() == ATypeTag.RECORD) {
+                    recType = (ARecordType) subType;
+                }
+                return new Pair<ARecordType, List<String>>(recType, returnList);
             }
-            ConstantExpression constExpr = (ConstantExpression) idxArg;
-            int fieldIndex = ((AInt32) ((AsterixConstantValue) constExpr.getValue()).getObject()).getIntegerValue();
-            return recType.getFieldNames()[fieldIndex];
+
         }
 
         ILogicalExpression argExpr = funcExpr.getArguments().get(0).getValue();

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/04b2b77a/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/am/IntroduceSelectAccessMethodRule.java
----------------------------------------------------------------------
diff --git a/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/am/IntroduceSelectAccessMethodRule.java b/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/am/IntroduceSelectAccessMethodRule.java
index 0f0db4c..cf11d8e 100644
--- a/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/am/IntroduceSelectAccessMethodRule.java
+++ b/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/am/IntroduceSelectAccessMethodRule.java
@@ -151,4 +151,4 @@ public class IntroduceSelectAccessMethodRule extends AbstractIntroduceAccessMeth
         select = null;
         selectCond = null;
     }
-}
\ No newline at end of file
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/04b2b77a/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/am/InvertedIndexAccessMethod.java
----------------------------------------------------------------------
diff --git a/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/am/InvertedIndexAccessMethod.java b/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/am/InvertedIndexAccessMethod.java
index 090b9e2..614cfbc 100644
--- a/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/am/InvertedIndexAccessMethod.java
+++ b/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/am/InvertedIndexAccessMethod.java
@@ -234,18 +234,25 @@ public class InvertedIndexAccessMethod implements IAccessMethod {
                 || arg2.getExpressionTag() == LogicalExpressionTag.CONSTANT) {
             return false;
         }
-        LogicalVariable fieldVar1 = getNonConstArgFieldVar(arg1, funcExpr, assignsAndUnnests,
+        LogicalVariable fieldVarExpr1 = getNonConstArgFieldExprPair(arg1, funcExpr, assignsAndUnnests,
                 matchedAssignOrUnnestIndex);
-        if (fieldVar1 == null) {
+        if (fieldVarExpr1 == null) {
             return false;
         }
-        LogicalVariable fieldVar2 = getNonConstArgFieldVar(arg2, funcExpr, assignsAndUnnests,
+        LogicalVariable fieldVarExpr2 = getNonConstArgFieldExprPair(arg2, funcExpr, assignsAndUnnests,
                 matchedAssignOrUnnestIndex);
-        if (fieldVar2 == null) {
+        if (fieldVarExpr2 == null) {
             return false;
         }
-        analysisCtx.matchedFuncExprs.add(new OptimizableFuncExpr(funcExpr,
-                new LogicalVariable[] { fieldVar1, fieldVar2 }, new IAlgebricksConstantValue[] { constThreshVal }));
+        OptimizableFuncExpr newOptFuncExpr = new OptimizableFuncExpr(funcExpr, new LogicalVariable[] { fieldVarExpr1,
+                fieldVarExpr2 }, new IAlgebricksConstantValue[] { constThreshVal });
+        for (IOptimizableFuncExpr optFuncExpr : analysisCtx.matchedFuncExprs) {
+            //avoid additional optFuncExpressions in case of a join
+            if (optFuncExpr.getFuncExpr().equals(funcExpr)) {
+                return true;
+            }
+        }
+        analysisCtx.matchedFuncExprs.add(newOptFuncExpr);
         return true;
     }
 
@@ -282,17 +289,23 @@ public class InvertedIndexAccessMethod implements IAccessMethod {
         }
         ConstantExpression constExpr = (ConstantExpression) constArg;
         IAlgebricksConstantValue constFilterVal = constExpr.getValue();
-        LogicalVariable fieldVar = getNonConstArgFieldVar(nonConstArg, funcExpr, assignsAndUnnests,
+        LogicalVariable fieldVarExpr = getNonConstArgFieldExprPair(nonConstArg, funcExpr, assignsAndUnnests,
                 matchedAssignOrUnnestIndex);
-        if (fieldVar == null) {
+        if (fieldVarExpr == null) {
             return false;
         }
-        analysisCtx.matchedFuncExprs.add(new OptimizableFuncExpr(funcExpr, new LogicalVariable[] { fieldVar },
-                new IAlgebricksConstantValue[] { constFilterVal, constThreshVal }));
+        OptimizableFuncExpr newOptFuncExpr = new OptimizableFuncExpr(funcExpr, new LogicalVariable[] { fieldVarExpr },
+                new IAlgebricksConstantValue[] { constFilterVal, constThreshVal });
+        for (IOptimizableFuncExpr optFuncExpr : analysisCtx.matchedFuncExprs) {
+            //avoid additional optFuncExpressions in case of a join
+            if (optFuncExpr.getFuncExpr().equals(funcExpr))
+                return true;
+        }
+        analysisCtx.matchedFuncExprs.add(newOptFuncExpr);
         return true;
     }
 
-    private LogicalVariable getNonConstArgFieldVar(ILogicalExpression nonConstArg,
+    private LogicalVariable getNonConstArgFieldExprPair(ILogicalExpression nonConstArg,
             AbstractFunctionCallExpression funcExpr, List<AbstractLogicalOperator> assignsAndUnnests,
             int matchedAssignOrUnnestIndex) {
         LogicalVariable fieldVar = null;
@@ -658,7 +671,11 @@ public class InvertedIndexAccessMethod implements IAccessMethod {
 
         // Create select ops for removing tuples that are filterable and not filterable, respectively.
         IVariableTypeEnvironment probeTypeEnv = context.getOutputTypeEnvironment(probeSubTree.root);
-        IAType inputSearchVarType = (IAType) probeTypeEnv.getVarType(inputSearchVar);
+        IAType inputSearchVarType;
+        if (chosenIndex.isEnforcingKeyFileds())
+            inputSearchVarType = optFuncExpr.getFieldType(optFuncExpr.findLogicalVar(inputSearchVar));
+        else
+            inputSearchVarType = (IAType) probeTypeEnv.getVarType(inputSearchVar);
         Mutable<ILogicalOperator> isFilterableSelectOpRef = new MutableObject<ILogicalOperator>();
         Mutable<ILogicalOperator> isNotFilterableSelectOpRef = new MutableObject<ILogicalOperator>();
         createIsFilterableSelectOps(replicateOp, inputSearchVar, inputSearchVarType, optFuncExpr, chosenIndex, context,
@@ -671,6 +688,7 @@ public class InvertedIndexAccessMethod implements IAccessMethod {
         Counter counter = new Counter(context.getVarCounter());
         LogicalOperatorDeepCopyVisitor deepCopyVisitor = new LogicalOperatorDeepCopyVisitor(counter);
         ILogicalOperator scanSubTree = deepCopyVisitor.deepCopy(indexSubTree.root, null);
+
         context.setVarCounter(counter.get());
         Map<LogicalVariable, LogicalVariable> copyVarMap = deepCopyVisitor.getVariableMapping();
         panicVarMap.putAll(copyVarMap);
@@ -735,8 +753,10 @@ public class InvertedIndexAccessMethod implements IAccessMethod {
                 break;
             }
             default: {
+                throw new AlgebricksException("Only strings, ordered and unordered list types supported.");
             }
         }
+
         SelectOperator isFilterableSelectOp = new SelectOperator(
                 new MutableObject<ILogicalExpression>(isFilterableExpr), false, null);
         isFilterableSelectOp.getInputs().add(new MutableObject<ILogicalOperator>(inputOp));
@@ -767,12 +787,10 @@ public class InvertedIndexAccessMethod implements IAccessMethod {
             // Find the type of the variable that is going to feed into the index search.
             if (optFuncExpr.getOperatorSubTree(0) == indexSubTree) {
                 // If the index is on a dataset in subtree 0, then subtree 1 will feed.
-                type = (IAType) context.getOutputTypeEnvironment(optFuncExpr.getOperatorSubTree(1).root).getVarType(
-                        optFuncExpr.getLogicalVar(1));
+                type = optFuncExpr.getFieldType(1);
             } else {
                 // If the index is on a dataset in subtree 1, then subtree 0 will feed.
-                type = (IAType) context.getOutputTypeEnvironment(optFuncExpr.getOperatorSubTree(0).root).getVarType(
-                        optFuncExpr.getLogicalVar(0));
+                type = optFuncExpr.getFieldType(0);
             }
             typeTag = type.getTypeTag();
         } else {
@@ -854,7 +872,10 @@ public class InvertedIndexAccessMethod implements IAccessMethod {
     }
 
     private boolean isEditDistanceFuncJoinOptimizable(Index index, IOptimizableFuncExpr optFuncExpr) {
-        return isEditDistanceFuncCompatible(optFuncExpr.getTypeTag(0), index.getIndexType());
+        if (index.isEnforcingKeyFileds())
+            return isEditDistanceFuncCompatible(index.getKeyFieldTypes().get(0).getTypeTag(), index.getIndexType());
+        else
+            return isEditDistanceFuncCompatible(optFuncExpr.getFieldType(0).getTypeTag(), index.getIndexType());
     }
 
     private boolean isEditDistanceFuncCompatible(ATypeTag typeTag, IndexType indexType) {
@@ -931,7 +952,7 @@ public class InvertedIndexAccessMethod implements IAccessMethod {
         for (int i = 0; i < variableCount; i++) {
             funcExpr = findTokensFunc(AsterixBuiltinFunctions.GRAM_TOKENS, optFuncExpr, i);
             if (funcExpr != null) {
-                return isJaccardFuncCompatible(funcExpr, optFuncExpr.getTypeTag(i), index.getIndexType());
+                return isJaccardFuncCompatible(funcExpr, optFuncExpr.getFieldType(i).getTypeTag(), index.getIndexType());
             }
         }
 
@@ -939,7 +960,7 @@ public class InvertedIndexAccessMethod implements IAccessMethod {
         for (int i = 0; i < variableCount; i++) {
             funcExpr = findTokensFunc(AsterixBuiltinFunctions.WORD_TOKENS, optFuncExpr, i);
             if (funcExpr != null) {
-                return isJaccardFuncCompatible(funcExpr, optFuncExpr.getTypeTag(i), index.getIndexType());
+                return isJaccardFuncCompatible(funcExpr, optFuncExpr.getFieldType(i).getTypeTag(), index.getIndexType());
             }
         }
 
@@ -953,8 +974,8 @@ public class InvertedIndexAccessMethod implements IAccessMethod {
             targetVar = optFuncExpr.getLogicalVar(i);
             if (targetVar == null)
                 continue;
-            return isJaccardFuncCompatible(optFuncExpr.getFuncExpr().getArguments().get(i).getValue(),
-                    optFuncExpr.getTypeTag(i), index.getIndexType());
+            return isJaccardFuncCompatible(optFuncExpr.getFuncExpr().getArguments().get(i).getValue(), optFuncExpr
+                    .getFieldType(i).getTypeTag(), index.getIndexType());
         }
 
         return false;
@@ -1049,7 +1070,10 @@ public class InvertedIndexAccessMethod implements IAccessMethod {
     }
 
     private boolean isContainsFuncJoinOptimizable(Index index, IOptimizableFuncExpr optFuncExpr) {
-        return isContainsFuncCompatible(optFuncExpr.getTypeTag(0), index.getIndexType());
+        if (index.isEnforcingKeyFileds())
+            return isContainsFuncCompatible(index.getKeyFieldTypes().get(0).getTypeTag(), index.getIndexType());
+        else
+            return isContainsFuncCompatible(optFuncExpr.getFieldType(0).getTypeTag(), index.getIndexType());
     }
 
     private boolean isContainsFuncCompatible(ATypeTag typeTag, IndexType indexType) {

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/04b2b77a/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/am/OptimizableFuncExpr.java
----------------------------------------------------------------------
diff --git a/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/am/OptimizableFuncExpr.java b/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/am/OptimizableFuncExpr.java
index 0d8351c..59813dd 100644
--- a/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/am/OptimizableFuncExpr.java
+++ b/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/am/OptimizableFuncExpr.java
@@ -14,8 +14,12 @@
  */
 package edu.uci.ics.asterix.optimizer.rules.am;
 
+import java.util.ArrayList;
+import java.util.List;
+
 import edu.uci.ics.asterix.om.functions.AsterixBuiltinFunctions;
-import edu.uci.ics.asterix.om.types.ATypeTag;
+import edu.uci.ics.asterix.om.types.IAType;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
 import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.AbstractFunctionCallExpression;
 import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IAlgebricksConstantValue;
@@ -27,8 +31,10 @@ import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IAlgebricksConsta
 public class OptimizableFuncExpr implements IOptimizableFuncExpr {
     protected final AbstractFunctionCallExpression funcExpr;
     protected final LogicalVariable[] logicalVars;
-    protected final String[] fieldNames;
-    protected final ATypeTag[] typeTags;
+    protected final LogicalVariable[] sourceVars;
+    protected final ILogicalExpression[] logicalExprs;
+    protected final List<List<String>> fieldNames;
+    protected final IAType[] fieldTypes;
     protected final OptimizableOperatorSubTree[] subTrees;
     protected final IAlgebricksConstantValue[] constantVals;
     protected boolean partialField;
@@ -37,9 +43,14 @@ public class OptimizableFuncExpr implements IOptimizableFuncExpr {
             IAlgebricksConstantValue[] constantVals) {
         this.funcExpr = funcExpr;
         this.logicalVars = logicalVars;
+        this.sourceVars = new LogicalVariable[logicalVars.length];
+        this.logicalExprs = new ILogicalExpression[logicalVars.length];
         this.constantVals = constantVals;
-        this.fieldNames = new String[logicalVars.length];
-        this.typeTags = new ATypeTag[logicalVars.length];
+        this.fieldNames = new ArrayList<List<String>>();
+        for (int i = 0; i < logicalVars.length; i++) {
+            fieldNames.add(new ArrayList<String>());
+        }
+        this.fieldTypes = new IAType[logicalVars.length];
         this.subTrees = new OptimizableOperatorSubTree[logicalVars.length];
 
         if (funcExpr.getFunctionIdentifier() == AsterixBuiltinFunctions.EDIT_DISTANCE_CONTAINS) {
@@ -54,9 +65,14 @@ public class OptimizableFuncExpr implements IOptimizableFuncExpr {
             IAlgebricksConstantValue constantVal) {
         this.funcExpr = funcExpr;
         this.logicalVars = new LogicalVariable[] { logicalVar };
+        this.sourceVars = new LogicalVariable[1];
+        this.logicalExprs = new ILogicalExpression[1];
         this.constantVals = new IAlgebricksConstantValue[] { constantVal };
-        this.fieldNames = new String[logicalVars.length];
-        this.typeTags = new ATypeTag[logicalVars.length];
+        this.fieldNames = new ArrayList<List<String>>();
+        for (int i = 0; i < logicalVars.length; i++) {
+            fieldNames.add(new ArrayList<String>());
+        }
+        this.fieldTypes = new IAType[logicalVars.length];
         this.subTrees = new OptimizableOperatorSubTree[logicalVars.length];
         if (funcExpr.getFunctionIdentifier() == AsterixBuiltinFunctions.EDIT_DISTANCE_CONTAINS) {
             this.partialField = true;
@@ -86,17 +102,33 @@ public class OptimizableFuncExpr implements IOptimizableFuncExpr {
     }
 
     @Override
-    public void setFieldName(int index, String fieldName) {
-        fieldNames[index] = fieldName;
+    public void setLogicalExpr(int index, ILogicalExpression logExpr) {
+        logicalExprs[index] = logExpr;
+    }
+
+    @Override
+    public ILogicalExpression getLogicalExpr(int index) {
+        return logicalExprs[index];
+    }
+
+    @Override
+    public void setFieldName(int index, List<String> fieldName) {
+        fieldNames.set(index, fieldName);
     }
 
     @Override
-    public String getFieldName(int index) {
-        return fieldNames[index];
+    public List<String> getFieldName(int index) {
+        return fieldNames.get(index);
     }
 
-    public String[] getFieldNames() {
-        return fieldNames;
+    @Override
+    public void setFieldType(int index, IAType fieldType) {
+        fieldTypes[index] = fieldType;
+    }
+
+    @Override
+    public IAType getFieldType(int index) {
+        return fieldTypes[index];
     }
 
     @Override
@@ -115,9 +147,9 @@ public class OptimizableFuncExpr implements IOptimizableFuncExpr {
     }
 
     @Override
-    public int findFieldName(String fieldName) {
-        for (int i = 0; i < fieldNames.length; i++) {
-            if (fieldName.equals(fieldNames[i])) {
+    public int findFieldName(List<String> fieldName) {
+        for (int i = 0; i < fieldNames.size(); i++) {
+            if (fieldName.equals(fieldNames.get(i))) {
                 return i;
             }
         }
@@ -157,12 +189,27 @@ public class OptimizableFuncExpr implements IOptimizableFuncExpr {
     }
 
     @Override
-    public void setTypeTag(int index, ATypeTag typeTag) {
-        typeTags[index] = typeTag;
+    public int hashCode() {
+        return funcExpr.hashCode();
+
     }
 
     @Override
-    public ATypeTag getTypeTag(int index) {
-        return typeTags[index];
+    public boolean equals(Object o) {
+        if (!(o instanceof OptimizableFuncExpr))
+            return false;
+        else
+            return funcExpr.equals(o);
+
     }
+
+    public void setSourceVar(int index, LogicalVariable var) {
+        sourceVars[index] = var;
+    }
+
+    @Override
+    public LogicalVariable getSourceVar(int index) {
+        return sourceVars[index];
+    }
+
 }

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/04b2b77a/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/am/OptimizableOperatorSubTree.java
----------------------------------------------------------------------
diff --git a/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/am/OptimizableOperatorSubTree.java b/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/am/OptimizableOperatorSubTree.java
index 392f9b1..c872c2a 100644
--- a/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/am/OptimizableOperatorSubTree.java
+++ b/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/am/OptimizableOperatorSubTree.java
@@ -36,7 +36,10 @@ import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
 import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
 import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.AbstractFunctionCallExpression;
 import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractScanOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractUnnestOperator;
 import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.DataSourceScanOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ExternalDataLookupOperator;
 import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.UnnestMapOperator;
 
 /**
@@ -47,7 +50,9 @@ public class OptimizableOperatorSubTree {
 
     public static enum DataSourceType {
         DATASOURCE_SCAN,
+        EXTERNAL_SCAN,
         PRIMARY_INDEX_LOOKUP,
+        COLLECTION_SCAN,
         NO_DATASOURCE
     }
 
@@ -67,24 +72,28 @@ public class OptimizableOperatorSubTree {
         root = subTreeOpRef.getValue();
         // Examine the op's children to match the expected patterns.
         AbstractLogicalOperator subTreeOp = (AbstractLogicalOperator) subTreeOpRef.getValue();
-        // Skip select operator.
-        if (subTreeOp.getOperatorTag() == LogicalOperatorTag.SELECT) {
-            subTreeOpRef = subTreeOp.getInputs().get(0);
-            subTreeOp = (AbstractLogicalOperator) subTreeOpRef.getValue();
-        }
-        // Check primary-index pattern.
-        if (subTreeOp.getOperatorTag() != LogicalOperatorTag.ASSIGN && subTreeOp.getOperatorTag() != LogicalOperatorTag.UNNEST) {
-            // Pattern may still match if we are looking for primary index matches as well.
-            return initializeDataSource(subTreeOpRef);
-        }
-        // Match (assign | unnest)+.
         do {
-            assignsAndUnnestsRefs.add(subTreeOpRef);
-            assignsAndUnnests.add(subTreeOp);
+            // Skip select operator.
+            if (subTreeOp.getOperatorTag() == LogicalOperatorTag.SELECT) {
+                subTreeOpRef = subTreeOp.getInputs().get(0);
+                subTreeOp = (AbstractLogicalOperator) subTreeOpRef.getValue();
+            }
+            // Check primary-index pattern.
+            if (subTreeOp.getOperatorTag() != LogicalOperatorTag.ASSIGN
+                    && subTreeOp.getOperatorTag() != LogicalOperatorTag.UNNEST) {
+                // Pattern may still match if we are looking for primary index matches as well.
+                return initializeDataSource(subTreeOpRef);
+            }
+            // Match (assign | unnest)+.
+            while (subTreeOp.getOperatorTag() == LogicalOperatorTag.ASSIGN
+                    || subTreeOp.getOperatorTag() == LogicalOperatorTag.UNNEST) {
+                assignsAndUnnestsRefs.add(subTreeOpRef);
+                assignsAndUnnests.add(subTreeOp);
 
-            subTreeOpRef = subTreeOp.getInputs().get(0);
-            subTreeOp = (AbstractLogicalOperator) subTreeOpRef.getValue();
-        } while (subTreeOp.getOperatorTag() == LogicalOperatorTag.ASSIGN || subTreeOp.getOperatorTag() == LogicalOperatorTag.UNNEST);
+                subTreeOpRef = subTreeOp.getInputs().get(0);
+                subTreeOp = (AbstractLogicalOperator) subTreeOpRef.getValue();
+            };
+        } while (subTreeOp.getOperatorTag() == LogicalOperatorTag.SELECT);
 
         // Match data source (datasource scan or primary index search).
         return initializeDataSource(subTreeOpRef);
@@ -96,6 +105,14 @@ public class OptimizableOperatorSubTree {
             dataSourceType = DataSourceType.DATASOURCE_SCAN;
             dataSourceRef = subTreeOpRef;
             return true;
+        } else if (subTreeOp.getOperatorTag() == LogicalOperatorTag.EXTERNAL_LOOKUP) {
+            dataSourceType = DataSourceType.EXTERNAL_SCAN;
+            dataSourceRef = subTreeOpRef;
+            return true;
+        } else if (subTreeOp.getOperatorTag() == LogicalOperatorTag.EMPTYTUPLESOURCE) {
+            dataSourceType = DataSourceType.COLLECTION_SCAN;
+            dataSourceRef = subTreeOpRef;
+            return true;
         } else if (subTreeOp.getOperatorTag() == LogicalOperatorTag.UNNEST_MAP) {
             UnnestMapOperator unnestMapOp = (UnnestMapOperator) subTreeOp;
             ILogicalExpression unnestExpr = unnestMapOp.getExpressionRef().getValue();
@@ -104,7 +121,7 @@ public class OptimizableOperatorSubTree {
                 if (f.getFunctionIdentifier().equals(AsterixBuiltinFunctions.INDEX_SEARCH)) {
                     AccessMethodJobGenParams jobGenParams = new AccessMethodJobGenParams();
                     jobGenParams.readFromFuncArgs(f.getArguments());
-                    if(jobGenParams.isPrimaryIndex()) {
+                    if (jobGenParams.isPrimaryIndex()) {
                         dataSourceType = DataSourceType.PRIMARY_INDEX_LOOKUP;
                         dataSourceRef = subTreeOpRef;
                         return true;
@@ -112,6 +129,7 @@ public class OptimizableOperatorSubTree {
                 }
             }
         }
+
         return false;
     }
 
@@ -130,7 +148,7 @@ public class OptimizableOperatorSubTree {
                 datasetName = datasetInfo.second;
                 break;
             case PRIMARY_INDEX_LOOKUP:
-                UnnestMapOperator unnestMapOp = (UnnestMapOperator) dataSourceRef.getValue();
+                AbstractUnnestOperator unnestMapOp = (AbstractUnnestOperator) dataSourceRef.getValue();
                 ILogicalExpression unnestExpr = unnestMapOp.getExpressionRef().getValue();
                 AbstractFunctionCallExpression f = (AbstractFunctionCallExpression) unnestExpr;
                 AccessMethodJobGenParams jobGenParams = new AccessMethodJobGenParams();
@@ -138,6 +156,14 @@ public class OptimizableOperatorSubTree {
                 datasetName = jobGenParams.getDatasetName();
                 dataverseName = jobGenParams.getDataverseName();
                 break;
+            case EXTERNAL_SCAN:
+                ExternalDataLookupOperator externalScan = (ExternalDataLookupOperator) dataSourceRef.getValue();
+                datasetInfo = AnalysisUtil.getDatasetInfo(externalScan);
+                dataverseName = datasetInfo.first;
+                datasetName = datasetInfo.second;
+                break;
+            case COLLECTION_SCAN:
+                return true;
             case NO_DATASOURCE:
             default:
                 return false;
@@ -178,8 +204,7 @@ public class OptimizableOperatorSubTree {
         recordType = null;
     }
 
-    public void getPrimaryKeyVars(List<LogicalVariable> target)
-            throws AlgebricksException {
+    public void getPrimaryKeyVars(List<LogicalVariable> target) throws AlgebricksException {
         switch (dataSourceType) {
             case DATASOURCE_SCAN:
                 DataSourceScanOperator dataSourceScan = (DataSourceScanOperator) dataSourceRef.getValue();
@@ -203,11 +228,12 @@ public class OptimizableOperatorSubTree {
     public List<LogicalVariable> getDataSourceVariables() throws AlgebricksException {
         switch (dataSourceType) {
             case DATASOURCE_SCAN:
-                DataSourceScanOperator dataSourceScan = (DataSourceScanOperator) dataSourceRef.getValue();
-                return dataSourceScan.getVariables();
+            case EXTERNAL_SCAN:
             case PRIMARY_INDEX_LOOKUP:
-                UnnestMapOperator unnestMapOp = (UnnestMapOperator) dataSourceRef.getValue();
-                return unnestMapOp.getVariables();
+                AbstractScanOperator scanOp = (AbstractScanOperator) dataSourceRef.getValue();
+                return scanOp.getVariables();
+            case COLLECTION_SCAN:
+                return new ArrayList<LogicalVariable>();
             case NO_DATASOURCE:
             default:
                 throw new AlgebricksException("The subtree does not have any data source.");

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/04b2b77a/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/am/RTreeAccessMethod.java
----------------------------------------------------------------------
diff --git a/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/am/RTreeAccessMethod.java b/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/am/RTreeAccessMethod.java
index 582c273..e2dffd1 100644
--- a/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/am/RTreeAccessMethod.java
+++ b/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/am/RTreeAccessMethod.java
@@ -14,7 +14,6 @@
  */
 package edu.uci.ics.asterix.optimizer.rules.am;
 
-import java.io.IOException;
 import java.util.ArrayList;
 import java.util.List;
 
@@ -44,10 +43,10 @@ import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.ConstantExpressio
 import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.ScalarFunctionCallExpression;
 import edu.uci.ics.hyracks.algebricks.core.algebra.functions.FunctionIdentifier;
 import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractBinaryJoinOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractDataSourceOperator;
 import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
 import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator.ExecutionMode;
 import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AssignOperator;
-import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.DataSourceScanOperator;
 import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ExternalDataLookupOperator;
 import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SelectOperator;
 import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.UnnestMapOperator;
@@ -168,18 +167,9 @@ public class RTreeAccessMethod implements IAccessMethod {
         Dataset dataset = indexSubTree.dataset;
         ARecordType recordType = indexSubTree.recordType;
 
-        Pair<IAType, Boolean> keyPairType = null;
-        //Get the type of the field from the optFuncExpr
-        for (String fieldName : ((OptimizableFuncExpr) optFuncExpr).getFieldNames()) {
-            try {
-                if (fieldName != null && recordType.getFieldType(fieldName) != null) {
-                    keyPairType = Index.getNonNullableKeyFieldType(fieldName, recordType);
-                    break;
-                }
-            } catch (IOException e) {
-                throw new AlgebricksException(e);
-            }
-        }
+        int optFieldIdx = AccessMethodUtils.chooseFirstOptFuncVar(chosenIndex, analysisCtx);
+        Pair<IAType, Boolean> keyPairType = Index.getNonNullableOpenFieldType(optFuncExpr.getFieldType(optFieldIdx),
+                optFuncExpr.getFieldName(optFieldIdx), recordType);
         if (keyPairType == null) {
             return null;
         }
@@ -189,7 +179,7 @@ public class RTreeAccessMethod implements IAccessMethod {
         int numDimensions = NonTaggedFormatUtil.getNumDimensions(spatialType.getTypeTag());
         int numSecondaryKeys = numDimensions * 2;
         // we made sure indexSubTree has datasource scan
-        DataSourceScanOperator dataSourceScan = (DataSourceScanOperator) indexSubTree.dataSourceRef.getValue();
+        AbstractDataSourceOperator dataSourceOp = (AbstractDataSourceOperator) indexSubTree.dataSourceRef.getValue();
         RTreeJobGenParams jobGenParams = new RTreeJobGenParams(chosenIndex.getIndexName(), IndexType.RTREE,
                 dataset.getDataverseName(), dataset.getDatasetName(), retainInput, retainNull, requiresBroadcast);
         // A spatial object is serialized in the constant of the func expr we are optimizing.
@@ -229,8 +219,8 @@ public class RTreeAccessMethod implements IAccessMethod {
         if (probeSubTree == null) {
             // We are optimizing a selection query.
             // Input to this assign is the EmptyTupleSource (which the dataSourceScan also must have had as input).
-            assignSearchKeys.getInputs().add(dataSourceScan.getInputs().get(0));
-            assignSearchKeys.setExecutionMode(dataSourceScan.getExecutionMode());
+            assignSearchKeys.getInputs().add(dataSourceOp.getInputs().get(0));
+            assignSearchKeys.setExecutionMode(dataSourceOp.getExecutionMode());
         } else {
             // We are optimizing a join, place the assign op top of the probe subtree.
             assignSearchKeys.getInputs().add(probeSubTree.rootRef);
@@ -242,11 +232,11 @@ public class RTreeAccessMethod implements IAccessMethod {
         // Generate the rest of the upstream plan which feeds the search results into the primary index.
         if (dataset.getDatasetType() == DatasetType.EXTERNAL) {
             ExternalDataLookupOperator externalDataAccessOp = AccessMethodUtils.createExternalDataLookupUnnestMap(
-                    dataSourceScan, dataset, recordType, secondaryIndexUnnestOp, context, chosenIndex, retainInput,
+                    dataSourceOp, dataset, recordType, secondaryIndexUnnestOp, context, chosenIndex, retainInput,
                     retainNull);
             return externalDataAccessOp;
         } else {
-            UnnestMapOperator primaryIndexUnnestOp = AccessMethodUtils.createPrimaryIndexUnnestMap(dataSourceScan,
+            UnnestMapOperator primaryIndexUnnestOp = AccessMethodUtils.createPrimaryIndexUnnestMap(dataSourceOp,
                     dataset, recordType, secondaryIndexUnnestOp, context, true, retainInput, false, false);
 
             return primaryIndexUnnestOp;

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/04b2b77a/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/typecast/StaticTypeCastUtil.java
----------------------------------------------------------------------
diff --git a/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/typecast/StaticTypeCastUtil.java b/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/typecast/StaticTypeCastUtil.java
index 47b4b7a..8e27099 100644
--- a/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/typecast/StaticTypeCastUtil.java
+++ b/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/typecast/StaticTypeCastUtil.java
@@ -304,7 +304,7 @@ public class StaticTypeCastUtil {
                     if (reqFieldType.getTypeTag() == ATypeTag.UNION
                             && NonTaggedFormatUtil.isOptionalField((AUnionType) reqFieldType)) {
                         IAType itemType = ((AUnionType) reqFieldType).getUnionList().get(
-                                NonTaggedFormatUtil.OPTIONAL_TYPE_INDEX_IN_UNION_LIST);
+                                AUnionType.OPTIONAL_TYPE_INDEX_IN_UNION_LIST);
                         reqFieldType = itemType;
                         if (fieldType.equals(BuiltinType.ANULL) || fieldType.equals(itemType)) {
                             fieldPermutation[j] = i;
@@ -325,7 +325,7 @@ public class StaticTypeCastUtil {
                     if (fieldType.getTypeTag() == ATypeTag.UNION
                             && NonTaggedFormatUtil.isOptionalField((AUnionType) fieldType)) {
                         IAType itemType = ((AUnionType) fieldType).getUnionList().get(
-                                NonTaggedFormatUtil.OPTIONAL_TYPE_INDEX_IN_UNION_LIST);
+                                AUnionType.OPTIONAL_TYPE_INDEX_IN_UNION_LIST);
                         if (reqFieldType.equals(itemType)) {
                             fieldPermutation[j] = i;
                             openFields[i] = false;
@@ -382,7 +382,7 @@ public class StaticTypeCastUtil {
                 if (reqFieldType.getTypeTag() == ATypeTag.UNION
                         && NonTaggedFormatUtil.isOptionalField((AUnionType) reqFieldType)) {
                     IAType itemType = ((AUnionType) reqFieldType).getUnionList().get(
-                            NonTaggedFormatUtil.OPTIONAL_TYPE_INDEX_IN_UNION_LIST);
+                            AUnionType.OPTIONAL_TYPE_INDEX_IN_UNION_LIST);
                     if (fieldType.equals(BuiltinType.ANULL) || fieldType.equals(itemType)) {
                         matched = true;
                         break;

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/04b2b77a/asterix-algebra/src/main/java/edu/uci/ics/asterix/translator/AqlExpressionToPlanTranslator.java
----------------------------------------------------------------------
diff --git a/asterix-algebra/src/main/java/edu/uci/ics/asterix/translator/AqlExpressionToPlanTranslator.java b/asterix-algebra/src/main/java/edu/uci/ics/asterix/translator/AqlExpressionToPlanTranslator.java
index e81d083..7a6078f 100644
--- a/asterix-algebra/src/main/java/edu/uci/ics/asterix/translator/AqlExpressionToPlanTranslator.java
+++ b/asterix-algebra/src/main/java/edu/uci/ics/asterix/translator/AqlExpressionToPlanTranslator.java
@@ -95,17 +95,18 @@ import edu.uci.ics.asterix.common.functions.FunctionConstants;
 import edu.uci.ics.asterix.common.functions.FunctionSignature;
 import edu.uci.ics.asterix.metadata.MetadataException;
 import edu.uci.ics.asterix.metadata.MetadataManager;
-import edu.uci.ics.asterix.metadata.declared.LoadableDataSource;
 import edu.uci.ics.asterix.metadata.declared.AqlDataSource.AqlDataSourceType;
 import edu.uci.ics.asterix.metadata.declared.AqlMetadataProvider;
 import edu.uci.ics.asterix.metadata.declared.AqlSourceId;
 import edu.uci.ics.asterix.metadata.declared.DatasetDataSource;
+import edu.uci.ics.asterix.metadata.declared.LoadableDataSource;
 import edu.uci.ics.asterix.metadata.declared.ResultSetDataSink;
 import edu.uci.ics.asterix.metadata.declared.ResultSetSinkId;
 import edu.uci.ics.asterix.metadata.entities.Dataset;
 import edu.uci.ics.asterix.metadata.entities.Function;
 import edu.uci.ics.asterix.metadata.functions.ExternalFunctionCompilerUtil;
 import edu.uci.ics.asterix.metadata.utils.DatasetUtils;
+import edu.uci.ics.asterix.om.base.AOrderedList;
 import edu.uci.ics.asterix.om.base.AString;
 import edu.uci.ics.asterix.om.constants.AsterixConstantValue;
 import edu.uci.ics.asterix.om.functions.AsterixBuiltinFunctions;
@@ -210,7 +211,7 @@ public class AqlExpressionToPlanTranslator extends AbstractAqlTranslator impleme
         IAType itemType = metadataProvider.findType(clffs.getDataverseName(), dataset.getItemTypeName());
         DatasetDataSource targetDatasource = validateDatasetInfo(metadataProvider, stmt.getDataverseName(),
                 stmt.getDatasetName());
-        List<String> partitionKeys = DatasetUtils.getPartitioningKeys(targetDatasource.getDataset());
+        List<List<String>> partitionKeys = DatasetUtils.getPartitioningKeys(targetDatasource.getDataset());
 
         LoadableDataSource lds;
         try {
@@ -240,7 +241,7 @@ public class AqlExpressionToPlanTranslator extends AbstractAqlTranslator impleme
         ArrayList<Mutable<ILogicalExpression>> pkExprs = new ArrayList<Mutable<ILogicalExpression>>();
         List<Mutable<ILogicalExpression>> varRefsForLoading = new ArrayList<Mutable<ILogicalExpression>>();
         LogicalVariable payloadVar = payloadVars.get(0);
-        for (String keyFieldName : partitionKeys) {
+        for (List<String> keyFieldName : partitionKeys) {
             prepareVarAndExpression(keyFieldName, payloadVar, pkVars, pkExprs, varRefsForLoading);
         }
 
@@ -256,7 +257,7 @@ public class AqlExpressionToPlanTranslator extends AbstractAqlTranslator impleme
             assign.setExplicitOrderingProperty(new LocalOrderProperty(orderColumns));
         }
 
-        String additionalFilteringField = DatasetUtils.getFilterField(targetDatasource.getDataset());
+        List<String> additionalFilteringField = DatasetUtils.getFilterField(targetDatasource.getDataset());
         List<LogicalVariable> additionalFilteringVars = null;
         List<Mutable<ILogicalExpression>> additionalFilteringAssignExpressions = null;
         List<Mutable<ILogicalExpression>> additionalFilteringExpressions = null;
@@ -340,12 +341,12 @@ public class AqlExpressionToPlanTranslator extends AbstractAqlTranslator impleme
             ArrayList<LogicalVariable> vars = new ArrayList<LogicalVariable>();
             ArrayList<Mutable<ILogicalExpression>> exprs = new ArrayList<Mutable<ILogicalExpression>>();
             List<Mutable<ILogicalExpression>> varRefsForLoading = new ArrayList<Mutable<ILogicalExpression>>();
-            List<String> partitionKeys = DatasetUtils.getPartitioningKeys(targetDatasource.getDataset());
-            for (String keyFieldName : partitionKeys) {
+            List<List<String>> partitionKeys = DatasetUtils.getPartitioningKeys(targetDatasource.getDataset());
+            for (List<String> keyFieldName : partitionKeys) {
                 prepareVarAndExpression(keyFieldName, resVar, vars, exprs, varRefsForLoading);
             }
 
-            String additionalFilteringField = DatasetUtils.getFilterField(targetDatasource.getDataset());
+            List<String> additionalFilteringField = DatasetUtils.getFilterField(targetDatasource.getDataset());
             List<LogicalVariable> additionalFilteringVars = null;
             List<Mutable<ILogicalExpression>> additionalFilteringAssignExpressions = null;
             List<Mutable<ILogicalExpression>> additionalFilteringExpressions = null;
@@ -412,16 +413,23 @@ public class AqlExpressionToPlanTranslator extends AbstractAqlTranslator impleme
     }
 
     @SuppressWarnings("unchecked")
-    private void prepareVarAndExpression(String field, LogicalVariable resVar,
+    private void prepareVarAndExpression(List<String> field, LogicalVariable resVar,
             List<LogicalVariable> additionalFilteringVars,
             List<Mutable<ILogicalExpression>> additionalFilteringAssignExpressions,
             List<Mutable<ILogicalExpression>> varRefs) {
-        IFunctionInfo finfoAccess = FunctionUtils.getFunctionInfo(AsterixBuiltinFunctions.FIELD_ACCESS_BY_NAME);
-
-        ScalarFunctionCallExpression f = new ScalarFunctionCallExpression(finfoAccess,
-                new MutableObject<ILogicalExpression>(new VariableReferenceExpression(METADATA_DUMMY_VAR)),
-                new MutableObject<ILogicalExpression>(new ConstantExpression(new AsterixConstantValue(
-                        new AString(field)))));
+        IFunctionInfo finfoAccess;
+        ScalarFunctionCallExpression f;
+        if (field.size() > 1) {
+            finfoAccess = FunctionUtils.getFunctionInfo(AsterixBuiltinFunctions.FIELD_ACCESS_NESTED);
+            f = new ScalarFunctionCallExpression(finfoAccess, new MutableObject<ILogicalExpression>(
+                    new VariableReferenceExpression(METADATA_DUMMY_VAR)), new MutableObject<ILogicalExpression>(
+                    new ConstantExpression(new AsterixConstantValue(new AOrderedList(field)))));
+        } else {
+            finfoAccess = FunctionUtils.getFunctionInfo(AsterixBuiltinFunctions.FIELD_ACCESS_BY_NAME);
+            f = new ScalarFunctionCallExpression(finfoAccess, new MutableObject<ILogicalExpression>(
+                    new VariableReferenceExpression(METADATA_DUMMY_VAR)), new MutableObject<ILogicalExpression>(
+                    new ConstantExpression(new AsterixConstantValue(new AString(field.get(0))))));
+        }
         f.substituteVar(METADATA_DUMMY_VAR, resVar);
         additionalFilteringAssignExpressions.add(new MutableObject<ILogicalExpression>(f));
         LogicalVariable v = context.newVar();

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/04b2b77a/asterix-algebra/src/main/java/edu/uci/ics/asterix/translator/AqlPlusExpressionToPlanTranslator.java
----------------------------------------------------------------------
diff --git a/asterix-algebra/src/main/java/edu/uci/ics/asterix/translator/AqlPlusExpressionToPlanTranslator.java b/asterix-algebra/src/main/java/edu/uci/ics/asterix/translator/AqlPlusExpressionToPlanTranslator.java
index ad98f93..1ac27a1 100644
--- a/asterix-algebra/src/main/java/edu/uci/ics/asterix/translator/AqlPlusExpressionToPlanTranslator.java
+++ b/asterix-algebra/src/main/java/edu/uci/ics/asterix/translator/AqlPlusExpressionToPlanTranslator.java
@@ -266,11 +266,11 @@ public class AqlPlusExpressionToPlanTranslator extends AbstractAqlTranslator imp
             }
             ARecordType itemType = (ARecordType) metadata.findType(dataset.getDataverseName(),
                     dataset.getItemTypeName());
-            List<String> partitioningKeys = DatasetUtils.getPartitioningKeys(dataset);
+            List<List<String>> partitioningKeys = DatasetUtils.getPartitioningKeys(dataset);
             ArrayList<LogicalVariable> vars = new ArrayList<LogicalVariable>();
             ArrayList<Mutable<ILogicalExpression>> exprs = new ArrayList<Mutable<ILogicalExpression>>();
             List<Mutable<ILogicalExpression>> varRefsForLoading = new ArrayList<Mutable<ILogicalExpression>>();
-            for (String partitioningKey : partitioningKeys) {
+            for (List<String> partitioningKey : partitioningKeys) {
                 Triple<ICopyEvaluatorFactory, ScalarFunctionCallExpression, IAType> partitioner = format
                         .partitioningEvaluatorFactory(itemType, partitioningKey);
                 AbstractFunctionCallExpression f = partitioner.second.cloneExpression();

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/04b2b77a/asterix-algebra/src/main/java/edu/uci/ics/asterix/translator/CompiledStatements.java
----------------------------------------------------------------------
diff --git a/asterix-algebra/src/main/java/edu/uci/ics/asterix/translator/CompiledStatements.java b/asterix-algebra/src/main/java/edu/uci/ics/asterix/translator/CompiledStatements.java
index e403cce..6e9f197 100644
--- a/asterix-algebra/src/main/java/edu/uci/ics/asterix/translator/CompiledStatements.java
+++ b/asterix-algebra/src/main/java/edu/uci/ics/asterix/translator/CompiledStatements.java
@@ -197,19 +197,23 @@ public class CompiledStatements {
         private final String indexName;
         private final String dataverseName;
         private final String datasetName;
-        private final List<String> keyFields;
+        private final List<List<String>> keyFields;
+        private final List<IAType> keyTypes;
+        private final boolean isEnforced;
         private final IndexType indexType;
 
         // Specific to NGram index.
         private final int gramLength;
 
         public CompiledCreateIndexStatement(String indexName, String dataverseName, String datasetName,
-                List<String> keyFields, int gramLength, IndexType indexType) {
+                List<List<String>> keyFields, List<IAType> keyTypes, boolean isEnforced, int gramLength, IndexType indexType) {
             this.indexName = indexName;
             this.dataverseName = dataverseName;
             this.datasetName = datasetName;
             this.keyFields = keyFields;
+            this.keyTypes = keyTypes;
             this.gramLength = gramLength;
+            this.isEnforced = isEnforced;
             this.indexType = indexType;
         }
 
@@ -225,10 +229,14 @@ public class CompiledStatements {
             return indexName;
         }
 
-        public List<String> getKeyFields() {
+        public List<List<String>> getKeyFields() {
             return keyFields;
         }
 
+        public List<IAType> getKeyFieldTypes() {
+            return keyTypes;
+        }
+
         public IndexType getIndexType() {
             return indexType;
         }
@@ -237,6 +245,10 @@ public class CompiledStatements {
             return gramLength;
         }
 
+        public boolean isEnforced() {
+            return isEnforced;
+        }
+
         @Override
         public Kind getKind() {
             return Kind.CREATE_INDEX;
@@ -524,29 +536,37 @@ public class CompiledStatements {
 
     public static class CompiledIndexCompactStatement extends CompiledCompactStatement {
         private final String indexName;
-        private final List<String> keyFields;
+        private final List<List<String>> keyFields;
+        private final List<IAType> keyTypes;
         private final IndexType indexType;
+        private final boolean isEnforced;
 
         // Specific to NGram index.
         private final int gramLength;
 
         public CompiledIndexCompactStatement(String dataverseName, String datasetName, String indexName,
-                List<String> keyFields, int gramLength, IndexType indexType) {
+                List<List<String>> keyFields, List<IAType> keyTypes, boolean isEnforced, int gramLength, IndexType indexType) {
             super(dataverseName, datasetName);
             this.indexName = indexName;
             this.keyFields = keyFields;
+            this.keyTypes = keyTypes;
             this.gramLength = gramLength;
             this.indexType = indexType;
+            this.isEnforced = isEnforced;
         }
 
         public String getIndexName() {
             return indexName;
         }
 
-        public List<String> getKeyFields() {
+        public List<List<String>> getKeyFields() {
             return keyFields;
         }
 
+        public List<IAType> getKeyTypes() {
+            return keyTypes;
+        }
+
         public IndexType getIndexType() {
             return indexType;
         }
@@ -554,6 +574,9 @@ public class CompiledStatements {
         public int getGramLength() {
             return gramLength;
         }
-    }
 
-}
\ No newline at end of file
+        public boolean isEnforced() {
+            return isEnforced;
+        }
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/04b2b77a/asterix-algebra/src/main/java/edu/uci/ics/asterix/translator/TypeTranslator.java
----------------------------------------------------------------------
diff --git a/asterix-algebra/src/main/java/edu/uci/ics/asterix/translator/TypeTranslator.java b/asterix-algebra/src/main/java/edu/uci/ics/asterix/translator/TypeTranslator.java
index 7554346..d14b425 100644
--- a/asterix-algebra/src/main/java/edu/uci/ics/asterix/translator/TypeTranslator.java
+++ b/asterix-algebra/src/main/java/edu/uci/ics/asterix/translator/TypeTranslator.java
@@ -24,7 +24,6 @@ import java.util.Map;
 import edu.uci.ics.asterix.aql.expression.OrderedListTypeDefinition;
 import edu.uci.ics.asterix.aql.expression.RecordTypeDefinition;
 import edu.uci.ics.asterix.aql.expression.RecordTypeDefinition.RecordKind;
-import edu.uci.ics.asterix.aql.expression.TypeDecl;
 import edu.uci.ics.asterix.aql.expression.TypeExpression;
 import edu.uci.ics.asterix.aql.expression.TypeReferenceExpression;
 import edu.uci.ics.asterix.aql.expression.UnorderedListTypeDefinition;
@@ -49,29 +48,20 @@ import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
 
 public class TypeTranslator {
 
-    public static Map<TypeSignature, IAType> computeTypes(MetadataTransactionContext mdTxnCtx, TypeDecl tDec,
-            String defaultDataverse) throws AlgebricksException, MetadataException {
+    public static Map<TypeSignature, IAType> computeTypes(MetadataTransactionContext mdTxnCtx, TypeExpression typeExpr,
+            String typeName, String typeDataverse) throws AlgebricksException, MetadataException {
         Map<TypeSignature, IAType> typeMap = new HashMap<TypeSignature, IAType>();
-        Map<String, Map<ARecordType, List<Integer>>> incompleteFieldTypes = new HashMap<String, Map<ARecordType, List<Integer>>>();
-        Map<TypeSignature, List<AbstractCollectionType>> incompleteItemTypes = new HashMap<TypeSignature, List<AbstractCollectionType>>();
-        Map<TypeSignature, List<TypeSignature>> incompleteTopLevelTypeReferences = new HashMap<TypeSignature, List<TypeSignature>>();
-        String typeDataverse = tDec.getDataverseName() == null ? defaultDataverse : tDec.getDataverseName().getValue();
-        firstPass(tDec, typeMap, incompleteFieldTypes, incompleteItemTypes, incompleteTopLevelTypeReferences,
-                typeDataverse);
-        secondPass(mdTxnCtx, typeMap, incompleteFieldTypes, incompleteItemTypes, incompleteTopLevelTypeReferences,
-                typeDataverse);
-
-        return typeMap;
+        return computeTypes(mdTxnCtx, typeExpr, typeName, typeDataverse, typeMap);
     }
 
-    public static Map<TypeSignature, IAType> computeTypes(MetadataTransactionContext mdTxnCtx, TypeDecl tDec,
-            String defaultDataverse, Map<TypeSignature, IAType> typeMap) throws AlgebricksException, MetadataException {
+    public static Map<TypeSignature, IAType> computeTypes(MetadataTransactionContext mdTxnCtx, TypeExpression typeExpr,
+            String typeName, String typeDataverse, Map<TypeSignature, IAType> typeMap) throws AlgebricksException,
+            MetadataException {
         Map<String, Map<ARecordType, List<Integer>>> incompleteFieldTypes = new HashMap<String, Map<ARecordType, List<Integer>>>();
         Map<TypeSignature, List<AbstractCollectionType>> incompleteItemTypes = new HashMap<TypeSignature, List<AbstractCollectionType>>();
         Map<TypeSignature, List<TypeSignature>> incompleteTopLevelTypeReferences = new HashMap<TypeSignature, List<TypeSignature>>();
-        String typeDataverse = tDec.getDataverseName() == null ? defaultDataverse : tDec.getDataverseName().getValue();
-        firstPass(tDec, typeMap, incompleteFieldTypes, incompleteItemTypes, incompleteTopLevelTypeReferences,
-                typeDataverse);
+        firstPass(typeExpr, typeName, typeMap, incompleteFieldTypes, incompleteItemTypes,
+                incompleteTopLevelTypeReferences, typeDataverse);
         secondPass(mdTxnCtx, typeMap, incompleteFieldTypes, incompleteItemTypes, incompleteTopLevelTypeReferences,
                 typeDataverse);
 
@@ -80,46 +70,45 @@ public class TypeTranslator {
 
     private static Map<String, BuiltinType> builtinTypeMap = AsterixBuiltinTypeMap.getBuiltinTypes();
 
-    private static void firstPass(TypeDecl td, Map<TypeSignature, IAType> typeMap,
+    private static void firstPass(TypeExpression typeExpr, String typeName, Map<TypeSignature, IAType> typeMap,
             Map<String, Map<ARecordType, List<Integer>>> incompleteFieldTypes,
             Map<TypeSignature, List<AbstractCollectionType>> incompleteItemTypes,
             Map<TypeSignature, List<TypeSignature>> incompleteTopLevelTypeReferences, String typeDataverse)
             throws AlgebricksException {
 
-        TypeExpression texpr = td.getTypeDef();
-        String tdname = td.getIdent().getValue();
-        if (builtinTypeMap.get(tdname) != null) {
-            throw new AlgebricksException("Cannot redefine builtin type " + tdname + " .");
+        if (builtinTypeMap.get(typeName) != null) {
+            throw new AlgebricksException("Cannot redefine builtin type " + typeName + " .");
         }
-        TypeSignature typeSignature = new TypeSignature(typeDataverse, tdname);
+        TypeSignature typeSignature = new TypeSignature(typeDataverse, typeName);
         try {
-            switch (texpr.getTypeKind()) {
+            switch (typeExpr.getTypeKind()) {
                 case TYPEREFERENCE: {
-                    TypeReferenceExpression tre = (TypeReferenceExpression) texpr;
-                    IAType t = solveTypeReference(typeSignature, typeMap);
+                    TypeReferenceExpression tre = (TypeReferenceExpression) typeExpr;
+                    IAType t = solveTypeReference(new TypeSignature(typeDataverse, tre.getIdent().getValue()), typeMap);
                     if (t != null) {
                         typeMap.put(typeSignature, t);
                     } else {
-                        addIncompleteTopLevelTypeReference(tdname, tre, incompleteTopLevelTypeReferences, typeDataverse);
+                        addIncompleteTopLevelTypeReference(typeName, tre, incompleteTopLevelTypeReferences,
+                                typeDataverse);
                     }
                     break;
                 }
                 case RECORD: {
-                    RecordTypeDefinition rtd = (RecordTypeDefinition) texpr;
+                    RecordTypeDefinition rtd = (RecordTypeDefinition) typeExpr;
                     ARecordType recType = computeRecordType(typeSignature, rtd, typeMap, incompleteFieldTypes,
                             incompleteItemTypes, typeDataverse);
                     typeMap.put(typeSignature, recType);
                     break;
                 }
                 case ORDEREDLIST: {
-                    OrderedListTypeDefinition oltd = (OrderedListTypeDefinition) texpr;
+                    OrderedListTypeDefinition oltd = (OrderedListTypeDefinition) typeExpr;
                     AOrderedListType olType = computeOrderedListType(typeSignature, oltd, typeMap, incompleteItemTypes,
                             incompleteFieldTypes, typeDataverse);
                     typeMap.put(typeSignature, olType);
                     break;
                 }
                 case UNORDEREDLIST: {
-                    UnorderedListTypeDefinition ultd = (UnorderedListTypeDefinition) texpr;
+                    UnorderedListTypeDefinition ultd = (UnorderedListTypeDefinition) typeExpr;
                     AUnorderedListType ulType = computeUnorderedListType(typeSignature, ultd, typeMap,
                             incompleteItemTypes, incompleteFieldTypes, typeDataverse);
                     typeMap.put(typeSignature, ulType);


Mime
View raw message