asterixdb-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From buyin...@apache.org
Subject [2/2] incubator-asterixdb git commit: Supprt querying meta record with meta().
Date Thu, 03 Mar 2016 16:48:02 GMT
Supprt querying meta record with meta().

Change-Id: Ie65417b6baf209ca0ab413cfa4a5f7fc5156ca63
Reviewed-on: https://asterix-gerrit.ics.uci.edu/685
Tested-by: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Reviewed-by: Till Westmann <tillw@apache.org>


Project: http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/commit/cc17a182
Tree: http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/tree/cc17a182
Diff: http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/diff/cc17a182

Branch: refs/heads/master
Commit: cc17a18282cc17341baa124412a20649b4815fd0
Parents: c9bfe25
Author: Yingyi Bu <yingyi@couchbase.com>
Authored: Thu Mar 3 00:37:51 2016 -0800
Committer: Yingyi Bu <buyingyi@gmail.com>
Committed: Thu Mar 3 08:42:57 2016 -0800

----------------------------------------------------------------------
 .../physical/ExternalDataLookupPOperator.java   |   3 +-
 .../asterix/optimizer/base/RuleCollections.java |   2 +
 .../optimizer/rules/ConstantFoldingRule.java    |   2 +-
 ...IntroduceSecondaryIndexInsertDeleteRule.java | 109 +++++----
 .../rules/MetaFunctionToMetaVariableRule.java   | 220 +++++++++++++++++++
 .../am/AbstractIntroduceAccessMethodRule.java   |  36 +--
 .../optimizer/rules/am/AccessMethodUtils.java   | 106 +++------
 .../optimizer/rules/am/BTreeAccessMethod.java   |  11 +-
 .../am/IntroduceLSMComponentFilterRule.java     |  11 +-
 .../rules/am/InvertedIndexAccessMethod.java     |  15 +-
 .../optimizer/rules/am/RTreeAccessMethod.java   |   7 +-
 .../rules/util/EquivalenceClassUtils.java       |   2 +-
 .../LangExpressionToPlanTranslator.java         |   4 +-
 .../asterix/translator/util/ValidateUtil.java   |  35 +--
 .../asterix/aql/translator/QueryTranslator.java |  29 ++-
 .../file/SecondaryIndexOperationsHelper.java    |  15 +-
 .../dataset_with_meta-1.1.ddl.aql               |   4 +-
 .../dataset_with_meta-2.1.ddl.aql               |   4 +-
 .../dataset_with_meta-3.1.ddl.aql               |   2 +-
 .../dataset_with_meta-5.1.ddl.aql               |   4 +-
 .../dataset_with_meta-6.1.ddl.aql               |   4 +-
 .../dataset_with_meta-6.1.ddl.aql               |   4 +-
 .../dataset_with_meta-1.1.adm                   |   2 +-
 .../dataset_with_meta-2.1.adm                   |   2 +-
 .../dataset_with_meta-5.3.adm                   |   2 +-
 .../queries/meta/primary_index_with_meta-1.aql  |  39 ++++
 .../queries/meta/primary_index_with_meta-2.aql  |  39 ++++
 .../meta/primary_index_with_meta_index_join.aql |  52 +++++
 .../primary_index_with_meta_self_index_join.aql |  43 ++++
 .../results/meta/primary_index_with_meta-1.plan |   8 +
 .../results/meta/primary_index_with_meta-2.plan |   8 +
 .../primary_index_with_meta_index_join.plan     |  17 ++
 ...primary_index_with_meta_self_index_join.plan |  17 ++
 .../query_dataset_with_meta-1.1.ddl.aql         |   4 +-
 .../query_dataset_with_meta-2.1.ddl.aql         |   4 +-
 .../query_dataset_with_meta_failure.1.ddl.aql   |  34 +++
 ...query_dataset_with_meta_failure.2.update.aql |  19 ++
 .../query_dataset_with_meta_failure.3.query.aql |  26 +++
 .../query_dataset_with_meta_failure.4.ddl.aql   |  20 ++
 ..._dataset_with_meta_primary_index-1.1.ddl.aql |  34 +++
 ...taset_with_meta_primary_index-1.2.update.aql |  19 ++
 ...ataset_with_meta_primary_index-1.3.query.aql |  25 +++
 ..._dataset_with_meta_primary_index-1.4.ddl.aql |  20 ++
 ..._dataset_with_meta_primary_index-2.1.ddl.aql |  34 +++
 ...taset_with_meta_primary_index-2.2.update.aql |  19 ++
 ...ataset_with_meta_primary_index-2.3.query.aql |  25 +++
 ..._dataset_with_meta_primary_index-2.4.ddl.aql |  20 ++
 .../query_dataset_with_meta_failure.1.adm       |   0
 ...uery_dataset_with_meta_primary_index-1.1.adm |   0
 ...uery_dataset_with_meta_primary_index-2.1.adm |   0
 .../src/test/resources/runtimets/testsuite.xml  |  16 ++
 .../common/exceptions/AsterixException.java     |   4 +-
 .../metadata/declared/AqlDataSource.java        |  16 ++
 .../metadata/declared/AqlMetadataProvider.java  |  23 +-
 .../metadata/declared/DatasetDataSource.java    |  23 +-
 .../apache/asterix/metadata/entities/Index.java |  17 +-
 .../entities/InternalDatasetDetails.java        |   8 +
 .../IndexTupleTranslator.java                   |  36 ++-
 .../asterix/metadata/utils/DatasetUtils.java    |  37 +---
 .../metadata/utils/KeyFieldTypeUtils.java       | 209 ++++++++++++++++++
 .../om/functions/AsterixBuiltinFunctions.java   |   7 +
 .../impl/FieldAccessNestedResultType.java       |  17 +-
 .../impl/OptionalOpenARecordTypeComputer.java   |  43 ++++
 .../apache/asterix/om/types/ARecordType.java    |   9 +-
 .../runtime/formats/NonTaggedDataFormat.java    |   9 +-
 65 files changed, 1299 insertions(+), 336 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/cc17a182/asterix-algebra/src/main/java/org/apache/asterix/algebra/operators/physical/ExternalDataLookupPOperator.java
----------------------------------------------------------------------
diff --git a/asterix-algebra/src/main/java/org/apache/asterix/algebra/operators/physical/ExternalDataLookupPOperator.java b/asterix-algebra/src/main/java/org/apache/asterix/algebra/operators/physical/ExternalDataLookupPOperator.java
index 4c0818b..1f5b47e 100644
--- a/asterix-algebra/src/main/java/org/apache/asterix/algebra/operators/physical/ExternalDataLookupPOperator.java
+++ b/asterix-algebra/src/main/java/org/apache/asterix/algebra/operators/physical/ExternalDataLookupPOperator.java
@@ -114,7 +114,8 @@ public class ExternalDataLookupPOperator extends AbstractScanPOperator {
     public void computeDeliveredProperties(ILogicalOperator op, IOptimizationContext context)
             throws AlgebricksException {
         AqlDataSource ds = new DatasetDataSource(datasetId, datasetId.getDataverseName(), datasetId.getDatasourceName(),
-                recordType, null /*external dataset doesn't have meta records.*/, AqlDataSourceType.EXTERNAL_DATASET);
+                recordType, null /*external dataset doesn't have meta records.*/, AqlDataSourceType.EXTERNAL_DATASET,
+                dataset.getDatasetDetails());
         IDataSourcePropertiesProvider dspp = ds.getPropertiesProvider();
         AbstractScanOperator as = (AbstractScanOperator) op;
         deliveredProperties = dspp.computePropertiesVector(as.getVariables());

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/cc17a182/asterix-algebra/src/main/java/org/apache/asterix/optimizer/base/RuleCollections.java
----------------------------------------------------------------------
diff --git a/asterix-algebra/src/main/java/org/apache/asterix/optimizer/base/RuleCollections.java b/asterix-algebra/src/main/java/org/apache/asterix/optimizer/base/RuleCollections.java
index ad5283a..8627b97 100644
--- a/asterix-algebra/src/main/java/org/apache/asterix/optimizer/base/RuleCollections.java
+++ b/asterix-algebra/src/main/java/org/apache/asterix/optimizer/base/RuleCollections.java
@@ -50,6 +50,7 @@ import org.apache.asterix.optimizer.rules.IntroduceSecondaryIndexInsertDeleteRul
 import org.apache.asterix.optimizer.rules.IntroduceStaticTypeCastForInsertRule;
 import org.apache.asterix.optimizer.rules.IntroduceUnnestForCollectionToSequenceRule;
 import org.apache.asterix.optimizer.rules.LoadRecordFieldsRule;
+import org.apache.asterix.optimizer.rules.MetaFunctionToMetaVariableRule;
 import org.apache.asterix.optimizer.rules.NestGroupByRule;
 import org.apache.asterix.optimizer.rules.PushAggFuncIntoStandaloneAggregateRule;
 import org.apache.asterix.optimizer.rules.PushAggregateIntoGroupbyRule;
@@ -167,6 +168,7 @@ public final class RuleCollections {
         normalization.add(new ConstantFoldingRule());
         normalization.add(new RemoveRedundantSelectRule());
         normalization.add(new UnnestToDataScanRule());
+        normalization.add(new MetaFunctionToMetaVariableRule());
         normalization.add(new IfElseToSwitchCaseFunctionRule());
         normalization.add(new FuzzyEqRule());
         normalization.add(new SimilarityCheckRule());

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/cc17a182/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/ConstantFoldingRule.java
----------------------------------------------------------------------
diff --git a/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/ConstantFoldingRule.java b/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/ConstantFoldingRule.java
index ebe01a7..c534454 100644
--- a/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/ConstantFoldingRule.java
+++ b/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/ConstantFoldingRule.java
@@ -90,7 +90,7 @@ public class ConstantFoldingRule implements IAlgebraicRewriteRule {
             AsterixBuiltinFunctions.GET_RECORD_FIELD_VALUE, AsterixBuiltinFunctions.FIELD_ACCESS_NESTED,
             AsterixBuiltinFunctions.GET_ITEM, AsterixBuiltinFunctions.OPEN_RECORD_CONSTRUCTOR,
             AsterixBuiltinFunctions.FIELD_ACCESS_BY_INDEX, AsterixBuiltinFunctions.CAST_RECORD,
-            AsterixBuiltinFunctions.CAST_LIST);
+            AsterixBuiltinFunctions.CAST_LIST, AsterixBuiltinFunctions.META);
 
     /** Throws exceptions in substituiteProducedVariable, setVarType, and one getVarType method. */
     private static final IVariableTypeEnvironment _emptyTypeEnv = new IVariableTypeEnvironment() {

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/cc17a182/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/IntroduceSecondaryIndexInsertDeleteRule.java
----------------------------------------------------------------------
diff --git a/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/IntroduceSecondaryIndexInsertDeleteRule.java b/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/IntroduceSecondaryIndexInsertDeleteRule.java
index 956e6c5..ab09188 100644
--- a/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/IntroduceSecondaryIndexInsertDeleteRule.java
+++ b/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/IntroduceSecondaryIndexInsertDeleteRule.java
@@ -18,7 +18,6 @@
  */
 package org.apache.asterix.optimizer.rules;
 
-import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.Comparator;
@@ -558,67 +557,62 @@ public class IntroduceSecondaryIndexInsertDeleteRule implements IAlgebraicRewrit
             throws AsterixException, AlgebricksException {
         ARecordType enforcedType = initialType;
         for (int i = 0; i < index.getKeyFieldNames().size(); i++) {
-            try {
-                Stack<Pair<ARecordType, String>> nestedTypeStack = new Stack<Pair<ARecordType, String>>();
-                List<String> splits = index.getKeyFieldNames().get(i);
-                ARecordType nestedFieldType = enforcedType;
-                boolean openRecords = false;
-                String bridgeName = nestedFieldType.getTypeName();
-                int j;
-                // Build the stack for the enforced type
-                for (j = 1; j < splits.size(); j++) {
-                    nestedTypeStack.push(new Pair<ARecordType, String>(nestedFieldType, splits.get(j - 1)));
-                    bridgeName = nestedFieldType.getTypeName();
-                    nestedFieldType = (ARecordType) enforcedType.getSubFieldType(splits.subList(0, j));
-                    if (nestedFieldType == null) {
-                        openRecords = true;
-                        break;
-                    }
+
+            Stack<Pair<ARecordType, String>> nestedTypeStack = new Stack<Pair<ARecordType, String>>();
+            List<String> splits = index.getKeyFieldNames().get(i);
+            ARecordType nestedFieldType = enforcedType;
+            boolean openRecords = false;
+            String bridgeName = nestedFieldType.getTypeName();
+            int j;
+            // Build the stack for the enforced type
+            for (j = 1; j < splits.size(); j++) {
+                nestedTypeStack.push(new Pair<ARecordType, String>(nestedFieldType, splits.get(j - 1)));
+                bridgeName = nestedFieldType.getTypeName();
+                nestedFieldType = (ARecordType) enforcedType.getSubFieldType(splits.subList(0, j));
+                if (nestedFieldType == null) {
+                    openRecords = true;
+                    break;
                 }
-                if (openRecords == true) {
-                    // create the smallest record
-                    enforcedType = new ARecordType(splits.get(splits.size() - 2),
-                            new String[] { splits.get(splits.size() - 1) },
-                            new IAType[] { AUnionType.createNullableType(index.getKeyFieldTypes().get(i)) }, true);
-                    // create the open part of the nested field
-                    for (int k = splits.size() - 3; k > (j - 2); k--) {
-                        enforcedType = new ARecordType(splits.get(k), new String[] { splits.get(k + 1) },
-                                new IAType[] { AUnionType.createNullableType(enforcedType) }, true);
-                    }
-                    // Bridge the gap
-                    Pair<ARecordType, String> gapPair = nestedTypeStack.pop();
-                    ARecordType parent = gapPair.first;
+            }
+            if (openRecords == true) {
+                // create the smallest record
+                enforcedType = new ARecordType(splits.get(splits.size() - 2),
+                        new String[] { splits.get(splits.size() - 1) },
+                        new IAType[] { AUnionType.createNullableType(index.getKeyFieldTypes().get(i)) }, true);
+                // create the open part of the nested field
+                for (int k = splits.size() - 3; k > (j - 2); k--) {
+                    enforcedType = new ARecordType(splits.get(k), new String[] { splits.get(k + 1) },
+                            new IAType[] { AUnionType.createNullableType(enforcedType) }, true);
+                }
+                // Bridge the gap
+                Pair<ARecordType, String> gapPair = nestedTypeStack.pop();
+                ARecordType parent = gapPair.first;
 
-                    IAType[] parentFieldTypes = ArrayUtils.addAll(parent.getFieldTypes().clone(),
-                            new IAType[] { AUnionType.createNullableType(enforcedType) });
-                    enforcedType = new ARecordType(bridgeName,
-                            ArrayUtils.addAll(parent.getFieldNames(), enforcedType.getTypeName()), parentFieldTypes,
-                            true);
+                IAType[] parentFieldTypes = ArrayUtils.addAll(parent.getFieldTypes().clone(),
+                        new IAType[] { AUnionType.createNullableType(enforcedType) });
+                enforcedType = new ARecordType(bridgeName,
+                        ArrayUtils.addAll(parent.getFieldNames(), enforcedType.getTypeName()), parentFieldTypes, true);
 
-                } else {
-                    // Schema is closed all the way to the field
-                    // enforced fields are either null or strongly typed
-                    enforcedType = new ARecordType(nestedFieldType.getTypeName(),
-                            ArrayUtils.addAll(nestedFieldType.getFieldNames(), splits.get(splits.size() - 1)),
-                            ArrayUtils.addAll(nestedFieldType.getFieldTypes(),
-                                    AUnionType.createNullableType(index.getKeyFieldTypes().get(i))),
-                            nestedFieldType.isOpen());
-                }
+            } else {
+                // Schema is closed all the way to the field
+                // enforced fields are either null or strongly typed
+                enforcedType = new ARecordType(nestedFieldType.getTypeName(),
+                        ArrayUtils.addAll(nestedFieldType.getFieldNames(), splits.get(splits.size() - 1)),
+                        ArrayUtils.addAll(nestedFieldType.getFieldTypes(),
+                                AUnionType.createNullableType(index.getKeyFieldTypes().get(i))),
+                        nestedFieldType.isOpen());
+            }
 
-                // Create the enforcedtype for the nested fields in the schema, from the ground up
-                if (nestedTypeStack.size() > 0) {
-                    while (!nestedTypeStack.isEmpty()) {
-                        Pair<ARecordType, String> nestedTypePair = nestedTypeStack.pop();
-                        ARecordType nestedRecType = nestedTypePair.first;
-                        IAType[] nestedRecTypeFieldTypes = nestedRecType.getFieldTypes().clone();
-                        nestedRecTypeFieldTypes[nestedRecType.getFieldIndex(nestedTypePair.second)] = enforcedType;
-                        enforcedType = new ARecordType(nestedRecType.getTypeName(), nestedRecType.getFieldNames(),
-                                nestedRecTypeFieldTypes, nestedRecType.isOpen());
-                    }
+            // Create the enforcedtype for the nested fields in the schema, from the ground up
+            if (nestedTypeStack.size() > 0) {
+                while (!nestedTypeStack.isEmpty()) {
+                    Pair<ARecordType, String> nestedTypePair = nestedTypeStack.pop();
+                    ARecordType nestedRecType = nestedTypePair.first;
+                    IAType[] nestedRecTypeFieldTypes = nestedRecType.getFieldTypes().clone();
+                    nestedRecTypeFieldTypes[nestedRecType.getFieldIndex(nestedTypePair.second)] = enforcedType;
+                    enforcedType = new ARecordType(nestedRecType.getTypeName(), nestedRecType.getFieldNames(),
+                            nestedRecTypeFieldTypes, nestedRecType.isOpen());
                 }
-
-            } catch (IOException e) {
-                throw new AsterixException(e);
             }
         }
         return enforcedType;
@@ -628,6 +622,7 @@ public class IntroduceSecondaryIndexInsertDeleteRule implements IAlgebraicRewrit
      * This method takes a list of {fields}: a subset of {recordFields}, the original record variable
      * and populate expressions with expressions which evaluate to those fields (using field access functions) and
      * variables to represent them
+     *
      * @param fields
      *            desired fields
      * @param recordFields

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/cc17a182/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/MetaFunctionToMetaVariableRule.java
----------------------------------------------------------------------
diff --git a/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/MetaFunctionToMetaVariableRule.java b/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/MetaFunctionToMetaVariableRule.java
new file mode 100644
index 0000000..cd06303
--- /dev/null
+++ b/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/MetaFunctionToMetaVariableRule.java
@@ -0,0 +1,220 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.optimizer.rules;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.asterix.metadata.declared.AqlDataSource;
+import org.apache.asterix.om.functions.AsterixBuiltinFunctions;
+import org.apache.commons.lang3.mutable.Mutable;
+import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
+import org.apache.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import org.apache.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import org.apache.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import org.apache.hyracks.algebricks.core.algebra.base.LogicalExpressionTag;
+import org.apache.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
+import org.apache.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import org.apache.hyracks.algebricks.core.algebra.expressions.AbstractFunctionCallExpression;
+import org.apache.hyracks.algebricks.core.algebra.expressions.VariableReferenceExpression;
+import org.apache.hyracks.algebricks.core.algebra.operators.logical.DataSourceScanOperator;
+import org.apache.hyracks.algebricks.core.algebra.visitors.ILogicalExpressionReferenceTransform;
+import org.apache.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
+
+/**
+ * This rule rewrites all meta() function calls in a query plan
+ * to proper variable references.
+ */
+public class MetaFunctionToMetaVariableRule implements IAlgebraicRewriteRule {
+    // The rule can only apply once.
+    private boolean hasApplied = false;
+    private boolean rewritten = false;
+
+    @Override
+    public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context)
+            throws AlgebricksException {
+        if (hasApplied) {
+            return false;
+        }
+        hasApplied = true;
+        visit(opRef);
+        return rewritten;
+    }
+
+    private ILogicalExpressionReferenceTransformWithCondition visit(Mutable<ILogicalOperator> opRef)
+            throws AlgebricksException {
+        ILogicalOperator op = opRef.getValue();
+
+        // Reaches NTS or ETS.
+        if (op.getInputs().size() == 0) {
+            return NoOpExpressionReferenceTransform.INSTANCE;
+        }
+
+        // Datascan returns an useful transform if the meta part presents in the dataset.
+        if (op.getOperatorTag() == LogicalOperatorTag.DATASOURCESCAN) {
+            DataSourceScanOperator scanOp = (DataSourceScanOperator) op;
+            ILogicalExpressionReferenceTransformWithCondition inputTransfomer = visit(op.getInputs().get(0));
+            AqlDataSource dataSource = (AqlDataSource) scanOp.getDataSource();
+            if (!dataSource.hasMeta()) {
+                return inputTransfomer;
+            };
+            List<LogicalVariable> allVars = scanOp.getVariables();
+            LogicalVariable dataVar = dataSource.getDataRecordVariable(allVars);
+            LogicalVariable metaVar = dataSource.getMetaVariable(allVars);
+            LogicalExpressionReferenceTransform currentTransformer = new LogicalExpressionReferenceTransform(dataVar,
+                    metaVar);
+            if (inputTransfomer.equals(NoOpExpressionReferenceTransform.INSTANCE)) {
+                return currentTransformer;
+            } else {
+                // Requires an argument variable to resolve ambiguity.
+                List<ILogicalExpressionReferenceTransformWithCondition> transformers = new ArrayList<>();
+                inputTransfomer.setVariableRequired();
+                currentTransformer.setVariableRequired();
+                transformers.add(inputTransfomer);
+                transformers.add(currentTransformer);
+                return new CompositeExpressionReferenceTransform(transformers);
+            }
+        }
+
+        // Visits children in the depth-first order.
+        List<ILogicalExpressionReferenceTransformWithCondition> transformers = new ArrayList<>();
+        for (Mutable<ILogicalOperator> childRef : op.getInputs()) {
+            ILogicalExpressionReferenceTransformWithCondition transformer = visit(childRef);
+            if (!transformer.equals(NoOpExpressionReferenceTransform.INSTANCE)) {
+                transformers.add(transformer);
+            }
+        }
+        ILogicalExpressionReferenceTransformWithCondition currentTransformer = null;
+        if (transformers.size() == 0) {
+            currentTransformer = NoOpExpressionReferenceTransform.INSTANCE;
+        } else if (transformers.size() == 1) {
+            currentTransformer = transformers.get(0);
+        } else {
+            // Transformers in a CompositeTransformer should require argument variable check.
+            for (ILogicalExpressionReferenceTransformWithCondition transformer : transformers) {
+                transformer.setVariableRequired();
+            }
+            currentTransformer = new CompositeExpressionReferenceTransform(transformers);
+        }
+        rewritten |= op.acceptExpressionTransform(currentTransformer);
+        return currentTransformer;
+    }
+}
+
+interface ILogicalExpressionReferenceTransformWithCondition extends ILogicalExpressionReferenceTransform {
+    default void setVariableRequired() {
+
+    }
+}
+
+class NoOpExpressionReferenceTransform implements ILogicalExpressionReferenceTransformWithCondition {
+    static NoOpExpressionReferenceTransform INSTANCE = new NoOpExpressionReferenceTransform();
+
+    private NoOpExpressionReferenceTransform() {
+
+    }
+
+    @Override
+    public boolean transform(Mutable<ILogicalExpression> expression) throws AlgebricksException {
+        return false;
+    }
+
+}
+
+class LogicalExpressionReferenceTransform implements ILogicalExpressionReferenceTransformWithCondition {
+    private final LogicalVariable dataVar;
+    private final LogicalVariable metaVar;
+    private boolean variableRequired = false;
+
+    LogicalExpressionReferenceTransform(LogicalVariable dataVar, LogicalVariable metaVar) {
+        this.dataVar = dataVar;
+        this.metaVar = metaVar;
+    }
+
+    @Override
+    public void setVariableRequired() {
+        this.variableRequired = true;
+    }
+
+    @Override
+    public boolean transform(Mutable<ILogicalExpression> exprRef) throws AlgebricksException {
+        ILogicalExpression expr = exprRef.getValue();
+        if (expr.getExpressionTag() != LogicalExpressionTag.FUNCTION_CALL) {
+            return false;
+        }
+        AbstractFunctionCallExpression funcExpr = (AbstractFunctionCallExpression) expr;
+        List<Mutable<ILogicalExpression>> argRefs = funcExpr.getArguments();
+
+        // Recursively transform argument expressions.
+        for (Mutable<ILogicalExpression> argRef : argRefs) {
+            transform(argRef);
+        }
+
+        if (!funcExpr.getFunctionIdentifier().equals(AsterixBuiltinFunctions.META)) {
+            return false;
+        }
+        // The user query provides more than one parameter for the meta function.
+        if (argRefs.size() > 1) {
+            throw new AlgebricksException("The meta function can at most have one argument!");
+        }
+
+        // The user query provides exact one parameter for the meta function.
+        if (argRefs.size() == 1) {
+            ILogicalExpression argExpr = argRefs.get(0).getValue();
+            if (argExpr.getExpressionTag() != LogicalExpressionTag.VARIABLE) {
+                return false;
+            }
+            VariableReferenceExpression argVarExpr = (VariableReferenceExpression) argExpr;
+            LogicalVariable argVar = argVarExpr.getVariableReference();
+            if (!dataVar.equals(argVar)) {
+                return false;
+            }
+            exprRef.setValue(new VariableReferenceExpression(metaVar));
+            return true;
+        }
+
+        // The user query provides zero parameter for the meta function.
+        if (variableRequired) {
+            throw new AlgebricksException("Cannot resolve to ambiguity on the meta function call --"
+                    + " there are more than once dataset choices!");
+        }
+        exprRef.setValue(new VariableReferenceExpression(metaVar));
+        return true;
+    }
+}
+
+class CompositeExpressionReferenceTransform implements ILogicalExpressionReferenceTransformWithCondition {
+    private final List<ILogicalExpressionReferenceTransformWithCondition> transformers;
+
+    public CompositeExpressionReferenceTransform(List<ILogicalExpressionReferenceTransformWithCondition> transforms) {
+        this.transformers = transforms;
+    }
+
+    @Override
+    public boolean transform(Mutable<ILogicalExpression> expression) throws AlgebricksException {
+        // Tries transfomations one by one.
+        for (ILogicalExpressionReferenceTransform transformer : transformers) {
+            if (transformer.transform(expression)) {
+                return true;
+            }
+        }
+        return false;
+    }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/cc17a182/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/AbstractIntroduceAccessMethodRule.java
----------------------------------------------------------------------
diff --git a/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/AbstractIntroduceAccessMethodRule.java b/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/AbstractIntroduceAccessMethodRule.java
index cbf0dec..0152c12 100644
--- a/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/AbstractIntroduceAccessMethodRule.java
+++ b/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/AbstractIntroduceAccessMethodRule.java
@@ -18,7 +18,6 @@
  */
 package org.apache.asterix.optimizer.rules.am;
 
-import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Iterator;
@@ -229,9 +228,11 @@ public abstract class AbstractIntroduceAccessMethodRule implements IAlgebraicRew
                     //Prune indexes based on field types
                     List<IAType> indexedTypes = new ArrayList<IAType>();
                     //retrieve types of expressions joined/selected with an indexed field
-                    for (int j = 0; j < optFuncExpr.getNumLogicalVars(); j++)
-                        if (j != exprAndVarIdx.second)
+                    for (int j = 0; j < optFuncExpr.getNumLogicalVars(); j++) {
+                        if (j != exprAndVarIdx.second) {
                             indexedTypes.add(optFuncExpr.getFieldType(j));
+                        }
+                    }
 
                     //add constants in case of select
                     if (indexedTypes.size() < 2 && optFuncExpr.getNumLogicalVars() == 1
@@ -247,8 +248,9 @@ public abstract class AbstractIntroduceAccessMethodRule implements IAlgebraicRew
 
                                 @Override
                                 public Object getVarType(LogicalVariable var) throws AlgebricksException {
-                                    if (var.equals(optFuncExpr.getSourceVar(exprAndVarIdx.second)))
+                                    if (var.equals(optFuncExpr.getSourceVar(exprAndVarIdx.second))) {
                                         return keyType;
+                                    }
                                     throw new IllegalArgumentException();
                                 }
 
@@ -256,8 +258,9 @@ public abstract class AbstractIntroduceAccessMethodRule implements IAlgebraicRew
                                 public Object getVarType(LogicalVariable var, List<LogicalVariable> nonNullVariables,
                                         List<List<LogicalVariable>> correlatedNullableVariableLists)
                                                 throws AlgebricksException {
-                                    if (var.equals(optFuncExpr.getSourceVar(exprAndVarIdx.second)))
+                                    if (var.equals(optFuncExpr.getSourceVar(exprAndVarIdx.second))) {
                                         return keyType;
+                                    }
                                     throw new IllegalArgumentException();
                                 }
 
@@ -282,9 +285,11 @@ public abstract class AbstractIntroduceAccessMethodRule implements IAlgebraicRew
                     boolean jaccardSimilarity = optFuncExpr.getFuncExpr().getFunctionIdentifier().getName()
                             .startsWith("similarity-jaccard-check");
 
-                    for (int j = 0; j < indexedTypes.size(); j++)
-                        for (int k = j + 1; k < indexedTypes.size(); k++)
+                    for (int j = 0; j < indexedTypes.size(); j++) {
+                        for (int k = j + 1; k < indexedTypes.size(); k++) {
                             typeMatch &= isMatched(indexedTypes.get(j), indexedTypes.get(k), jaccardSimilarity);
+                        }
+                    }
 
                     // Check if any field name in the optFuncExpr matches.
                     if (optFuncExpr.findFieldName(keyField) != -1) {
@@ -333,8 +338,9 @@ public abstract class AbstractIntroduceAccessMethodRule implements IAlgebraicRew
 
     private boolean isMatched(IAType type1, IAType type2, boolean useListDomain) throws AlgebricksException {
         if (ATypeHierarchy.isSameTypeDomain(Index.getNonNullableType(type1).first.getTypeTag(),
-                Index.getNonNullableType(type2).first.getTypeTag(), useListDomain))
+                Index.getNonNullableType(type2).first.getTypeTag(), useListDomain)) {
             return true;
+        }
         return ATypeHierarchy.canPromote(Index.getNonNullableType(type1).first.getTypeTag(),
                 Index.getNonNullableType(type2).first.getTypeTag());
     }
@@ -438,9 +444,10 @@ public abstract class AbstractIntroduceAccessMethodRule implements IAlgebraicRew
             if (index.getKeyFieldNames().contains(fieldName) && index.getPendingOp() == IMetadataEntity.PENDING_NO_OP) {
                 indexCandidates.add(index);
                 if (optFuncExpr.getFieldType(varIdx) == BuiltinType.ANULL
-                        || optFuncExpr.getFieldType(varIdx) == BuiltinType.ANY)
+                        || optFuncExpr.getFieldType(varIdx) == BuiltinType.ANY) {
                     optFuncExpr.setFieldType(varIdx,
                             index.getKeyFieldTypes().get(index.getKeyFieldNames().indexOf(fieldName)));
+                }
                 analysisCtx.addIndexExpr(matchedSubTree.dataset, index, matchedFuncExprIndex, varIdx);
             }
         }
@@ -455,9 +462,10 @@ public abstract class AbstractIntroduceAccessMethodRule implements IAlgebraicRew
             IOptimizationContext context) throws AlgebricksException {
         int optFuncExprIndex = 0;
         List<Index> datasetIndexes = new ArrayList<Index>();
-        if (subTree.dataSourceType != DataSourceType.COLLECTION_SCAN)
+        if (subTree.dataSourceType != DataSourceType.COLLECTION_SCAN) {
             datasetIndexes = metadataProvider.getDatasetIndexes(subTree.dataset.getDataverseName(),
                     subTree.dataset.getDatasetName());
+        }
         for (IOptimizableFuncExpr optFuncExpr : analysisCtx.matchedFuncExprs) {
             // Try to match variables from optFuncExpr to assigns or unnests.
             for (int assignOrUnnestIndex = 0; assignOrUnnestIndex < subTree.assignsAndUnnests
@@ -744,12 +752,8 @@ public abstract class AbstractIntroduceAccessMethodRule implements IAlgebraicRew
                 }
 
                 if (!isByName) {
-                    try {
-                        fieldName = ((ARecordType) recordType.getSubFieldType(parentFieldNames))
-                                .getFieldNames()[fieldIndex];
-                    } catch (IOException e) {
-                        throw new AlgebricksException(e);
-                    }
+                    fieldName = ((ARecordType) recordType.getSubFieldType(parentFieldNames))
+                            .getFieldNames()[fieldIndex];
                 }
                 optFuncExpr.setSourceVar(funcVarIndex, ((AssignOperator) op).getVariables().get(assignVarIndex));
                 //add fieldName to the nested fieldName, return

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/cc17a182/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/AccessMethodUtils.java
----------------------------------------------------------------------
diff --git a/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/AccessMethodUtils.java b/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/AccessMethodUtils.java
index f39d190..294a098 100644
--- a/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/AccessMethodUtils.java
+++ b/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/AccessMethodUtils.java
@@ -19,7 +19,6 @@
 
 package org.apache.asterix.optimizer.rules.am;
 
-import java.io.IOException;
 import java.util.ArrayList;
 import java.util.HashSet;
 import java.util.List;
@@ -36,6 +35,7 @@ import org.apache.asterix.metadata.entities.Dataset;
 import org.apache.asterix.metadata.entities.ExternalDatasetDetails;
 import org.apache.asterix.metadata.entities.Index;
 import org.apache.asterix.metadata.utils.DatasetUtils;
+import org.apache.asterix.metadata.utils.KeyFieldTypeUtils;
 import org.apache.asterix.om.base.ABoolean;
 import org.apache.asterix.om.base.AInt32;
 import org.apache.asterix.om.base.AInt64;
@@ -47,7 +47,6 @@ import org.apache.asterix.om.types.ARecordType;
 import org.apache.asterix.om.types.ATypeTag;
 import org.apache.asterix.om.types.IAType;
 import org.apache.asterix.om.types.hierachy.ATypeHierarchy;
-import org.apache.asterix.om.util.NonTaggedFormatUtil;
 import org.apache.commons.lang3.mutable.Mutable;
 import org.apache.commons.lang3.mutable.MutableObject;
 import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
@@ -82,14 +81,17 @@ import org.apache.hyracks.algebricks.core.algebra.util.OperatorPropertiesUtil;
  */
 public class AccessMethodUtils {
 
-    public static void appendPrimaryIndexTypes(Dataset dataset, IAType itemType, List<Object> target)
-            throws IOException, AlgebricksException {
+    public static void appendPrimaryIndexTypes(Dataset dataset, IAType itemType, IAType metaItemType,
+            List<Object> target) throws AlgebricksException {
         ARecordType recordType = (ARecordType) itemType;
-        List<List<String>> partitioningKeys = DatasetUtils.getPartitioningKeys(dataset);
-        for (List<String> partitioningKey : partitioningKeys) {
-            target.add(recordType.getSubFieldType(partitioningKey));
-        }
+        ARecordType metaRecordType = (ARecordType) metaItemType;
+        target.addAll(KeyFieldTypeUtils.getPartitoningKeyTypes(dataset, recordType, metaRecordType));
+        // Adds data record type.
         target.add(itemType);
+        // Adds meta record type if any.
+        if (dataset.hasMetaPart()) {
+            target.add(metaItemType);
+        }
     }
 
     public static ConstantExpression createStringConstant(String str) {
@@ -172,8 +174,9 @@ public class AccessMethodUtils {
                 constantExpressionType);
         for (IOptimizableFuncExpr optFuncExpr : analysisCtx.matchedFuncExprs) {
             //avoid additional optFuncExpressions in case of a join
-            if (optFuncExpr.getFuncExpr().equals(funcExpr))
+            if (optFuncExpr.getFuncExpr().equals(funcExpr)) {
                 return true;
+            }
         }
         analysisCtx.matchedFuncExprs.add(newOptFuncExpr);
         return true;
@@ -196,61 +199,29 @@ public class AccessMethodUtils {
                 new LogicalVariable[] { fieldVar1, fieldVar2 }, new ILogicalExpression[0], new IAType[0]);
         for (IOptimizableFuncExpr optFuncExpr : analysisCtx.matchedFuncExprs) {
             //avoid additional optFuncExpressions in case of a join
-            if (optFuncExpr.getFuncExpr().equals(funcExpr))
+            if (optFuncExpr.getFuncExpr().equals(funcExpr)) {
                 return true;
+            }
         }
         analysisCtx.matchedFuncExprs.add(newOptFuncExpr);
         return true;
     }
 
-    public static int getNumSecondaryKeys(Index index, ARecordType recordType) throws AlgebricksException {
-        switch (index.getIndexType()) {
-            case BTREE:
-            case SINGLE_PARTITION_WORD_INVIX:
-            case SINGLE_PARTITION_NGRAM_INVIX:
-            case LENGTH_PARTITIONED_WORD_INVIX:
-            case LENGTH_PARTITIONED_NGRAM_INVIX: {
-                return index.getKeyFieldNames().size();
-            }
-            case RTREE: {
-                Pair<IAType, Boolean> keyPairType = Index.getNonNullableOpenFieldType(index.getKeyFieldTypes().get(0),
-                        index.getKeyFieldNames().get(0), recordType);
-                IAType keyType = keyPairType.first;
-                int numDimensions = NonTaggedFormatUtil.getNumDimensions(keyType.getTypeTag());
-                return numDimensions * 2;
-            }
-            default: {
-                throw new AlgebricksException("Unknown index kind: " + index.getIndexType());
-            }
-        }
-    }
-
     /**
      * Appends the types of the fields produced by the given secondary index to dest.
      */
-    public static void appendSecondaryIndexTypes(Dataset dataset, ARecordType recordType, Index index,
-            boolean primaryKeysOnly, List<Object> dest) throws AlgebricksException {
+    public static void appendSecondaryIndexTypes(Dataset dataset, ARecordType recordType, ARecordType metaRecordType,
+            Index index, boolean primaryKeysOnly, List<Object> dest) throws AlgebricksException {
         if (!primaryKeysOnly) {
             switch (index.getIndexType()) {
                 case BTREE:
                 case SINGLE_PARTITION_WORD_INVIX:
                 case SINGLE_PARTITION_NGRAM_INVIX: {
-                    for (int i = 0; i < index.getKeyFieldNames().size(); i++) {
-                        Pair<IAType, Boolean> keyPairType = Index.getNonNullableOpenFieldType(
-                                index.getKeyFieldTypes().get(i), index.getKeyFieldNames().get(i), recordType);
-                        dest.add(keyPairType.first);
-                    }
+                    dest.addAll(KeyFieldTypeUtils.getBTreeIndexKeyTypes(index, recordType, metaRecordType));
                     break;
                 }
                 case RTREE: {
-                    Pair<IAType, Boolean> keyPairType = Index.getNonNullableOpenFieldType(
-                            index.getKeyFieldTypes().get(0), index.getKeyFieldNames().get(0), recordType);
-                    IAType keyType = keyPairType.first;
-                    IAType nestedKeyType = NonTaggedFormatUtil.getNestedSpatialType(keyType.getTypeTag());
-                    int numKeys = getNumSecondaryKeys(index, recordType);
-                    for (int i = 0; i < numKeys; i++) {
-                        dest.add(nestedKeyType);
-                    }
+                    dest.addAll(KeyFieldTypeUtils.getRTreeIndexKeyTypes(index, recordType, metaRecordType));
                     break;
                 }
                 case LENGTH_PARTITIONED_NGRAM_INVIX:
@@ -268,20 +239,13 @@ public class AccessMethodUtils {
                 throw new AlgebricksException(e);
             }
         } else {
-            List<List<String>> partitioningKeys = DatasetUtils.getPartitioningKeys(dataset);
-            for (List<String> partitioningKey : partitioningKeys) {
-                try {
-                    dest.add(recordType.getSubFieldType(partitioningKey));
-                } catch (IOException e) {
-                    throw new AlgebricksException(e);
-                }
-            }
+            dest.addAll(KeyFieldTypeUtils.getPartitoningKeyTypes(dataset, recordType, metaRecordType));
         }
     }
 
-    public static void appendSecondaryIndexOutputVars(Dataset dataset, ARecordType recordType, Index index,
-            boolean primaryKeysOnly, IOptimizationContext context, List<LogicalVariable> dest)
-                    throws AlgebricksException {
+    public static void appendSecondaryIndexOutputVars(Dataset dataset, ARecordType recordType,
+            ARecordType metaRecordType, Index index, boolean primaryKeysOnly, IOptimizationContext context,
+            List<LogicalVariable> dest) throws AlgebricksException {
         int numPrimaryKeys = 0;
         if (dataset.getDatasetType() == DatasetType.EXTERNAL) {
             numPrimaryKeys = IndexingConstants
@@ -289,7 +253,7 @@ public class AccessMethodUtils {
         } else {
             numPrimaryKeys = DatasetUtils.getPartitioningKeys(dataset).size();
         }
-        int numSecondaryKeys = getNumSecondaryKeys(index, recordType);
+        int numSecondaryKeys = KeyFieldTypeUtils.getNumSecondaryKeys(index, recordType, metaRecordType);
         int numVars = (primaryKeysOnly) ? numPrimaryKeys : numPrimaryKeys + numSecondaryKeys;
         for (int i = 0; i < numVars; i++) {
             dest.add(context.newVar());
@@ -422,9 +386,10 @@ public class AccessMethodUtils {
         return indexExprs.get(0).second;
     }
 
-    public static UnnestMapOperator createSecondaryIndexUnnestMap(Dataset dataset, ARecordType recordType, Index index,
-            ILogicalOperator inputOp, AccessMethodJobGenParams jobGenParams, IOptimizationContext context,
-            boolean outputPrimaryKeysOnly, boolean retainInput) throws AlgebricksException {
+    public static UnnestMapOperator createSecondaryIndexUnnestMap(Dataset dataset, ARecordType recordType,
+            ARecordType metaRecordType, Index index, ILogicalOperator inputOp, AccessMethodJobGenParams jobGenParams,
+            IOptimizationContext context, boolean outputPrimaryKeysOnly, boolean retainInput)
+                    throws AlgebricksException {
         // The job gen parameters are transferred to the actual job gen via the UnnestMapOperator's function arguments.
         ArrayList<Mutable<ILogicalExpression>> secondaryIndexFuncArgs = new ArrayList<Mutable<ILogicalExpression>>();
         jobGenParams.writeToFuncArgs(secondaryIndexFuncArgs);
@@ -432,9 +397,10 @@ public class AccessMethodUtils {
         List<LogicalVariable> secondaryIndexUnnestVars = new ArrayList<LogicalVariable>();
         List<Object> secondaryIndexOutputTypes = new ArrayList<Object>();
         // Append output variables/types generated by the secondary-index search (not forwarded from input).
-        appendSecondaryIndexOutputVars(dataset, recordType, index, outputPrimaryKeysOnly, context,
+        appendSecondaryIndexOutputVars(dataset, recordType, metaRecordType, index, outputPrimaryKeysOnly, context,
                 secondaryIndexUnnestVars);
-        appendSecondaryIndexTypes(dataset, recordType, index, outputPrimaryKeysOnly, secondaryIndexOutputTypes);
+        appendSecondaryIndexTypes(dataset, recordType, metaRecordType, index, outputPrimaryKeysOnly,
+                secondaryIndexOutputTypes);
         // An index search is expressed as an unnest over an index-search function.
         IFunctionInfo secondaryIndexSearch = FunctionUtil.getFunctionInfo(AsterixBuiltinFunctions.INDEX_SEARCH);
         UnnestingFunctionCallExpression secondaryIndexSearchFunc = new UnnestingFunctionCallExpression(
@@ -452,9 +418,9 @@ public class AccessMethodUtils {
     }
 
     public static UnnestMapOperator createPrimaryIndexUnnestMap(AbstractDataSourceOperator dataSourceOp,
-            Dataset dataset, ARecordType recordType, ILogicalOperator inputOp, IOptimizationContext context,
-            boolean sortPrimaryKeys, boolean retainInput, boolean retainNull, boolean requiresBroadcast)
-                    throws AlgebricksException {
+            Dataset dataset, ARecordType recordType, ARecordType metaRecordType, ILogicalOperator inputOp,
+            IOptimizationContext context, boolean sortPrimaryKeys, boolean retainInput, boolean retainNull,
+            boolean requiresBroadcast) throws AlgebricksException {
         List<LogicalVariable> primaryKeyVars = AccessMethodUtils.getPrimaryKeyVarsFromSecondaryUnnestMap(dataset,
                 inputOp);
         // Optionally add a sort on the primary-index keys before searching the primary index.
@@ -488,11 +454,7 @@ public class AccessMethodUtils {
         List<Object> primaryIndexOutputTypes = new ArrayList<Object>();
         // Append output variables/types generated by the primary-index search (not forwarded from input).
         primaryIndexUnnestVars.addAll(dataSourceOp.getVariables());
-        try {
-            appendPrimaryIndexTypes(dataset, recordType, primaryIndexOutputTypes);
-        } catch (IOException e) {
-            throw new AlgebricksException(e);
-        }
+        appendPrimaryIndexTypes(dataset, recordType, metaRecordType, primaryIndexOutputTypes);
         // An index search is expressed as an unnest over an index-search function.
         IFunctionInfo primaryIndexSearch = FunctionUtil.getFunctionInfo(AsterixBuiltinFunctions.INDEX_SEARCH);
         AbstractFunctionCallExpression primaryIndexSearchFunc = new ScalarFunctionCallExpression(primaryIndexSearch,

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/cc17a182/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/BTreeAccessMethod.java
----------------------------------------------------------------------
diff --git a/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/BTreeAccessMethod.java b/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/BTreeAccessMethod.java
index 1bbb13b..15dbe95 100644
--- a/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/BTreeAccessMethod.java
+++ b/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/BTreeAccessMethod.java
@@ -19,7 +19,6 @@
 
 package org.apache.asterix.optimizer.rules.am;
 
-import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.BitSet;
@@ -481,7 +480,7 @@ public class BTreeAccessMethod implements IAccessMethod {
         }
 
         UnnestMapOperator secondaryIndexUnnestOp = AccessMethodUtils.createSecondaryIndexUnnestMap(dataset, recordType,
-                chosenIndex, inputOp, jobGenParams, context, false, retainInput);
+                metaRecordType, chosenIndex, inputOp, jobGenParams, context, false, retainInput);
 
         // Generate the rest of the upstream plan which feeds the search results into the primary index.
         UnnestMapOperator primaryIndexUnnestOp = null;
@@ -494,7 +493,7 @@ public class BTreeAccessMethod implements IAccessMethod {
             return externalDataAccessOp;
         } else if (!isPrimaryIndex) {
             primaryIndexUnnestOp = AccessMethodUtils.createPrimaryIndexUnnestMap(dataSourceOp, dataset, recordType,
-                    secondaryIndexUnnestOp, context, true, retainInput, retainNull, false);
+                    metaRecordType, secondaryIndexUnnestOp, context, true, retainInput, retainNull, false);
 
             // Adds equivalence classes --- one equivalent class between a primary key
             // variable and a record field-access expression.
@@ -502,11 +501,7 @@ public class BTreeAccessMethod implements IAccessMethod {
                     dataSourceOp.getVariables(), recordType, metaRecordType, dataset, context);
         } else {
             List<Object> primaryIndexOutputTypes = new ArrayList<Object>();
-            try {
-                AccessMethodUtils.appendPrimaryIndexTypes(dataset, recordType, primaryIndexOutputTypes);
-            } catch (IOException e) {
-                throw new AlgebricksException(e);
-            }
+            AccessMethodUtils.appendPrimaryIndexTypes(dataset, recordType, metaRecordType, primaryIndexOutputTypes);
             List<LogicalVariable> scanVariables = dataSourceOp.getVariables();
             primaryIndexUnnestOp = new UnnestMapOperator(scanVariables, secondaryIndexUnnestOp.getExpressionRef(),
                     primaryIndexOutputTypes, retainInput);

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/cc17a182/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/IntroduceLSMComponentFilterRule.java
----------------------------------------------------------------------
diff --git a/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/IntroduceLSMComponentFilterRule.java b/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/IntroduceLSMComponentFilterRule.java
index 08a4e74..a2bf6d9 100644
--- a/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/IntroduceLSMComponentFilterRule.java
+++ b/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/IntroduceLSMComponentFilterRule.java
@@ -29,6 +29,7 @@ import org.apache.asterix.metadata.declared.DatasetDataSource;
 import org.apache.asterix.metadata.entities.Dataset;
 import org.apache.asterix.metadata.entities.Index;
 import org.apache.asterix.metadata.utils.DatasetUtils;
+import org.apache.asterix.metadata.utils.KeyFieldTypeUtils;
 import org.apache.asterix.om.base.AInt32;
 import org.apache.asterix.om.base.AString;
 import org.apache.asterix.om.constants.AsterixConstantValue;
@@ -109,7 +110,7 @@ public class IntroduceLSMComponentFilterRule implements IAlgebraicRewriteRule {
 
         for (int i = 0; i < analysisCtx.matchedFuncExprs.size(); i++) {
             IOptimizableFuncExpr optFuncExpr = analysisCtx.matchedFuncExprs.get(i);
-            boolean found = findMacthedExprFieldName(optFuncExpr, op, dataset, recType, datasetIndexes);
+            boolean found = findMacthedExprFieldName(optFuncExpr, op, dataset, recType, datasetIndexes, context);
             if (found && optFuncExpr.getFieldName(0).equals(filterFieldName)) {
                 optFuncExprs.add(optFuncExpr);
             }
@@ -301,7 +302,8 @@ public class IntroduceLSMComponentFilterRule implements IAlgebraicRewriteRule {
     }
 
     private boolean findMacthedExprFieldName(IOptimizableFuncExpr optFuncExpr, AbstractLogicalOperator op,
-            Dataset dataset, ARecordType recType, List<Index> datasetIndexes) throws AlgebricksException {
+            Dataset dataset, ARecordType recType, List<Index> datasetIndexes, IOptimizationContext context)
+                    throws AlgebricksException {
         AbstractLogicalOperator descendantOp = (AbstractLogicalOperator) op.getInputs().get(0).getValue();
         while (descendantOp != null) {
             if (descendantOp.getOperatorTag() == LogicalOperatorTag.ASSIGN) {
@@ -368,7 +370,10 @@ public class IntroduceLSMComponentFilterRule implements IAlgebraicRewriteRule {
                         }
                     }
 
-                    int numSecondaryKeys = AccessMethodUtils.getNumSecondaryKeys(index, recType);
+                    IAType metaItemType = ((AqlMetadataProvider) context.getMetadataProvider())
+                            .findType(dataset.getMetaItemTypeDataverseName(), dataset.getMetaItemTypeName());
+                    ARecordType metaRecType = (ARecordType) metaItemType;
+                    int numSecondaryKeys = KeyFieldTypeUtils.getNumSecondaryKeys(index, recType, metaRecType);
                     List<String> fieldName;
                     if (varIndex >= numSecondaryKeys) {
                         fieldName = DatasetUtils.getPartitioningKeys(dataset).get(varIndex - numSecondaryKeys);

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/cc17a182/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/InvertedIndexAccessMethod.java
----------------------------------------------------------------------
diff --git a/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/InvertedIndexAccessMethod.java b/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/InvertedIndexAccessMethod.java
index 824fe7a..70d7088 100644
--- a/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/InvertedIndexAccessMethod.java
+++ b/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/InvertedIndexAccessMethod.java
@@ -66,7 +66,6 @@ import org.apache.hyracks.algebricks.core.algebra.operators.logical.AbstractLogi
 import org.apache.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator.ExecutionMode;
 import org.apache.hyracks.algebricks.core.algebra.operators.logical.AssignOperator;
 import org.apache.hyracks.algebricks.core.algebra.operators.logical.DataSourceScanOperator;
-import org.apache.hyracks.algebricks.core.algebra.operators.logical.EmptyTupleSourceOperator;
 import org.apache.hyracks.algebricks.core.algebra.operators.logical.InnerJoinOperator;
 import org.apache.hyracks.algebricks.core.algebra.operators.logical.ReplicateOperator;
 import org.apache.hyracks.algebricks.core.algebra.operators.logical.SelectOperator;
@@ -364,13 +363,13 @@ public class InvertedIndexAccessMethod implements IAccessMethod {
     @Override
     public ILogicalOperator createSecondaryToPrimaryPlan(Mutable<ILogicalExpression> conditionRef,
             OptimizableOperatorSubTree indexSubTree, OptimizableOperatorSubTree probeSubTree, Index chosenIndex,
-            AccessMethodAnalysisContext analysisCtx,
-            boolean retainInput, boolean retainNull, boolean requiresBroadcast, IOptimizationContext context)
-                    throws AlgebricksException {
+            AccessMethodAnalysisContext analysisCtx, boolean retainInput, boolean retainNull, boolean requiresBroadcast,
+            IOptimizationContext context) throws AlgebricksException {
 
         IOptimizableFuncExpr optFuncExpr = AccessMethodUtils.chooseFirstOptFuncExpr(chosenIndex, analysisCtx);
         Dataset dataset = indexSubTree.dataset;
         ARecordType recordType = indexSubTree.recordType;
+        ARecordType metaRecordType = indexSubTree.metaRecordType;
         // we made sure indexSubTree has datasource scan
         DataSourceScanOperator dataSourceScan = (DataSourceScanOperator) indexSubTree.dataSourceRef.getValue();
 
@@ -407,11 +406,11 @@ public class InvertedIndexAccessMethod implements IAccessMethod {
         }
         jobGenParams.setKeyVarList(keyVarList);
         UnnestMapOperator secondaryIndexUnnestOp = AccessMethodUtils.createSecondaryIndexUnnestMap(dataset, recordType,
-                chosenIndex, inputOp, jobGenParams, context, true, retainInput);
+                metaRecordType, chosenIndex, inputOp, jobGenParams, context, true, retainInput);
 
         // Generate the rest of the upstream plan which feeds the search results into the primary index.
         UnnestMapOperator primaryIndexUnnestOp = AccessMethodUtils.createPrimaryIndexUnnestMap(dataSourceScan, dataset,
-                recordType, secondaryIndexUnnestOp, context, true, retainInput, retainNull, false);
+                recordType, metaRecordType, secondaryIndexUnnestOp, context, true, retainInput, retainNull, false);
 
         return primaryIndexUnnestOp;
     }
@@ -436,8 +435,8 @@ public class InvertedIndexAccessMethod implements IAccessMethod {
     public boolean applySelectPlanTransformation(Mutable<ILogicalOperator> selectRef,
             OptimizableOperatorSubTree subTree, Index chosenIndex, AccessMethodAnalysisContext analysisCtx,
             IOptimizationContext context) throws AlgebricksException {
-        ILogicalOperator indexPlanRootOp = createSecondaryToPrimaryPlan(null, subTree, null, chosenIndex, analysisCtx, false,
-                false, false, context);
+        ILogicalOperator indexPlanRootOp = createSecondaryToPrimaryPlan(null, subTree, null, chosenIndex, analysisCtx,
+                false, false, false, context);
         // Replace the datasource scan with the new plan rooted at primaryIndexUnnestMap.
         subTree.dataSourceRef.setValue(indexPlanRootOp);
         return true;

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/cc17a182/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/RTreeAccessMethod.java
----------------------------------------------------------------------
diff --git a/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/RTreeAccessMethod.java b/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/RTreeAccessMethod.java
index 3c925e2..111fcf4 100644
--- a/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/RTreeAccessMethod.java
+++ b/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/RTreeAccessMethod.java
@@ -179,6 +179,7 @@ public class RTreeAccessMethod implements IAccessMethod {
         IOptimizableFuncExpr optFuncExpr = AccessMethodUtils.chooseFirstOptFuncExpr(chosenIndex, analysisCtx);
         Dataset dataset = indexSubTree.dataset;
         ARecordType recordType = indexSubTree.recordType;
+        ARecordType metaRecordType = indexSubTree.metaRecordType;
 
         int optFieldIdx = AccessMethodUtils.chooseFirstOptFuncVar(chosenIndex, analysisCtx);
         Pair<IAType, Boolean> keyPairType = Index.getNonNullableOpenFieldType(optFuncExpr.getFieldType(optFieldIdx),
@@ -239,7 +240,7 @@ public class RTreeAccessMethod implements IAccessMethod {
         }
 
         UnnestMapOperator secondaryIndexUnnestOp = AccessMethodUtils.createSecondaryIndexUnnestMap(dataset, recordType,
-                chosenIndex, assignSearchKeys, jobGenParams, context, false, retainInput);
+                metaRecordType, chosenIndex, assignSearchKeys, jobGenParams, context, false, retainInput);
 
         // Generate the rest of the upstream plan which feeds the search results into the primary index.
         if (dataset.getDatasetType() == DatasetType.EXTERNAL) {
@@ -248,8 +249,8 @@ public class RTreeAccessMethod implements IAccessMethod {
             return externalDataAccessOp;
         } else {
             UnnestMapOperator primaryIndexUnnestOp = AccessMethodUtils.createPrimaryIndexUnnestMap(dataSourceOp,
-                    dataset, recordType, secondaryIndexUnnestOp, context, true, retainInput, false, false);
-
+                    dataset, recordType, metaRecordType, secondaryIndexUnnestOp, context, true, retainInput, false,
+                    false);
             return primaryIndexUnnestOp;
         }
     }

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/cc17a182/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/util/EquivalenceClassUtils.java
----------------------------------------------------------------------
diff --git a/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/util/EquivalenceClassUtils.java b/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/util/EquivalenceClassUtils.java
index e85fecf..242e15b 100644
--- a/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/util/EquivalenceClassUtils.java
+++ b/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/util/EquivalenceClassUtils.java
@@ -103,7 +103,7 @@ public class EquivalenceClassUtils {
         for (int pkIndex = 0; pkIndex < primaryKey.size(); ++pkIndex) {
             LogicalVariable referredRecordVar = recordVar;
             String pkFieldName = primaryKey.get(pkIndex).get(0);
-            int source = keySourceIndicators == null ? 0 : keySourceIndicators.get(pkIndex);
+            int source = keySourceIndicators.get(pkIndex);
             Integer fieldIndexInRecord;
             if (source == 0) {
                 // The field is from the main record.

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/cc17a182/asterix-algebra/src/main/java/org/apache/asterix/translator/LangExpressionToPlanTranslator.java
----------------------------------------------------------------------
diff --git a/asterix-algebra/src/main/java/org/apache/asterix/translator/LangExpressionToPlanTranslator.java b/asterix-algebra/src/main/java/org/apache/asterix/translator/LangExpressionToPlanTranslator.java
index c635125..28de56b 100644
--- a/asterix-algebra/src/main/java/org/apache/asterix/translator/LangExpressionToPlanTranslator.java
+++ b/asterix-algebra/src/main/java/org/apache/asterix/translator/LangExpressionToPlanTranslator.java
@@ -457,8 +457,8 @@ class LangExpressionToPlanTranslator
         IAType metaItemType = metadataProvider.findType(dataset.getMetaItemTypeDataverseName(),
                 dataset.getMetaItemTypeName());
         DatasetDataSource dataSource = new DatasetDataSource(sourceId, dataset.getDataverseName(),
-                dataset.getDatasetName(), itemType, metaItemType, AqlDataSourceType.INTERNAL_DATASET);
-
+                dataset.getDatasetName(), itemType, metaItemType, AqlDataSourceType.INTERNAL_DATASET,
+                dataset.getDatasetDetails());
         return dataSource;
     }
 

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/cc17a182/asterix-algebra/src/main/java/org/apache/asterix/translator/util/ValidateUtil.java
----------------------------------------------------------------------
diff --git a/asterix-algebra/src/main/java/org/apache/asterix/translator/util/ValidateUtil.java b/asterix-algebra/src/main/java/org/apache/asterix/translator/util/ValidateUtil.java
index 95a8802..03e3fe8 100644
--- a/asterix-algebra/src/main/java/org/apache/asterix/translator/util/ValidateUtil.java
+++ b/asterix-algebra/src/main/java/org/apache/asterix/translator/util/ValidateUtil.java
@@ -18,12 +18,12 @@
  */
 package org.apache.asterix.translator.util;
 
-import java.io.IOException;
 import java.util.ArrayList;
 import java.util.List;
 
 import org.apache.asterix.common.config.DatasetConfig.IndexType;
 import org.apache.asterix.common.exceptions.AsterixException;
+import org.apache.asterix.metadata.utils.KeyFieldTypeUtils;
 import org.apache.asterix.om.types.ARecordType;
 import org.apache.asterix.om.types.ATypeTag;
 import org.apache.asterix.om.types.BuiltinType;
@@ -47,8 +47,7 @@ public class ValidateUtil {
      * @throws AsterixException
      *             (if the validation failed), IOException
      */
-    public static void validateFilterField(ARecordType recType, List<String> filterField)
-            throws AsterixException, IOException {
+    public static void validateFilterField(ARecordType recType, List<String> filterField) throws AsterixException {
         IAType fieldType = recType.getSubFieldType(filterField);
         if (fieldType == null) {
             throw new AsterixException("A field with this name  \"" + filterField + "\" could not be found.");
@@ -88,7 +87,7 @@ public class ValidateUtil {
      */
     public static List<IAType> validatePartitioningExpressions(ARecordType recType, ARecordType metaRecType,
             List<List<String>> partitioningExprs, List<Integer> keySourceIndicators, boolean autogenerated)
-                    throws AsterixException, IOException {
+                    throws AsterixException {
         List<IAType> partitioningExprTypes = new ArrayList<IAType>(partitioningExprs.size());
         if (autogenerated) {
             if (partitioningExprs.size() > 1) {
@@ -104,11 +103,10 @@ public class ValidateUtil {
                         + ". Autogenerated primary keys must be of type " + ATypeTag.UUID + ".");
             }
         } else {
-            for (int i = 0; i < partitioningExprs.size(); i++) {
-                List<String> fieldName = partitioningExprs.get(i);
-                boolean useMeta = keySourceIndicators.get(i) > 0;
-                IAType fieldType = useMeta ? metaRecType.getSubFieldType(fieldName)
-                        : recType.getSubFieldType(fieldName);
+            partitioningExprTypes = KeyFieldTypeUtils.getKeyTypes(recType, metaRecType, partitioningExprs,
+                    keySourceIndicators);
+            for (int fidx = 0; fidx < partitioningExprTypes.size(); ++fidx) {
+                IAType fieldType = partitioningExprTypes.get(fidx);
                 switch (fieldType.getTypeTag()) {
                     case INT8:
                     case INT16:
@@ -124,13 +122,13 @@ public class ValidateUtil {
                     case DATETIME:
                     case YEARMONTHDURATION:
                     case DAYTIMEDURATION:
-                        partitioningExprTypes.add(fieldType);
                         break;
                     case UNION:
-                        throw new AsterixException("The partitioning key \"" + fieldName + "\" cannot be nullable");
+                        throw new AsterixException(
+                                "The partitioning key \"" + partitioningExprs.get(fidx) + "\" cannot be nullable");
                     default:
-                        throw new AsterixException("The partitioning key \"" + fieldName + "\" cannot be of type "
-                                + fieldType.getTypeTag() + ".");
+                        throw new AsterixException("The partitioning key \"" + partitioningExprs.get(fidx)
+                                + "\" cannot be of type " + fieldType.getTypeTag() + ".");
                 }
             }
         }
@@ -151,12 +149,15 @@ public class ValidateUtil {
      * @throws AsterixException
      *             (if the validation failed), IOException
      */
-    public static void validateKeyFields(ARecordType recType, List<List<String>> keyFieldNames,
-            List<IAType> keyFieldTypes, IndexType indexType) throws AsterixException, IOException {
+    public static void validateKeyFields(ARecordType recType, ARecordType metaRecType, List<List<String>> keyFieldNames,
+            List<Integer> keySourceIndicators, List<IAType> keyFieldTypes, IndexType indexType)
+                    throws AsterixException {
+        List<IAType> fieldTypes = KeyFieldTypeUtils.getKeyTypes(recType, metaRecType, keyFieldNames,
+                keySourceIndicators);
         int pos = 0;
         boolean openFieldCompositeIdx = false;
-        for (List<String> fieldName : keyFieldNames) {
-            IAType fieldType = recType.getSubFieldType(fieldName);
+        for (IAType fieldType : fieldTypes) {
+            List<String> fieldName = keyFieldNames.get(pos);
             if (fieldType == null) {
                 fieldType = keyFieldTypes.get(pos);
                 if (keyFieldTypes.get(pos) == BuiltinType.ANULL) {

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/cc17a182/asterix-app/src/main/java/org/apache/asterix/aql/translator/QueryTranslator.java
----------------------------------------------------------------------
diff --git a/asterix-app/src/main/java/org/apache/asterix/aql/translator/QueryTranslator.java b/asterix-app/src/main/java/org/apache/asterix/aql/translator/QueryTranslator.java
index 75f325b..00bc254 100644
--- a/asterix-app/src/main/java/org/apache/asterix/aql/translator/QueryTranslator.java
+++ b/asterix-app/src/main/java/org/apache/asterix/aql/translator/QueryTranslator.java
@@ -141,6 +141,7 @@ import org.apache.asterix.metadata.entities.NodeGroup;
 import org.apache.asterix.metadata.feeds.FeedMetadataUtil;
 import org.apache.asterix.metadata.utils.DatasetUtils;
 import org.apache.asterix.metadata.utils.ExternalDatasetsRegistry;
+import org.apache.asterix.metadata.utils.KeyFieldTypeUtils;
 import org.apache.asterix.metadata.utils.MetadataLockManager;
 import org.apache.asterix.om.types.ARecordType;
 import org.apache.asterix.om.types.ATypeTag;
@@ -811,6 +812,7 @@ public class QueryTranslator extends AbstractLangTranslator {
         CreateIndexStatement stmtCreateIndex = (CreateIndexStatement) stmt;
         String dataverseName = getActiveDataverse(stmtCreateIndex.getDataverseName());
         String datasetName = stmtCreateIndex.getDatasetName().getValue();
+        List<Integer> keySourceIndicators = stmtCreateIndex.getFieldSourceIndicators();
 
         MetadataTransactionContext mdTxnCtx = MetadataManager.INSTANCE.beginTransaction();
         boolean bActiveTxn = true;
@@ -838,19 +840,24 @@ public class QueryTranslator extends AbstractLangTranslator {
             indexName = stmtCreateIndex.getIndexName().getValue();
             Index idx = MetadataManager.INSTANCE.getIndex(metadataProvider.getMetadataTxnContext(), dataverseName,
                     datasetName, indexName);
-
-            String itemTypeName = ds.getItemTypeName();
             Datatype dt = MetadataManager.INSTANCE.getDatatype(metadataProvider.getMetadataTxnContext(),
-                    ds.getItemTypeDataverseName(), itemTypeName);
-            IAType itemType = dt.getDatatype();
-            ARecordType aRecordType = (ARecordType) itemType;
+                    ds.getItemTypeDataverseName(), ds.getItemTypeName());
+            ARecordType aRecordType = (ARecordType) dt.getDatatype();
+            ARecordType metaRecordType = null;
+            if (ds.hasMetaPart()) {
+                Datatype metaDt = MetadataManager.INSTANCE.getDatatype(metadataProvider.getMetadataTxnContext(),
+                        ds.getMetaItemTypeDataverseName(), ds.getMetaItemTypeName());
+                metaRecordType = (ARecordType) metaDt.getDatatype();
+            }
 
             List<List<String>> indexFields = new ArrayList<List<String>>();
             List<IAType> indexFieldTypes = new ArrayList<IAType>();
+            int keyIndex = 0;
             for (Pair<List<String>, TypeExpression> fieldExpr : stmtCreateIndex.getFieldExprs()) {
                 IAType fieldType = null;
-                boolean isOpen = aRecordType.isOpen();
-                ARecordType subType = aRecordType;
+                ARecordType subType = KeyFieldTypeUtils.chooseSource(keySourceIndicators, keyIndex, aRecordType,
+                        metaRecordType);
+                boolean isOpen = subType.isOpen();
                 int i = 0;
                 if (fieldExpr.first.size() > 1 && !isOpen) {
                     for (; i < fieldExpr.first.size() - 1;) {
@@ -885,9 +892,11 @@ public class QueryTranslator extends AbstractLangTranslator {
 
                 indexFields.add(fieldExpr.first);
                 indexFieldTypes.add(fieldType);
+                ++keyIndex;
             }
 
-            ValidateUtil.validateKeyFields(aRecordType, indexFields, indexFieldTypes, stmtCreateIndex.getIndexType());
+            ValidateUtil.validateKeyFields(aRecordType, metaRecordType, indexFields, keySourceIndicators,
+                    indexFieldTypes, stmtCreateIndex.getIndexType());
 
             if (idx != null) {
                 if (stmtCreateIndex.getIfNotExists()) {
@@ -995,8 +1004,8 @@ public class QueryTranslator extends AbstractLangTranslator {
 
             //#. add a new index with PendingAddOp
             Index index = new Index(dataverseName, datasetName, indexName, stmtCreateIndex.getIndexType(), indexFields,
-                    stmtCreateIndex.getFieldSourceIndicators(), indexFieldTypes, stmtCreateIndex.getGramLength(),
-                    stmtCreateIndex.isEnforced(), false, IMetadataEntity.PENDING_ADD_OP);
+                    keySourceIndicators, indexFieldTypes, stmtCreateIndex.getGramLength(), stmtCreateIndex.isEnforced(),
+                    false, IMetadataEntity.PENDING_ADD_OP);
             MetadataManager.INSTANCE.addIndex(metadataProvider.getMetadataTxnContext(), index);
 
             ARecordType enforcedType = null;

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/cc17a182/asterix-app/src/main/java/org/apache/asterix/file/SecondaryIndexOperationsHelper.java
----------------------------------------------------------------------
diff --git a/asterix-app/src/main/java/org/apache/asterix/file/SecondaryIndexOperationsHelper.java b/asterix-app/src/main/java/org/apache/asterix/file/SecondaryIndexOperationsHelper.java
index 24df771..bd49293 100644
--- a/asterix-app/src/main/java/org/apache/asterix/file/SecondaryIndexOperationsHelper.java
+++ b/asterix-app/src/main/java/org/apache/asterix/file/SecondaryIndexOperationsHelper.java
@@ -20,7 +20,6 @@
 package org.apache.asterix.file;
 
 import java.io.DataOutput;
-import java.io.IOException;
 import java.util.List;
 import java.util.Map;
 
@@ -255,12 +254,7 @@ public abstract class SecondaryIndexOperationsHelper {
             secondaryBTreeFields[i] = i;
         }
 
-        IAType type;
-        try {
-            type = itemType.getSubFieldType(filterFieldName);
-        } catch (IOException e) {
-            throw new AlgebricksException(e);
-        }
+        IAType type = itemType.getSubFieldType(filterFieldName);
         filterCmpFactories[0] = AqlBinaryComparatorFactoryProvider.INSTANCE.getBinaryComparatorFactory(type, true);
         filterTypeTraits[0] = AqlTypeTraitProvider.INSTANCE.getTypeTrait(type);
         secondaryFilterFields[0] = getNumSecondaryKeys() + numPrimaryKeys;
@@ -278,12 +272,7 @@ public abstract class SecondaryIndexOperationsHelper {
         primaryBloomFilterKeyFields = new int[numPrimaryKeys];
         ISerializerDeserializerProvider serdeProvider = metadataProvider.getFormat().getSerdeProvider();
         for (int i = 0; i < numPrimaryKeys; i++) {
-            IAType keyType;
-            try {
-                keyType = itemType.getSubFieldType(partitioningKeys.get(i));
-            } catch (IOException e) {
-                throw new AlgebricksException(e);
-            }
+            IAType keyType = itemType.getSubFieldType(partitioningKeys.get(i));
             primaryRecFields[i] = serdeProvider.getSerializerDeserializer(keyType);
             primaryComparatorFactories[i] = AqlBinaryComparatorFactoryProvider.INSTANCE
                     .getBinaryComparatorFactory(keyType, true);

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/cc17a182/asterix-app/src/test/resources/metadata/queries/basic/dataset_with_meta-1/dataset_with_meta-1.1.ddl.aql
----------------------------------------------------------------------
diff --git a/asterix-app/src/test/resources/metadata/queries/basic/dataset_with_meta-1/dataset_with_meta-1.1.ddl.aql b/asterix-app/src/test/resources/metadata/queries/basic/dataset_with_meta-1/dataset_with_meta-1.1.ddl.aql
index 6539dbc..f2682fa 100644
--- a/asterix-app/src/test/resources/metadata/queries/basic/dataset_with_meta-1/dataset_with_meta-1.1.ddl.aql
+++ b/asterix-app/src/test/resources/metadata/queries/basic/dataset_with_meta-1/dataset_with_meta-1.1.ddl.aql
@@ -21,7 +21,7 @@ drop dataverse test if exists;
 create dataverse test;
 use dataverse test;
 
-create type EmptyType as open {
+create type AuxiliaryType as open {
     id: string
 }
 
@@ -30,6 +30,6 @@ create type LineType as open {
   text: string
 }
 
-create dataset Book(LineType) with meta(EmptyType)
+create dataset Book(LineType) with meta(AuxiliaryType)
 primary key id;
 

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/cc17a182/asterix-app/src/test/resources/metadata/queries/basic/dataset_with_meta-2/dataset_with_meta-2.1.ddl.aql
----------------------------------------------------------------------
diff --git a/asterix-app/src/test/resources/metadata/queries/basic/dataset_with_meta-2/dataset_with_meta-2.1.ddl.aql b/asterix-app/src/test/resources/metadata/queries/basic/dataset_with_meta-2/dataset_with_meta-2.1.ddl.aql
index cec01d2..2f4f181 100644
--- a/asterix-app/src/test/resources/metadata/queries/basic/dataset_with_meta-2/dataset_with_meta-2.1.ddl.aql
+++ b/asterix-app/src/test/resources/metadata/queries/basic/dataset_with_meta-2/dataset_with_meta-2.1.ddl.aql
@@ -20,7 +20,7 @@
 drop dataverse meta if exists;
 create dataverse meta;
 use dataverse meta;
-create type EmptyType as open {
+create type AuxiliaryType as open {
     id: string
 }
 
@@ -34,6 +34,6 @@ create type LineType as open {
   text: string
 }
 
-create dataset Book(LineType) with meta(meta.EmptyType)
+create dataset Book(LineType) with meta(meta.AuxiliaryType)
 primary key id;
 

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/cc17a182/asterix-app/src/test/resources/metadata/queries/basic/dataset_with_meta-3/dataset_with_meta-3.1.ddl.aql
----------------------------------------------------------------------
diff --git a/asterix-app/src/test/resources/metadata/queries/basic/dataset_with_meta-3/dataset_with_meta-3.1.ddl.aql b/asterix-app/src/test/resources/metadata/queries/basic/dataset_with_meta-3/dataset_with_meta-3.1.ddl.aql
index 78bd7ab..138a211 100644
--- a/asterix-app/src/test/resources/metadata/queries/basic/dataset_with_meta-3/dataset_with_meta-3.1.ddl.aql
+++ b/asterix-app/src/test/resources/metadata/queries/basic/dataset_with_meta-3/dataset_with_meta-3.1.ddl.aql
@@ -21,7 +21,7 @@ drop dataverse test if exists;
 create dataverse test;
 use dataverse test;
 
-create type EmptyType as open {
+create type AuxiliaryType as open {
     id: string
 }
 

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/cc17a182/asterix-app/src/test/resources/metadata/queries/basic/dataset_with_meta-5/dataset_with_meta-5.1.ddl.aql
----------------------------------------------------------------------
diff --git a/asterix-app/src/test/resources/metadata/queries/basic/dataset_with_meta-5/dataset_with_meta-5.1.ddl.aql b/asterix-app/src/test/resources/metadata/queries/basic/dataset_with_meta-5/dataset_with_meta-5.1.ddl.aql
index ddb42c8..07df579 100644
--- a/asterix-app/src/test/resources/metadata/queries/basic/dataset_with_meta-5/dataset_with_meta-5.1.ddl.aql
+++ b/asterix-app/src/test/resources/metadata/queries/basic/dataset_with_meta-5/dataset_with_meta-5.1.ddl.aql
@@ -21,7 +21,7 @@ drop dataverse test if exists;
 create dataverse test;
 use dataverse test;
 
-create type EmptyType as open {
+create type AuxiliaryType as open {
   "key":int32
 }
 
@@ -31,6 +31,6 @@ create type LineType as open {
   text: string
 }
 
-create dataset Book(LineType) with meta(EmptyType)
+create dataset Book(LineType) with meta(AuxiliaryType)
 primary key meta()."key";
 

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/cc17a182/asterix-app/src/test/resources/metadata/queries/basic/dataset_with_meta-6/dataset_with_meta-6.1.ddl.aql
----------------------------------------------------------------------
diff --git a/asterix-app/src/test/resources/metadata/queries/basic/dataset_with_meta-6/dataset_with_meta-6.1.ddl.aql b/asterix-app/src/test/resources/metadata/queries/basic/dataset_with_meta-6/dataset_with_meta-6.1.ddl.aql
index 510137f..c987a4c 100644
--- a/asterix-app/src/test/resources/metadata/queries/basic/dataset_with_meta-6/dataset_with_meta-6.1.ddl.aql
+++ b/asterix-app/src/test/resources/metadata/queries/basic/dataset_with_meta-6/dataset_with_meta-6.1.ddl.aql
@@ -21,7 +21,7 @@ drop dataverse test if exists;
 create dataverse test;
 use dataverse test;
 
-create type EmptyType as open {
+create type AuxiliaryType as open {
     id: string
 }
 
@@ -30,7 +30,7 @@ create type LineType as open {
   text: string
 }
 
-create dataset Book(LineType) with meta(EmptyType)
+create dataset Book(LineType) with meta(AuxiliaryType)
 primary key id;
 
 create index MetaIndex on Book(meta().id) type btree;

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/cc17a182/asterix-app/src/test/resources/metadata/queries/basic/dataset_with_meta-7/dataset_with_meta-6.1.ddl.aql
----------------------------------------------------------------------
diff --git a/asterix-app/src/test/resources/metadata/queries/basic/dataset_with_meta-7/dataset_with_meta-6.1.ddl.aql b/asterix-app/src/test/resources/metadata/queries/basic/dataset_with_meta-7/dataset_with_meta-6.1.ddl.aql
index 41b50d0..94f253f 100644
--- a/asterix-app/src/test/resources/metadata/queries/basic/dataset_with_meta-7/dataset_with_meta-6.1.ddl.aql
+++ b/asterix-app/src/test/resources/metadata/queries/basic/dataset_with_meta-7/dataset_with_meta-6.1.ddl.aql
@@ -21,7 +21,7 @@ drop dataverse test if exists;
 create dataverse test;
 use dataverse test;
 
-create type EmptyType as open {
+create type AuxiliaryType as open {
     id: string
 }
 
@@ -30,7 +30,7 @@ create type LineType as open {
   text: string
 }
 
-create dataset Book(LineType) with meta(EmptyType)
+create dataset Book(LineType) with meta(AuxiliaryType)
 primary key id;
 
 create index MetaIndex on Book(meta().id, id) type btree;

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/cc17a182/asterix-app/src/test/resources/metadata/results/basic/dataset_with_meta-1/dataset_with_meta-1.1.adm
----------------------------------------------------------------------
diff --git a/asterix-app/src/test/resources/metadata/results/basic/dataset_with_meta-1/dataset_with_meta-1.1.adm b/asterix-app/src/test/resources/metadata/results/basic/dataset_with_meta-1/dataset_with_meta-1.1.adm
index 776e07f..e41fe03 100644
--- a/asterix-app/src/test/resources/metadata/results/basic/dataset_with_meta-1/dataset_with_meta-1.1.adm
+++ b/asterix-app/src/test/resources/metadata/results/basic/dataset_with_meta-1/dataset_with_meta-1.1.adm
@@ -1 +1 @@
-{ "DataverseName": "test", "DatasetName": "Book", "DatatypeDataverseName": "test", "DatatypeName": "LineType", "DatasetType": "INTERNAL", "GroupName": "DEFAULT_NG_ALL_NODES", "CompactionPolicy": "prefix", "CompactionPolicyProperties": [ { "Name": "max-mergable-component-size", "Value": "1073741824" }, { "Name": "max-tolerance-component-count", "Value": "5" } ], "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ [ "id" ] ], "PrimaryKey": [ [ "id" ] ], "Autogenerated": false }, "ExternalDetails": null, "Hints": {{  }}, "Timestamp": "Wed Feb 24 17:32:57 PST 2016", "DatasetId": 101i32, "PendingOp": 0i32, "MetatypeDataverseName": "test", "MetatypeName": "EmptyType" }
+{ "DataverseName": "test", "DatasetName": "Book", "DatatypeDataverseName": "test", "DatatypeName": "LineType", "DatasetType": "INTERNAL", "GroupName": "DEFAULT_NG_ALL_NODES", "CompactionPolicy": "prefix", "CompactionPolicyProperties": [ { "Name": "max-mergable-component-size", "Value": "1073741824" }, { "Name": "max-tolerance-component-count", "Value": "5" } ], "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ [ "id" ] ], "PrimaryKey": [ [ "id" ] ], "Autogenerated": false }, "ExternalDetails": null, "Hints": {{  }}, "Timestamp": "Wed Feb 24 17:32:57 PST 2016", "DatasetId": 101i32, "PendingOp": 0i32, "MetatypeDataverseName": "test", "MetatypeName": "AuxiliaryType" }

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/cc17a182/asterix-app/src/test/resources/metadata/results/basic/dataset_with_meta-2/dataset_with_meta-2.1.adm
----------------------------------------------------------------------
diff --git a/asterix-app/src/test/resources/metadata/results/basic/dataset_with_meta-2/dataset_with_meta-2.1.adm b/asterix-app/src/test/resources/metadata/results/basic/dataset_with_meta-2/dataset_with_meta-2.1.adm
index f3f00e9..2663a16 100644
--- a/asterix-app/src/test/resources/metadata/results/basic/dataset_with_meta-2/dataset_with_meta-2.1.adm
+++ b/asterix-app/src/test/resources/metadata/results/basic/dataset_with_meta-2/dataset_with_meta-2.1.adm
@@ -1 +1 @@
-{ "DataverseName": "test", "DatasetName": "Book", "DatatypeDataverseName": "test", "DatatypeName": "LineType", "DatasetType": "INTERNAL", "GroupName": "DEFAULT_NG_ALL_NODES", "CompactionPolicy": "prefix", "CompactionPolicyProperties": [ { "Name": "max-mergable-component-size", "Value": "1073741824" }, { "Name": "max-tolerance-component-count", "Value": "5" } ], "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ [ "id" ] ], "PrimaryKey": [ [ "id" ] ], "Autogenerated": false }, "ExternalDetails": null, "Hints": {{  }}, "Timestamp": "Wed Feb 24 17:32:57 PST 2016", "DatasetId": 101i32, "PendingOp": 0i32, "MetatypeDataverseName": "meta", "MetatypeName": "EmptyType" }
+{ "DataverseName": "test", "DatasetName": "Book", "DatatypeDataverseName": "test", "DatatypeName": "LineType", "DatasetType": "INTERNAL", "GroupName": "DEFAULT_NG_ALL_NODES", "CompactionPolicy": "prefix", "CompactionPolicyProperties": [ { "Name": "max-mergable-component-size", "Value": "1073741824" }, { "Name": "max-tolerance-component-count", "Value": "5" } ], "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ [ "id" ] ], "PrimaryKey": [ [ "id" ] ], "Autogenerated": false }, "ExternalDetails": null, "Hints": {{  }}, "Timestamp": "Wed Feb 24 17:32:57 PST 2016", "DatasetId": 101i32, "PendingOp": 0i32, "MetatypeDataverseName": "meta", "MetatypeName": "AuxiliaryType" }



Mime
View raw message