asterixdb-notifications mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From "Yingyi Bu (Code Review)" <do-not-re...@asterixdb.incubator.apache.org>
Subject Change in asterixdb[master]: Supprt querying meta record with meta().
Date Wed, 02 Mar 2016 02:43:57 GMT
Yingyi Bu has uploaded a new change for review.

  https://asterix-gerrit.ics.uci.edu/685

Change subject: Supprt querying meta record with meta().
......................................................................

Supprt querying meta record with meta().

Change-Id: Ie65417b6baf209ca0ab413cfa4a5f7fc5156ca63
---
M asterix-algebra/src/main/java/org/apache/asterix/algebra/operators/physical/ExternalDataLookupPOperator.java
M asterix-algebra/src/main/java/org/apache/asterix/optimizer/base/RuleCollections.java
M asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/ConstantFoldingRule.java
A asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/MetaFunctionToMetaVariableRule.java
M asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/AccessMethodUtils.java
M asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/BTreeAccessMethod.java
M asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/InvertedIndexAccessMethod.java
M asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/RTreeAccessMethod.java
M asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/util/EquivalenceClassUtils.java
M asterix-algebra/src/main/java/org/apache/asterix/translator/LangExpressionToPlanTranslator.java
M asterix-algebra/src/main/java/org/apache/asterix/translator/util/ValidateUtil.java
M asterix-app/src/main/java/org/apache/asterix/aql/translator/QueryTranslator.java
A asterix-app/src/test/resources/runtimets/queries/meta/query_dataset_with_meta_primary_index-1/query_dataset_with_meta_primary_index-1.1.ddl.aql
A asterix-app/src/test/resources/runtimets/queries/meta/query_dataset_with_meta_primary_index-1/query_dataset_with_meta_primary_index-1.2.update.aql
A asterix-app/src/test/resources/runtimets/queries/meta/query_dataset_with_meta_primary_index-1/query_dataset_with_meta_primary_index-1.3.query.aql
A asterix-app/src/test/resources/runtimets/queries/meta/query_dataset_with_meta_primary_index-1/query_dataset_with_meta_primary_index-1.4.ddl.aql
A asterix-app/src/test/resources/runtimets/queries/meta/query_dataset_with_meta_primary_index-2/query_dataset_with_meta_primary_index-2.1.ddl.aql
A asterix-app/src/test/resources/runtimets/queries/meta/query_dataset_with_meta_primary_index-2/query_dataset_with_meta_primary_index-2.2.update.aql
A asterix-app/src/test/resources/runtimets/queries/meta/query_dataset_with_meta_primary_index-2/query_dataset_with_meta_primary_index-2.3.query.aql
A asterix-app/src/test/resources/runtimets/queries/meta/query_dataset_with_meta_primary_index-2/query_dataset_with_meta_primary_index-2.4.ddl.aql
A asterix-app/src/test/resources/runtimets/results/meta/query_dataset_with_meta_primary_index-1/query_dataset_with_meta_primary_index-1.1.adm
A asterix-app/src/test/resources/runtimets/results/meta/query_dataset_with_meta_primary_index-2/query_dataset_with_meta_primary_index-2.1.adm
M asterix-app/src/test/resources/runtimets/testsuite.xml
M asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/AqlMetadataProvider.java
M asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/DatasetDataSource.java
M asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/InternalDatasetDetails.java
M asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/IndexTupleTranslator.java
M asterix-om/src/main/java/org/apache/asterix/om/functions/AsterixBuiltinFunctions.java
A asterix-om/src/main/java/org/apache/asterix/om/typecomputer/impl/OptionalOpenARecordTypeComputer.java
29 files changed, 584 insertions(+), 68 deletions(-)


  git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb refs/changes/85/685/1

diff --git a/asterix-algebra/src/main/java/org/apache/asterix/algebra/operators/physical/ExternalDataLookupPOperator.java b/asterix-algebra/src/main/java/org/apache/asterix/algebra/operators/physical/ExternalDataLookupPOperator.java
index 4c0818b..1f5b47e 100644
--- a/asterix-algebra/src/main/java/org/apache/asterix/algebra/operators/physical/ExternalDataLookupPOperator.java
+++ b/asterix-algebra/src/main/java/org/apache/asterix/algebra/operators/physical/ExternalDataLookupPOperator.java
@@ -114,7 +114,8 @@
     public void computeDeliveredProperties(ILogicalOperator op, IOptimizationContext context)
             throws AlgebricksException {
         AqlDataSource ds = new DatasetDataSource(datasetId, datasetId.getDataverseName(), datasetId.getDatasourceName(),
-                recordType, null /*external dataset doesn't have meta records.*/, AqlDataSourceType.EXTERNAL_DATASET);
+                recordType, null /*external dataset doesn't have meta records.*/, AqlDataSourceType.EXTERNAL_DATASET,
+                dataset.getDatasetDetails());
         IDataSourcePropertiesProvider dspp = ds.getPropertiesProvider();
         AbstractScanOperator as = (AbstractScanOperator) op;
         deliveredProperties = dspp.computePropertiesVector(as.getVariables());
diff --git a/asterix-algebra/src/main/java/org/apache/asterix/optimizer/base/RuleCollections.java b/asterix-algebra/src/main/java/org/apache/asterix/optimizer/base/RuleCollections.java
index ad5283a..8627b97 100644
--- a/asterix-algebra/src/main/java/org/apache/asterix/optimizer/base/RuleCollections.java
+++ b/asterix-algebra/src/main/java/org/apache/asterix/optimizer/base/RuleCollections.java
@@ -50,6 +50,7 @@
 import org.apache.asterix.optimizer.rules.IntroduceStaticTypeCastForInsertRule;
 import org.apache.asterix.optimizer.rules.IntroduceUnnestForCollectionToSequenceRule;
 import org.apache.asterix.optimizer.rules.LoadRecordFieldsRule;
+import org.apache.asterix.optimizer.rules.MetaFunctionToMetaVariableRule;
 import org.apache.asterix.optimizer.rules.NestGroupByRule;
 import org.apache.asterix.optimizer.rules.PushAggFuncIntoStandaloneAggregateRule;
 import org.apache.asterix.optimizer.rules.PushAggregateIntoGroupbyRule;
@@ -167,6 +168,7 @@
         normalization.add(new ConstantFoldingRule());
         normalization.add(new RemoveRedundantSelectRule());
         normalization.add(new UnnestToDataScanRule());
+        normalization.add(new MetaFunctionToMetaVariableRule());
         normalization.add(new IfElseToSwitchCaseFunctionRule());
         normalization.add(new FuzzyEqRule());
         normalization.add(new SimilarityCheckRule());
diff --git a/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/ConstantFoldingRule.java b/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/ConstantFoldingRule.java
index ebe01a7..c534454 100644
--- a/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/ConstantFoldingRule.java
+++ b/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/ConstantFoldingRule.java
@@ -90,7 +90,7 @@
             AsterixBuiltinFunctions.GET_RECORD_FIELD_VALUE, AsterixBuiltinFunctions.FIELD_ACCESS_NESTED,
             AsterixBuiltinFunctions.GET_ITEM, AsterixBuiltinFunctions.OPEN_RECORD_CONSTRUCTOR,
             AsterixBuiltinFunctions.FIELD_ACCESS_BY_INDEX, AsterixBuiltinFunctions.CAST_RECORD,
-            AsterixBuiltinFunctions.CAST_LIST);
+            AsterixBuiltinFunctions.CAST_LIST, AsterixBuiltinFunctions.META);
 
     /** Throws exceptions in substituiteProducedVariable, setVarType, and one getVarType method. */
     private static final IVariableTypeEnvironment _emptyTypeEnv = new IVariableTypeEnvironment() {
diff --git a/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/MetaFunctionToMetaVariableRule.java b/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/MetaFunctionToMetaVariableRule.java
new file mode 100644
index 0000000..5e57b9c
--- /dev/null
+++ b/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/MetaFunctionToMetaVariableRule.java
@@ -0,0 +1,194 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.optimizer.rules;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.asterix.metadata.declared.AqlDataSource;
+import org.apache.asterix.om.functions.AsterixBuiltinFunctions;
+import org.apache.commons.lang3.mutable.Mutable;
+import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
+import org.apache.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import org.apache.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import org.apache.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import org.apache.hyracks.algebricks.core.algebra.base.LogicalExpressionTag;
+import org.apache.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
+import org.apache.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import org.apache.hyracks.algebricks.core.algebra.expressions.AbstractFunctionCallExpression;
+import org.apache.hyracks.algebricks.core.algebra.expressions.VariableReferenceExpression;
+import org.apache.hyracks.algebricks.core.algebra.operators.logical.DataSourceScanOperator;
+import org.apache.hyracks.algebricks.core.algebra.visitors.ILogicalExpressionReferenceTransform;
+import org.apache.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
+
+public class MetaFunctionToMetaVariableRule implements IAlgebraicRewriteRule {
+    // The rule can only apply once.
+    private boolean hasApplied = false;
+    private boolean rewritten = false;
+
+    @Override
+    public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context)
+            throws AlgebricksException {
+        if (hasApplied) {
+            return false;
+        }
+        hasApplied = true;
+        visit(opRef);
+        return rewritten;
+    }
+
+    private ILogicalExpressionReferenceTransform visit(Mutable<ILogicalOperator> opRef) throws AlgebricksException {
+        ILogicalOperator op = opRef.getValue();
+
+        // Reaches NTS or ETS.
+        if (op.getInputs().size() == 0) {
+            return NoOpExpressionReferenceTransform.INSTANCE;
+        }
+
+        // Datascan returns an useful transform if the meta part presents in the dataset.
+        if (op.getOperatorTag() == LogicalOperatorTag.DATASOURCESCAN) {
+            DataSourceScanOperator scanOp = (DataSourceScanOperator) op;
+            AqlDataSource dataSource = (AqlDataSource) scanOp.getDataSource();
+            if (!dataSource.hasMeta()) {
+                return visit(op.getInputs().get(0));
+            };
+            List<LogicalVariable> allVars = scanOp.getVariables();
+            LogicalVariable dataVar = allVars.get(allVars.size() - 2);
+            LogicalVariable metaVar = allVars.get(allVars.size() - 1);
+            return new LogicalExpressionReferenceTransform(dataVar, metaVar);
+        }
+
+        // Visits children in the depth-first order.
+        List<ILogicalExpressionReferenceTransform> transformers = new ArrayList<>();
+        for (Mutable<ILogicalOperator> childRef : op.getInputs()) {
+            ILogicalExpressionReferenceTransform transformer = visit(childRef);
+            if (!transformer.equals(NoOpExpressionReferenceTransform.INSTANCE)) {
+                transformers.add(transformer);
+            }
+        }
+        ILogicalExpressionReferenceTransform currentTransformer = null;
+        if (transformers.size() == 0) {
+            currentTransformer = NoOpExpressionReferenceTransform.INSTANCE;
+        } else if (transformers.size() == 1) {
+            currentTransformer = transformers.get(0);
+        } else {
+            for (ILogicalExpressionReferenceTransform transformer : transformers) {
+                LogicalExpressionReferenceTransform logicalTransformer = (LogicalExpressionReferenceTransform) transformer;
+                logicalTransformer.setVariableRequired();
+            }
+            currentTransformer = new CompositeExpressionReferenceTransform(transformers);
+        }
+        rewritten |= op.acceptExpressionTransform(currentTransformer);
+        return currentTransformer;
+    }
+}
+
+class NoOpExpressionReferenceTransform implements ILogicalExpressionReferenceTransform {
+    static NoOpExpressionReferenceTransform INSTANCE = new NoOpExpressionReferenceTransform();
+
+    private NoOpExpressionReferenceTransform() {
+
+    }
+
+    @Override
+    public boolean transform(Mutable<ILogicalExpression> expression) throws AlgebricksException {
+        return false;
+    }
+}
+
+class LogicalExpressionReferenceTransform implements ILogicalExpressionReferenceTransform {
+    private final LogicalVariable dataVar;
+    private final LogicalVariable metaVar;
+    private boolean variableRequired = false;
+
+    LogicalExpressionReferenceTransform(LogicalVariable dataVar, LogicalVariable metaVar) {
+        this.dataVar = dataVar;
+        this.metaVar = metaVar;
+    }
+
+    void setVariableRequired() {
+        this.variableRequired = true;
+    }
+
+    @Override
+    public boolean transform(Mutable<ILogicalExpression> exprRef) throws AlgebricksException {
+        ILogicalExpression expr = exprRef.getValue();
+        if (expr.getExpressionTag() != LogicalExpressionTag.FUNCTION_CALL) {
+            return false;
+        }
+        AbstractFunctionCallExpression funcExpr = (AbstractFunctionCallExpression) expr;
+        List<Mutable<ILogicalExpression>> argRefs = funcExpr.getArguments();
+
+        // Recursively transform argument expressions.
+        for (Mutable<ILogicalExpression> argRef : argRefs) {
+            transform(argRef);
+        }
+
+        if (!funcExpr.getFunctionIdentifier().equals(AsterixBuiltinFunctions.META)) {
+            return false;
+        }
+        // The user query provides more than one parameter for the meta function.
+        if (argRefs.size() > 1) {
+            throw new AlgebricksException("The meta function can at most have one argument!");
+        }
+
+        // The user query provides exact one parameter for the meta function.
+        if (argRefs.size() == 1) {
+            ILogicalExpression argExpr = argRefs.get(0).getValue();
+            if (argExpr.getExpressionTag() != LogicalExpressionTag.VARIABLE) {
+                return false;
+            }
+            VariableReferenceExpression argVarExpr = (VariableReferenceExpression) argExpr;
+            LogicalVariable argVar = argVarExpr.getVariableReference();
+            if (!dataVar.equals(argVar)) {
+                return false;
+            }
+            exprRef.setValue(new VariableReferenceExpression(metaVar));
+            return true;
+        }
+
+        // The user query provides zero parameter for the meta function.
+        if (variableRequired) {
+            throw new AlgebricksException("Cannot resolve to ambiguity on the meta function call --"
+                    + " there are more than once dataset choices!");
+        }
+        exprRef.setValue(new VariableReferenceExpression(metaVar));
+        return true;
+    }
+}
+
+class CompositeExpressionReferenceTransform implements ILogicalExpressionReferenceTransform {
+    private final List<ILogicalExpressionReferenceTransform> transformers;
+
+    public CompositeExpressionReferenceTransform(List<ILogicalExpressionReferenceTransform> transforms) {
+        this.transformers = transforms;
+    }
+
+    @Override
+    public boolean transform(Mutable<ILogicalExpression> expression) throws AlgebricksException {
+        // Tries transfomations one by one.
+        for (ILogicalExpressionReferenceTransform transformer : transformers) {
+            if (transformer.transform(expression)) {
+                return true;
+            }
+        }
+        return false;
+    }
+
+}
diff --git a/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/AccessMethodUtils.java b/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/AccessMethodUtils.java
index f39d190..5cb01eb 100644
--- a/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/AccessMethodUtils.java
+++ b/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/AccessMethodUtils.java
@@ -35,6 +35,7 @@
 import org.apache.asterix.metadata.entities.Dataset;
 import org.apache.asterix.metadata.entities.ExternalDatasetDetails;
 import org.apache.asterix.metadata.entities.Index;
+import org.apache.asterix.metadata.entities.InternalDatasetDetails;
 import org.apache.asterix.metadata.utils.DatasetUtils;
 import org.apache.asterix.om.base.ABoolean;
 import org.apache.asterix.om.base.AInt32;
@@ -82,14 +83,25 @@
  */
 public class AccessMethodUtils {
 
-    public static void appendPrimaryIndexTypes(Dataset dataset, IAType itemType, List<Object> target)
-            throws IOException, AlgebricksException {
+    public static void appendPrimaryIndexTypes(Dataset dataset, IAType itemType, IAType metaItemType,
+            List<Object> target) throws IOException, AlgebricksException {
         ARecordType recordType = (ARecordType) itemType;
+        ARecordType metaRecordType = (ARecordType) metaItemType;
+        List<Integer> keySourceIndicators = ((InternalDatasetDetails) dataset.getDatasetDetails())
+                .getKeySourceIndicator();
         List<List<String>> partitioningKeys = DatasetUtils.getPartitioningKeys(dataset);
+        int index = 0;
         for (List<String> partitioningKey : partitioningKeys) {
-            target.add(recordType.getSubFieldType(partitioningKey));
+            target.add(keySourceIndicators.get(index) == 0 ? recordType.getSubFieldType(partitioningKey)
+                    : metaRecordType.getSubFieldType(partitioningKey));
+            ++index;
         }
+        // Adds data record type.
         target.add(itemType);
+        // Adds metea record type if any.
+        if (dataset.hasMetaPart()) {
+            target.add(metaItemType);
+        }
     }
 
     public static ConstantExpression createStringConstant(String str) {
@@ -172,8 +184,9 @@
                 constantExpressionType);
         for (IOptimizableFuncExpr optFuncExpr : analysisCtx.matchedFuncExprs) {
             //avoid additional optFuncExpressions in case of a join
-            if (optFuncExpr.getFuncExpr().equals(funcExpr))
+            if (optFuncExpr.getFuncExpr().equals(funcExpr)) {
                 return true;
+            }
         }
         analysisCtx.matchedFuncExprs.add(newOptFuncExpr);
         return true;
@@ -196,8 +209,9 @@
                 new LogicalVariable[] { fieldVar1, fieldVar2 }, new ILogicalExpression[0], new IAType[0]);
         for (IOptimizableFuncExpr optFuncExpr : analysisCtx.matchedFuncExprs) {
             //avoid additional optFuncExpressions in case of a join
-            if (optFuncExpr.getFuncExpr().equals(funcExpr))
+            if (optFuncExpr.getFuncExpr().equals(funcExpr)) {
                 return true;
+            }
         }
         analysisCtx.matchedFuncExprs.add(newOptFuncExpr);
         return true;
@@ -228,8 +242,9 @@
     /**
      * Appends the types of the fields produced by the given secondary index to dest.
      */
-    public static void appendSecondaryIndexTypes(Dataset dataset, ARecordType recordType, Index index,
-            boolean primaryKeysOnly, List<Object> dest) throws AlgebricksException {
+    public static void appendSecondaryIndexTypes(Dataset dataset, ARecordType recordType, ARecordType metaRecordType,
+            Index index, boolean primaryKeysOnly, List<Object> dest) throws AlgebricksException {
+        List<Integer> keySourceIndicators = index.getKeyFieldSourceIndicators();
         if (!primaryKeysOnly) {
             switch (index.getIndexType()) {
                 case BTREE:
@@ -237,17 +252,19 @@
                 case SINGLE_PARTITION_NGRAM_INVIX: {
                     for (int i = 0; i < index.getKeyFieldNames().size(); i++) {
                         Pair<IAType, Boolean> keyPairType = Index.getNonNullableOpenFieldType(
-                                index.getKeyFieldTypes().get(i), index.getKeyFieldNames().get(i), recordType);
+                                index.getKeyFieldTypes().get(i), index.getKeyFieldNames().get(i),
+                                keySourceIndicators.get(i) == 0 ? recordType : metaRecordType);
                         dest.add(keyPairType.first);
                     }
                     break;
                 }
                 case RTREE: {
+                    ARecordType targetRecType = keySourceIndicators.get(0) == 0 ? recordType : metaRecordType;
                     Pair<IAType, Boolean> keyPairType = Index.getNonNullableOpenFieldType(
-                            index.getKeyFieldTypes().get(0), index.getKeyFieldNames().get(0), recordType);
+                            index.getKeyFieldTypes().get(0), index.getKeyFieldNames().get(0), targetRecType);
                     IAType keyType = keyPairType.first;
                     IAType nestedKeyType = NonTaggedFormatUtil.getNestedSpatialType(keyType.getTypeTag());
-                    int numKeys = getNumSecondaryKeys(index, recordType);
+                    int numKeys = getNumSecondaryKeys(index, targetRecType);
                     for (int i = 0; i < numKeys; i++) {
                         dest.add(nestedKeyType);
                     }
@@ -269,12 +286,18 @@
             }
         } else {
             List<List<String>> partitioningKeys = DatasetUtils.getPartitioningKeys(dataset);
+            List<Integer> partitioningKeySourceIndicators = ((InternalDatasetDetails) dataset.getDatasetDetails())
+                    .getKeySourceIndicator();
+            int keyIndex = 0;
             for (List<String> partitioningKey : partitioningKeys) {
                 try {
-                    dest.add(recordType.getSubFieldType(partitioningKey));
+                    dest.add(partitioningKeySourceIndicators.get(keyIndex) == 0
+                            ? recordType.getSubFieldType(partitioningKey)
+                            : metaRecordType.getSubFieldType(partitioningKey));
                 } catch (IOException e) {
                     throw new AlgebricksException(e);
                 }
+                ++keyIndex;
             }
         }
     }
@@ -422,9 +445,10 @@
         return indexExprs.get(0).second;
     }
 
-    public static UnnestMapOperator createSecondaryIndexUnnestMap(Dataset dataset, ARecordType recordType, Index index,
-            ILogicalOperator inputOp, AccessMethodJobGenParams jobGenParams, IOptimizationContext context,
-            boolean outputPrimaryKeysOnly, boolean retainInput) throws AlgebricksException {
+    public static UnnestMapOperator createSecondaryIndexUnnestMap(Dataset dataset, ARecordType recordType,
+            ARecordType metaRecordType, Index index, ILogicalOperator inputOp, AccessMethodJobGenParams jobGenParams,
+            IOptimizationContext context, boolean outputPrimaryKeysOnly, boolean retainInput)
+                    throws AlgebricksException {
         // The job gen parameters are transferred to the actual job gen via the UnnestMapOperator's function arguments.
         ArrayList<Mutable<ILogicalExpression>> secondaryIndexFuncArgs = new ArrayList<Mutable<ILogicalExpression>>();
         jobGenParams.writeToFuncArgs(secondaryIndexFuncArgs);
@@ -434,7 +458,8 @@
         // Append output variables/types generated by the secondary-index search (not forwarded from input).
         appendSecondaryIndexOutputVars(dataset, recordType, index, outputPrimaryKeysOnly, context,
                 secondaryIndexUnnestVars);
-        appendSecondaryIndexTypes(dataset, recordType, index, outputPrimaryKeysOnly, secondaryIndexOutputTypes);
+        appendSecondaryIndexTypes(dataset, recordType, metaRecordType, index, outputPrimaryKeysOnly,
+                secondaryIndexOutputTypes);
         // An index search is expressed as an unnest over an index-search function.
         IFunctionInfo secondaryIndexSearch = FunctionUtil.getFunctionInfo(AsterixBuiltinFunctions.INDEX_SEARCH);
         UnnestingFunctionCallExpression secondaryIndexSearchFunc = new UnnestingFunctionCallExpression(
@@ -452,9 +477,9 @@
     }
 
     public static UnnestMapOperator createPrimaryIndexUnnestMap(AbstractDataSourceOperator dataSourceOp,
-            Dataset dataset, ARecordType recordType, ILogicalOperator inputOp, IOptimizationContext context,
-            boolean sortPrimaryKeys, boolean retainInput, boolean retainNull, boolean requiresBroadcast)
-                    throws AlgebricksException {
+            Dataset dataset, ARecordType recordType, ARecordType metaRecordType, ILogicalOperator inputOp,
+            IOptimizationContext context, boolean sortPrimaryKeys, boolean retainInput, boolean retainNull,
+            boolean requiresBroadcast) throws AlgebricksException {
         List<LogicalVariable> primaryKeyVars = AccessMethodUtils.getPrimaryKeyVarsFromSecondaryUnnestMap(dataset,
                 inputOp);
         // Optionally add a sort on the primary-index keys before searching the primary index.
@@ -489,7 +514,7 @@
         // Append output variables/types generated by the primary-index search (not forwarded from input).
         primaryIndexUnnestVars.addAll(dataSourceOp.getVariables());
         try {
-            appendPrimaryIndexTypes(dataset, recordType, primaryIndexOutputTypes);
+            appendPrimaryIndexTypes(dataset, recordType, metaRecordType, primaryIndexOutputTypes);
         } catch (IOException e) {
             throw new AlgebricksException(e);
         }
diff --git a/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/BTreeAccessMethod.java b/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/BTreeAccessMethod.java
index 1bbb13b..9825b9d 100644
--- a/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/BTreeAccessMethod.java
+++ b/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/BTreeAccessMethod.java
@@ -481,7 +481,7 @@
         }
 
         UnnestMapOperator secondaryIndexUnnestOp = AccessMethodUtils.createSecondaryIndexUnnestMap(dataset, recordType,
-                chosenIndex, inputOp, jobGenParams, context, false, retainInput);
+                metaRecordType, chosenIndex, inputOp, jobGenParams, context, false, retainInput);
 
         // Generate the rest of the upstream plan which feeds the search results into the primary index.
         UnnestMapOperator primaryIndexUnnestOp = null;
@@ -494,7 +494,7 @@
             return externalDataAccessOp;
         } else if (!isPrimaryIndex) {
             primaryIndexUnnestOp = AccessMethodUtils.createPrimaryIndexUnnestMap(dataSourceOp, dataset, recordType,
-                    secondaryIndexUnnestOp, context, true, retainInput, retainNull, false);
+                    metaRecordType, secondaryIndexUnnestOp, context, true, retainInput, retainNull, false);
 
             // Adds equivalence classes --- one equivalent class between a primary key
             // variable and a record field-access expression.
@@ -503,7 +503,7 @@
         } else {
             List<Object> primaryIndexOutputTypes = new ArrayList<Object>();
             try {
-                AccessMethodUtils.appendPrimaryIndexTypes(dataset, recordType, primaryIndexOutputTypes);
+                AccessMethodUtils.appendPrimaryIndexTypes(dataset, recordType, metaRecordType, primaryIndexOutputTypes);
             } catch (IOException e) {
                 throw new AlgebricksException(e);
             }
diff --git a/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/InvertedIndexAccessMethod.java b/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/InvertedIndexAccessMethod.java
index 824fe7a..70d7088 100644
--- a/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/InvertedIndexAccessMethod.java
+++ b/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/InvertedIndexAccessMethod.java
@@ -66,7 +66,6 @@
 import org.apache.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator.ExecutionMode;
 import org.apache.hyracks.algebricks.core.algebra.operators.logical.AssignOperator;
 import org.apache.hyracks.algebricks.core.algebra.operators.logical.DataSourceScanOperator;
-import org.apache.hyracks.algebricks.core.algebra.operators.logical.EmptyTupleSourceOperator;
 import org.apache.hyracks.algebricks.core.algebra.operators.logical.InnerJoinOperator;
 import org.apache.hyracks.algebricks.core.algebra.operators.logical.ReplicateOperator;
 import org.apache.hyracks.algebricks.core.algebra.operators.logical.SelectOperator;
@@ -364,13 +363,13 @@
     @Override
     public ILogicalOperator createSecondaryToPrimaryPlan(Mutable<ILogicalExpression> conditionRef,
             OptimizableOperatorSubTree indexSubTree, OptimizableOperatorSubTree probeSubTree, Index chosenIndex,
-            AccessMethodAnalysisContext analysisCtx,
-            boolean retainInput, boolean retainNull, boolean requiresBroadcast, IOptimizationContext context)
-                    throws AlgebricksException {
+            AccessMethodAnalysisContext analysisCtx, boolean retainInput, boolean retainNull, boolean requiresBroadcast,
+            IOptimizationContext context) throws AlgebricksException {
 
         IOptimizableFuncExpr optFuncExpr = AccessMethodUtils.chooseFirstOptFuncExpr(chosenIndex, analysisCtx);
         Dataset dataset = indexSubTree.dataset;
         ARecordType recordType = indexSubTree.recordType;
+        ARecordType metaRecordType = indexSubTree.metaRecordType;
         // we made sure indexSubTree has datasource scan
         DataSourceScanOperator dataSourceScan = (DataSourceScanOperator) indexSubTree.dataSourceRef.getValue();
 
@@ -407,11 +406,11 @@
         }
         jobGenParams.setKeyVarList(keyVarList);
         UnnestMapOperator secondaryIndexUnnestOp = AccessMethodUtils.createSecondaryIndexUnnestMap(dataset, recordType,
-                chosenIndex, inputOp, jobGenParams, context, true, retainInput);
+                metaRecordType, chosenIndex, inputOp, jobGenParams, context, true, retainInput);
 
         // Generate the rest of the upstream plan which feeds the search results into the primary index.
         UnnestMapOperator primaryIndexUnnestOp = AccessMethodUtils.createPrimaryIndexUnnestMap(dataSourceScan, dataset,
-                recordType, secondaryIndexUnnestOp, context, true, retainInput, retainNull, false);
+                recordType, metaRecordType, secondaryIndexUnnestOp, context, true, retainInput, retainNull, false);
 
         return primaryIndexUnnestOp;
     }
@@ -436,8 +435,8 @@
     public boolean applySelectPlanTransformation(Mutable<ILogicalOperator> selectRef,
             OptimizableOperatorSubTree subTree, Index chosenIndex, AccessMethodAnalysisContext analysisCtx,
             IOptimizationContext context) throws AlgebricksException {
-        ILogicalOperator indexPlanRootOp = createSecondaryToPrimaryPlan(null, subTree, null, chosenIndex, analysisCtx, false,
-                false, false, context);
+        ILogicalOperator indexPlanRootOp = createSecondaryToPrimaryPlan(null, subTree, null, chosenIndex, analysisCtx,
+                false, false, false, context);
         // Replace the datasource scan with the new plan rooted at primaryIndexUnnestMap.
         subTree.dataSourceRef.setValue(indexPlanRootOp);
         return true;
diff --git a/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/RTreeAccessMethod.java b/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/RTreeAccessMethod.java
index 3c925e2..111fcf4 100644
--- a/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/RTreeAccessMethod.java
+++ b/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/RTreeAccessMethod.java
@@ -179,6 +179,7 @@
         IOptimizableFuncExpr optFuncExpr = AccessMethodUtils.chooseFirstOptFuncExpr(chosenIndex, analysisCtx);
         Dataset dataset = indexSubTree.dataset;
         ARecordType recordType = indexSubTree.recordType;
+        ARecordType metaRecordType = indexSubTree.metaRecordType;
 
         int optFieldIdx = AccessMethodUtils.chooseFirstOptFuncVar(chosenIndex, analysisCtx);
         Pair<IAType, Boolean> keyPairType = Index.getNonNullableOpenFieldType(optFuncExpr.getFieldType(optFieldIdx),
@@ -239,7 +240,7 @@
         }
 
         UnnestMapOperator secondaryIndexUnnestOp = AccessMethodUtils.createSecondaryIndexUnnestMap(dataset, recordType,
-                chosenIndex, assignSearchKeys, jobGenParams, context, false, retainInput);
+                metaRecordType, chosenIndex, assignSearchKeys, jobGenParams, context, false, retainInput);
 
         // Generate the rest of the upstream plan which feeds the search results into the primary index.
         if (dataset.getDatasetType() == DatasetType.EXTERNAL) {
@@ -248,8 +249,8 @@
             return externalDataAccessOp;
         } else {
             UnnestMapOperator primaryIndexUnnestOp = AccessMethodUtils.createPrimaryIndexUnnestMap(dataSourceOp,
-                    dataset, recordType, secondaryIndexUnnestOp, context, true, retainInput, false, false);
-
+                    dataset, recordType, metaRecordType, secondaryIndexUnnestOp, context, true, retainInput, false,
+                    false);
             return primaryIndexUnnestOp;
         }
     }
diff --git a/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/util/EquivalenceClassUtils.java b/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/util/EquivalenceClassUtils.java
index e85fecf..242e15b 100644
--- a/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/util/EquivalenceClassUtils.java
+++ b/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/util/EquivalenceClassUtils.java
@@ -103,7 +103,7 @@
         for (int pkIndex = 0; pkIndex < primaryKey.size(); ++pkIndex) {
             LogicalVariable referredRecordVar = recordVar;
             String pkFieldName = primaryKey.get(pkIndex).get(0);
-            int source = keySourceIndicators == null ? 0 : keySourceIndicators.get(pkIndex);
+            int source = keySourceIndicators.get(pkIndex);
             Integer fieldIndexInRecord;
             if (source == 0) {
                 // The field is from the main record.
diff --git a/asterix-algebra/src/main/java/org/apache/asterix/translator/LangExpressionToPlanTranslator.java b/asterix-algebra/src/main/java/org/apache/asterix/translator/LangExpressionToPlanTranslator.java
index c635125..28de56b 100644
--- a/asterix-algebra/src/main/java/org/apache/asterix/translator/LangExpressionToPlanTranslator.java
+++ b/asterix-algebra/src/main/java/org/apache/asterix/translator/LangExpressionToPlanTranslator.java
@@ -457,8 +457,8 @@
         IAType metaItemType = metadataProvider.findType(dataset.getMetaItemTypeDataverseName(),
                 dataset.getMetaItemTypeName());
         DatasetDataSource dataSource = new DatasetDataSource(sourceId, dataset.getDataverseName(),
-                dataset.getDatasetName(), itemType, metaItemType, AqlDataSourceType.INTERNAL_DATASET);
-
+                dataset.getDatasetName(), itemType, metaItemType, AqlDataSourceType.INTERNAL_DATASET,
+                dataset.getDatasetDetails());
         return dataSource;
     }
 
diff --git a/asterix-algebra/src/main/java/org/apache/asterix/translator/util/ValidateUtil.java b/asterix-algebra/src/main/java/org/apache/asterix/translator/util/ValidateUtil.java
index 95a8802..9020e27 100644
--- a/asterix-algebra/src/main/java/org/apache/asterix/translator/util/ValidateUtil.java
+++ b/asterix-algebra/src/main/java/org/apache/asterix/translator/util/ValidateUtil.java
@@ -151,12 +151,14 @@
      * @throws AsterixException
      *             (if the validation failed), IOException
      */
-    public static void validateKeyFields(ARecordType recType, List<List<String>> keyFieldNames,
-            List<IAType> keyFieldTypes, IndexType indexType) throws AsterixException, IOException {
+    public static void validateKeyFields(ARecordType recType, ARecordType metaRecType, List<List<String>> keyFieldNames,
+            List<Integer> keySourceIndicators, List<IAType> keyFieldTypes, IndexType indexType)
+                    throws AsterixException, IOException {
         int pos = 0;
         boolean openFieldCompositeIdx = false;
         for (List<String> fieldName : keyFieldNames) {
-            IAType fieldType = recType.getSubFieldType(fieldName);
+            IAType fieldType = keySourceIndicators.get(pos) == 0 ? recType.getSubFieldType(fieldName)
+                    : metaRecType.getSubFieldType(fieldName);
             if (fieldType == null) {
                 fieldType = keyFieldTypes.get(pos);
                 if (keyFieldTypes.get(pos) == BuiltinType.ANULL) {
diff --git a/asterix-app/src/main/java/org/apache/asterix/aql/translator/QueryTranslator.java b/asterix-app/src/main/java/org/apache/asterix/aql/translator/QueryTranslator.java
index 75f325b..d2a0d03 100644
--- a/asterix-app/src/main/java/org/apache/asterix/aql/translator/QueryTranslator.java
+++ b/asterix-app/src/main/java/org/apache/asterix/aql/translator/QueryTranslator.java
@@ -811,6 +811,7 @@
         CreateIndexStatement stmtCreateIndex = (CreateIndexStatement) stmt;
         String dataverseName = getActiveDataverse(stmtCreateIndex.getDataverseName());
         String datasetName = stmtCreateIndex.getDatasetName().getValue();
+        List<Integer> keySourceIndicators = stmtCreateIndex.getFieldSourceIndicators();
 
         MetadataTransactionContext mdTxnCtx = MetadataManager.INSTANCE.beginTransaction();
         boolean bActiveTxn = true;
@@ -838,19 +839,23 @@
             indexName = stmtCreateIndex.getIndexName().getValue();
             Index idx = MetadataManager.INSTANCE.getIndex(metadataProvider.getMetadataTxnContext(), dataverseName,
                     datasetName, indexName);
-
-            String itemTypeName = ds.getItemTypeName();
             Datatype dt = MetadataManager.INSTANCE.getDatatype(metadataProvider.getMetadataTxnContext(),
-                    ds.getItemTypeDataverseName(), itemTypeName);
-            IAType itemType = dt.getDatatype();
-            ARecordType aRecordType = (ARecordType) itemType;
+                    ds.getItemTypeDataverseName(), ds.getItemTypeName());
+            ARecordType aRecordType = (ARecordType) dt.getDatatype();
+            ARecordType metaRecordType = null;
+            if (ds.hasMetaPart()) {
+                Datatype metaDt = MetadataManager.INSTANCE.getDatatype(metadataProvider.getMetadataTxnContext(),
+                        ds.getMetaItemTypeDataverseName(), ds.getMetaItemTypeName());
+                metaRecordType = (ARecordType) metaDt.getDatatype();
+            }
 
             List<List<String>> indexFields = new ArrayList<List<String>>();
             List<IAType> indexFieldTypes = new ArrayList<IAType>();
+            int keyIndex = 0;
             for (Pair<List<String>, TypeExpression> fieldExpr : stmtCreateIndex.getFieldExprs()) {
                 IAType fieldType = null;
-                boolean isOpen = aRecordType.isOpen();
-                ARecordType subType = aRecordType;
+                ARecordType subType = keySourceIndicators.get(keyIndex) == 0 ? aRecordType : metaRecordType;
+                boolean isOpen = subType.isOpen();
                 int i = 0;
                 if (fieldExpr.first.size() > 1 && !isOpen) {
                     for (; i < fieldExpr.first.size() - 1;) {
@@ -885,9 +890,11 @@
 
                 indexFields.add(fieldExpr.first);
                 indexFieldTypes.add(fieldType);
+                ++keyIndex;
             }
 
-            ValidateUtil.validateKeyFields(aRecordType, indexFields, indexFieldTypes, stmtCreateIndex.getIndexType());
+            ValidateUtil.validateKeyFields(aRecordType, metaRecordType, indexFields, keySourceIndicators,
+                    indexFieldTypes, stmtCreateIndex.getIndexType());
 
             if (idx != null) {
                 if (stmtCreateIndex.getIfNotExists()) {
@@ -995,8 +1002,8 @@
 
             //#. add a new index with PendingAddOp
             Index index = new Index(dataverseName, datasetName, indexName, stmtCreateIndex.getIndexType(), indexFields,
-                    stmtCreateIndex.getFieldSourceIndicators(), indexFieldTypes, stmtCreateIndex.getGramLength(),
-                    stmtCreateIndex.isEnforced(), false, IMetadataEntity.PENDING_ADD_OP);
+                    keySourceIndicators, indexFieldTypes, stmtCreateIndex.getGramLength(), stmtCreateIndex.isEnforced(),
+                    false, IMetadataEntity.PENDING_ADD_OP);
             MetadataManager.INSTANCE.addIndex(metadataProvider.getMetadataTxnContext(), index);
 
             ARecordType enforcedType = null;
diff --git a/asterix-app/src/test/resources/runtimets/queries/meta/query_dataset_with_meta_primary_index-1/query_dataset_with_meta_primary_index-1.1.ddl.aql b/asterix-app/src/test/resources/runtimets/queries/meta/query_dataset_with_meta_primary_index-1/query_dataset_with_meta_primary_index-1.1.ddl.aql
new file mode 100644
index 0000000..a341b29
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/meta/query_dataset_with_meta_primary_index-1/query_dataset_with_meta_primary_index-1.1.ddl.aql
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+drop dataverse test if exists;
+create dataverse test;
+use dataverse test;
+
+create type EmptyType as open {
+  "key":int32
+}
+
+create type LineType as open {
+  id:int32,
+  text: string
+}
+
+create dataset Book(LineType) with meta(EmptyType)
+primary key meta()."key";
diff --git a/asterix-app/src/test/resources/runtimets/queries/meta/query_dataset_with_meta_primary_index-1/query_dataset_with_meta_primary_index-1.2.update.aql b/asterix-app/src/test/resources/runtimets/queries/meta/query_dataset_with_meta_primary_index-1/query_dataset_with_meta_primary_index-1.2.update.aql
new file mode 100644
index 0000000..bd244d0
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/meta/query_dataset_with_meta_primary_index-1/query_dataset_with_meta_primary_index-1.2.update.aql
@@ -0,0 +1,19 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
diff --git a/asterix-app/src/test/resources/runtimets/queries/meta/query_dataset_with_meta_primary_index-1/query_dataset_with_meta_primary_index-1.3.query.aql b/asterix-app/src/test/resources/runtimets/queries/meta/query_dataset_with_meta_primary_index-1/query_dataset_with_meta_primary_index-1.3.query.aql
new file mode 100644
index 0000000..246046d
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/meta/query_dataset_with_meta_primary_index-1/query_dataset_with_meta_primary_index-1.3.query.aql
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+use dataverse test;
+
+for $x in dataset Book
+where meta($x)."key" >10
+return $x;
+
diff --git a/asterix-app/src/test/resources/runtimets/queries/meta/query_dataset_with_meta_primary_index-1/query_dataset_with_meta_primary_index-1.4.ddl.aql b/asterix-app/src/test/resources/runtimets/queries/meta/query_dataset_with_meta_primary_index-1/query_dataset_with_meta_primary_index-1.4.ddl.aql
new file mode 100644
index 0000000..dc10acd
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/meta/query_dataset_with_meta_primary_index-1/query_dataset_with_meta_primary_index-1.4.ddl.aql
@@ -0,0 +1,20 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+drop dataverse test if exists;
diff --git a/asterix-app/src/test/resources/runtimets/queries/meta/query_dataset_with_meta_primary_index-2/query_dataset_with_meta_primary_index-2.1.ddl.aql b/asterix-app/src/test/resources/runtimets/queries/meta/query_dataset_with_meta_primary_index-2/query_dataset_with_meta_primary_index-2.1.ddl.aql
new file mode 100644
index 0000000..a341b29
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/meta/query_dataset_with_meta_primary_index-2/query_dataset_with_meta_primary_index-2.1.ddl.aql
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+drop dataverse test if exists;
+create dataverse test;
+use dataverse test;
+
+create type EmptyType as open {
+  "key":int32
+}
+
+create type LineType as open {
+  id:int32,
+  text: string
+}
+
+create dataset Book(LineType) with meta(EmptyType)
+primary key meta()."key";
diff --git a/asterix-app/src/test/resources/runtimets/queries/meta/query_dataset_with_meta_primary_index-2/query_dataset_with_meta_primary_index-2.2.update.aql b/asterix-app/src/test/resources/runtimets/queries/meta/query_dataset_with_meta_primary_index-2/query_dataset_with_meta_primary_index-2.2.update.aql
new file mode 100644
index 0000000..bd244d0
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/meta/query_dataset_with_meta_primary_index-2/query_dataset_with_meta_primary_index-2.2.update.aql
@@ -0,0 +1,19 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
diff --git a/asterix-app/src/test/resources/runtimets/queries/meta/query_dataset_with_meta_primary_index-2/query_dataset_with_meta_primary_index-2.3.query.aql b/asterix-app/src/test/resources/runtimets/queries/meta/query_dataset_with_meta_primary_index-2/query_dataset_with_meta_primary_index-2.3.query.aql
new file mode 100644
index 0000000..6b8e4a9
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/meta/query_dataset_with_meta_primary_index-2/query_dataset_with_meta_primary_index-2.3.query.aql
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+use dataverse test;
+
+for $x in dataset Book
+where meta()."key" >10
+return $x;
+
diff --git a/asterix-app/src/test/resources/runtimets/queries/meta/query_dataset_with_meta_primary_index-2/query_dataset_with_meta_primary_index-2.4.ddl.aql b/asterix-app/src/test/resources/runtimets/queries/meta/query_dataset_with_meta_primary_index-2/query_dataset_with_meta_primary_index-2.4.ddl.aql
new file mode 100644
index 0000000..dc10acd
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/meta/query_dataset_with_meta_primary_index-2/query_dataset_with_meta_primary_index-2.4.ddl.aql
@@ -0,0 +1,20 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+drop dataverse test if exists;
diff --git a/asterix-app/src/test/resources/runtimets/results/meta/query_dataset_with_meta_primary_index-1/query_dataset_with_meta_primary_index-1.1.adm b/asterix-app/src/test/resources/runtimets/results/meta/query_dataset_with_meta_primary_index-1/query_dataset_with_meta_primary_index-1.1.adm
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/results/meta/query_dataset_with_meta_primary_index-1/query_dataset_with_meta_primary_index-1.1.adm
diff --git a/asterix-app/src/test/resources/runtimets/results/meta/query_dataset_with_meta_primary_index-2/query_dataset_with_meta_primary_index-2.1.adm b/asterix-app/src/test/resources/runtimets/results/meta/query_dataset_with_meta_primary_index-2/query_dataset_with_meta_primary_index-2.1.adm
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/results/meta/query_dataset_with_meta_primary_index-2/query_dataset_with_meta_primary_index-2.1.adm
diff --git a/asterix-app/src/test/resources/runtimets/testsuite.xml b/asterix-app/src/test/resources/runtimets/testsuite.xml
index 05f131e..fd4f32f 100644
--- a/asterix-app/src/test/resources/runtimets/testsuite.xml
+++ b/asterix-app/src/test/resources/runtimets/testsuite.xml
@@ -2916,6 +2916,16 @@
                 <output-dir compare="Text">query_dataset_with_meta-2</output-dir>
             </compilation-unit>
         </test-case>
+        <test-case FilePath="meta">
+            <compilation-unit name="query_dataset_with_meta_primary_index-1">
+                <output-dir compare="Text">query_dataset_with_meta_primary_index-1</output-dir>
+            </compilation-unit>
+        </test-case>
+        <test-case FilePath="meta">
+            <compilation-unit name="query_dataset_with_meta_primary_index-2">
+                <output-dir compare="Text">query_dataset_with_meta_primary_index-2</output-dir>
+            </compilation-unit>
+        </test-case>
     </test-group>
     <test-group name="misc">
         <test-case FilePath="misc">
diff --git a/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/AqlMetadataProvider.java b/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/AqlMetadataProvider.java
index 7ce7ba3..c7c1d1e 100644
--- a/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/AqlMetadataProvider.java
+++ b/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/AqlMetadataProvider.java
@@ -989,7 +989,7 @@
         AqlDataSourceType datasourceType = dataset.getDatasetType().equals(DatasetType.EXTERNAL)
                 ? AqlDataSourceType.EXTERNAL_DATASET : AqlDataSourceType.INTERNAL_DATASET;
         return new DatasetDataSource(aqlId, aqlId.getDataverseName(), aqlId.getDatasourceName(), itemType, metaItemType,
-                datasourceType);
+                datasourceType, dataset.getDatasetDetails());
     }
 
     @Override
diff --git a/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/DatasetDataSource.java b/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/DatasetDataSource.java
index 8b5956f..8e63478 100644
--- a/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/DatasetDataSource.java
+++ b/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/DatasetDataSource.java
@@ -21,9 +21,11 @@
 import java.io.IOException;
 import java.util.List;
 
+import org.apache.asterix.metadata.IDatasetDetails;
 import org.apache.asterix.metadata.MetadataManager;
 import org.apache.asterix.metadata.MetadataTransactionContext;
 import org.apache.asterix.metadata.entities.Dataset;
+import org.apache.asterix.metadata.entities.InternalDatasetDetails;
 import org.apache.asterix.metadata.utils.DatasetUtils;
 import org.apache.asterix.om.types.ARecordType;
 import org.apache.asterix.om.types.IAType;
@@ -36,7 +38,8 @@
     private Dataset dataset;
 
     public DatasetDataSource(AqlSourceId id, String datasourceDataverse, String datasourceName, IAType itemType,
-            IAType metaItemType, AqlDataSourceType datasourceType) throws AlgebricksException {
+            IAType metaItemType, AqlDataSourceType datasourceType, IDatasetDetails datasetDetails)
+                    throws AlgebricksException {
         super(id, itemType, metaItemType, datasourceType);
         MetadataTransactionContext ctx = null;
         try {
@@ -49,7 +52,7 @@
             MetadataManager.INSTANCE.commitTransaction(ctx);
             switch (dataset.getDatasetType()) {
                 case INTERNAL:
-                    initInternalDataset(itemType, metaItemType);
+                    initInternalDataset(itemType, metaItemType, datasetDetails);
                     break;
                 case EXTERNAL:
                     initExternalDataset(itemType);
@@ -74,13 +77,21 @@
         return dataset;
     }
 
-    private void initInternalDataset(IAType itemType, IAType metaItemType) throws IOException, AlgebricksException {
+    private void initInternalDataset(IAType itemType, IAType metaItemType, IDatasetDetails datasetDetails)
+            throws IOException, AlgebricksException {
+        InternalDatasetDetails internalDatasetDetails = (InternalDatasetDetails) datasetDetails;
         List<List<String>> partitioningKeys = DatasetUtils.getPartitioningKeys(dataset);
         ARecordType recordType = (ARecordType) itemType;
+        ARecordType metaRecordType = (ARecordType) metaItemType;
         int n = partitioningKeys.size();
         schemaTypes = metaItemType == null ? new IAType[n + 1] : new IAType[n + 2];
+        List<Integer> keySourceIndicators = internalDatasetDetails.getKeySourceIndicator();
         for (int i = 0; i < n; i++) {
-            schemaTypes[i] = recordType.getSubFieldType(partitioningKeys.get(i));
+            if (keySourceIndicators.get(i) == 0) {
+                schemaTypes[i] = recordType.getSubFieldType(partitioningKeys.get(i));
+            } else {
+                schemaTypes[i] = metaRecordType.getSubFieldType(partitioningKeys.get(i));
+            }
         }
         schemaTypes[n] = itemType;
         if (metaItemType != null) {
diff --git a/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/InternalDatasetDetails.java b/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/InternalDatasetDetails.java
index 6971037..b0c6c51 100644
--- a/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/InternalDatasetDetails.java
+++ b/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/InternalDatasetDetails.java
@@ -20,6 +20,7 @@
 package org.apache.asterix.metadata.entities;
 
 import java.io.DataOutput;
+import java.util.ArrayList;
 import java.util.List;
 
 import org.apache.asterix.builders.IARecordBuilder;
@@ -75,6 +76,13 @@
         this.partitioningStrategy = partitioningStrategy;
         this.partitioningKeys = partitioningKey;
         this.primaryKeys = primaryKey;
+        if (keyFieldIndicators == null) {
+            // Create a dummy list.
+            keyFieldIndicators = new ArrayList<>();
+            for (int index = 0; index < partitioningKey.size(); ++index) {
+                keyFieldIndicators.add(0);
+            }
+        }
         this.keySourceIndicators = keyFieldIndicators;
         this.primaryKeyTypes = primaryKeyType;
         this.autogenerated = autogenerated;
diff --git a/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/IndexTupleTranslator.java b/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/IndexTupleTranslator.java
index f310a03..241a68d 100644
--- a/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/IndexTupleTranslator.java
+++ b/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/IndexTupleTranslator.java
@@ -142,18 +142,7 @@
             IAType fieldType = AsterixBuiltinTypeMap.getTypeFromTypeName(metadataNode, jobId, dvName, typeName, false);
             searchKeyType.add(fieldType);
         }
-        // index key type information is not persisted, thus we extract type information from the record metadata
-        if (searchKeyType.isEmpty()) {
-            Dataset dSet = metadataNode.getDataset(jobId, dvName, dsName);
-            String datatypeName = dSet.getItemTypeName();
-            String datatypeDataverseName = dSet.getItemTypeDataverseName();
-            ARecordType recordDt = (ARecordType) metadataNode.getDatatype(jobId, datatypeDataverseName, datatypeName)
-                    .getDatatype();
-            for (int i = 0; i < searchKey.size(); i++) {
-                IAType fieldType = recordDt.getSubFieldType(searchKey.get(i));
-                searchKeyType.add(fieldType);
-            }
-        }
+
         int isEnforcedFieldPos = rec.getType().getFieldIndex(INDEX_ISENFORCED_FIELD_NAME);
         Boolean isEnforcingKeys = false;
         if (isEnforcedFieldPos > 0) {
@@ -183,6 +172,27 @@
                 keyFieldSourceIndicator.add(0);
             }
         }
+
+        // index key type information is not persisted, thus we extract type information from the record metadata
+        if (searchKeyType.isEmpty()) {
+            Dataset dSet = metadataNode.getDataset(jobId, dvName, dsName);
+            String datatypeName = dSet.getItemTypeName();
+            String datatypeDataverseName = dSet.getItemTypeDataverseName();
+            ARecordType recordDt = (ARecordType) metadataNode.getDatatype(jobId, datatypeDataverseName, datatypeName)
+                    .getDatatype();
+            String metatypeName = dSet.getMetaItemTypeName();
+            String metatypeDataverseName = dSet.getMetaItemTypeDataverseName();
+            ARecordType metaDt = null;
+            if (metatypeName != null && metatypeDataverseName != null) {
+                metaDt = (ARecordType) metadataNode.getDatatype(jobId, metatypeDataverseName, metatypeName)
+                        .getDatatype();
+            }
+            for (int i = 0; i < searchKey.size(); i++) {
+                IAType fieldType = keyFieldSourceIndicator.get(i) == 0 ? recordDt.getSubFieldType(searchKey.get(i))
+                        : metaDt.getSubFieldType(searchKey.get(i));
+                searchKeyType.add(fieldType);
+            }
+        }
         return new Index(dvName, dsName, indexName, indexStructure, searchKey, keyFieldSourceIndicator, searchKeyType,
                 gramLength, isEnforcingKeys, isPrimaryIndex, pendingOp);
     }
diff --git a/asterix-om/src/main/java/org/apache/asterix/om/functions/AsterixBuiltinFunctions.java b/asterix-om/src/main/java/org/apache/asterix/om/functions/AsterixBuiltinFunctions.java
index 997dccd..9f8d5d7 100644
--- a/asterix-om/src/main/java/org/apache/asterix/om/functions/AsterixBuiltinFunctions.java
+++ b/asterix-om/src/main/java/org/apache/asterix/om/functions/AsterixBuiltinFunctions.java
@@ -90,6 +90,7 @@
 import org.apache.asterix.om.typecomputer.impl.OptionalATemporalInstanceTypeComputer;
 import org.apache.asterix.om.typecomputer.impl.OptionalATimeTypeComputer;
 import org.apache.asterix.om.typecomputer.impl.OptionalAYearMonthDurationTypeComputer;
+import org.apache.asterix.om.typecomputer.impl.OptionalOpenARecordTypeComputer;
 import org.apache.asterix.om.typecomputer.impl.OrderedListConstructorResultType;
 import org.apache.asterix.om.typecomputer.impl.OrderedListOfAInt32TypeComputer;
 import org.apache.asterix.om.typecomputer.impl.OrderedListOfAInt64TypeComputer;
@@ -723,6 +724,9 @@
     public static final FunctionIdentifier EXTERNAL_LOOKUP = new FunctionIdentifier(FunctionConstants.ASTERIX_NS,
             "external-lookup", FunctionIdentifier.VARARGS);
 
+    public static final FunctionIdentifier META = new FunctionIdentifier(FunctionConstants.ASTERIX_NS, "meta",
+            FunctionIdentifier.VARARGS);
+
     public static IFunctionInfo getAsterixFunctionInfo(FunctionIdentifier fid) {
         return registeredFunctions.get(fid);
     }
@@ -1028,6 +1032,9 @@
         addFunction(INTERVAL_CONSTRUCTOR_START_FROM_DATETIME, OptionalAIntervalTypeComputer.INSTANCE, true);
         addFunction(INTERVAL_CONSTRUCTOR_START_FROM_TIME, OptionalAIntervalTypeComputer.INSTANCE, true);
 
+        // meta() function
+        addFunction(META, OptionalOpenARecordTypeComputer.INSTANCE, true);
+
         addPrivateFunction(COLLECTION_TO_SEQUENCE, CollectionToSequenceTypeComputer.INSTANCE, true);
 
         // external lookup
diff --git a/asterix-om/src/main/java/org/apache/asterix/om/typecomputer/impl/OptionalOpenARecordTypeComputer.java b/asterix-om/src/main/java/org/apache/asterix/om/typecomputer/impl/OptionalOpenARecordTypeComputer.java
new file mode 100644
index 0000000..b630f44
--- /dev/null
+++ b/asterix-om/src/main/java/org/apache/asterix/om/typecomputer/impl/OptionalOpenARecordTypeComputer.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.om.typecomputer.impl;
+
+import org.apache.asterix.om.pointables.base.DefaultOpenFieldType;
+import org.apache.asterix.om.typecomputer.base.IResultTypeComputer;
+import org.apache.asterix.om.types.AUnionType;
+import org.apache.asterix.om.types.IAType;
+import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
+import org.apache.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import org.apache.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
+import org.apache.hyracks.algebricks.core.algebra.metadata.IMetadataProvider;
+
+public class OptionalOpenARecordTypeComputer implements IResultTypeComputer {
+
+    public static final OptionalOpenARecordTypeComputer INSTANCE = new OptionalOpenARecordTypeComputer();
+
+    private OptionalOpenARecordTypeComputer() {
+    }
+
+    @Override
+    public IAType computeType(ILogicalExpression expression, IVariableTypeEnvironment env,
+            IMetadataProvider<?, ?> metadataProvider) throws AlgebricksException {
+        return AUnionType.createNullableType(DefaultOpenFieldType.NESTED_OPEN_RECORD_TYPE, "OptionalOpenRecord");
+    }
+
+}

-- 
To view, visit https://asterix-gerrit.ics.uci.edu/685
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings

Gerrit-MessageType: newchange
Gerrit-Change-Id: Ie65417b6baf209ca0ab413cfa4a5f7fc5156ca63
Gerrit-PatchSet: 1
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Yingyi Bu <buyingyi@gmail.com>

Mime
View raw message