asterixdb-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From amo...@apache.org
Subject [5/5] asterixdb git commit: Fix Indexing on Open fields and Meta fields
Date Wed, 22 Jun 2016 21:05:36 GMT
Fix Indexing on Open fields and Meta fields

This change fix the following cases:
1. Build a secondary index on a meta field: success
2. Build an open index on a meta field: failure
3. Build a secondary index on an open field in record part: success

Testing ingestion and querying are working correctly for these cases.

Change-Id: I6195149940f150250a65f2515e9ac9d6de2a33f9
Reviewed-on: https://asterix-gerrit.ics.uci.edu/930
Tested-by: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Reviewed-by: Yingyi Bu <buyingyi@gmail.com>


Project: http://git-wip-us.apache.org/repos/asf/asterixdb/repo
Commit: http://git-wip-us.apache.org/repos/asf/asterixdb/commit/dfa3c117
Tree: http://git-wip-us.apache.org/repos/asf/asterixdb/tree/dfa3c117
Diff: http://git-wip-us.apache.org/repos/asf/asterixdb/diff/dfa3c117

Branch: refs/heads/master
Commit: dfa3c117407f96d4c86ae3da4508aea4e0ce8a94
Parents: 64c021e
Author: Abdullah Alamoudi <bamousaa@gmail.com>
Authored: Tue Jun 21 11:18:50 2016 +0300
Committer: abdullah alamoudi <bamousaa@gmail.com>
Committed: Wed Jun 22 14:04:11 2016 -0700

----------------------------------------------------------------------
 ...IntroduceSecondaryIndexInsertDeleteRule.java |  84 +++--
 .../am/AbstractIntroduceAccessMethodRule.java   | 186 ++++++----
 .../LangExpressionToPlanTranslator.java         | 369 +++++++++----------
 .../external/ExternalIndexingOperations.java    |  53 ++-
 .../asterix/aql/translator/QueryTranslator.java |  66 ++--
 .../apache/asterix/file/DatasetOperations.java  |   4 +-
 .../apache/asterix/file/IndexOperations.java    |  42 ++-
 .../file/SecondaryBTreeOperationsHelper.java    |  33 +-
 .../file/SecondaryIndexOperationsHelper.java    |  49 ++-
 .../SecondaryInvertedIndexOperationsHelper.java |  30 +-
 .../file/SecondaryRTreeOperationsHelper.java    |   4 +-
 .../meta/secondary_index-lojoin_with_meta-1.aql |  51 +++
 .../meta/secondary_index-nljoin_with_meta-1.aql |  40 ++
 .../meta/secondary_index-nljoin_with_meta-2.aql |  46 +++
 .../meta/secondary_index-nljoin_with_meta-3.aql |  40 ++
 .../meta/secondary_index-nljoin_with_meta-4.aql |  43 +++
 .../meta/secondary_index_with_meta-1.aql        |  39 ++
 .../meta/secondary_index_with_meta-2.aql        |  39 ++
 .../secondary_index-lojoin_with_meta-1.plan     |  47 +++
 .../secondary_index-nljoin_with_meta-1.plan     |  23 ++
 .../secondary_index-nljoin_with_meta-2.plan     |  23 ++
 .../secondary_index-nljoin_with_meta-3.plan     |  23 ++
 .../secondary_index-nljoin_with_meta-4.plan     |  23 ++
 .../meta/secondary_index_with_meta-1.plan       |  16 +
 .../meta/secondary_index_with_meta-2.plan       |  15 +
 ...-feed-with-meta-open-index-in-meta.1.ddl.aql |  59 +++
 ...-feed-with-meta-open-index-in-meta.2.ddl.aql |  26 ++
 ...meta-pk-in-meta-index-after-ingest.1.ddl.aql |  57 +++
 ...a-pk-in-meta-index-after-ingest.2.update.aql |  28 ++
 ...meta-pk-in-meta-index-after-ingest.3.ddl.aql |  27 ++
 ...ta-pk-in-meta-index-after-ingest.4.query.aql |  31 ++
 ...meta-pk-in-meta-index-after-ingest.5.ddl.aql |  25 ++
 ...with-meta-pk-in-meta-index-in-meta.1.ddl.aql |  59 +++
 ...h-meta-pk-in-meta-index-in-meta.2.update.aql |  28 ++
 ...th-meta-pk-in-meta-index-in-meta.3.query.aql |  31 ++
 ...with-meta-pk-in-meta-index-in-meta.4.ddl.aql |  25 ++
 ...ta-index-with-missing-after-ingest.1.ddl.aql |  57 +++
 ...index-with-missing-after-ingest.2.update.aql |  28 ++
 ...ta-index-with-missing-after-ingest.3.ddl.aql |  27 ++
 ...-index-with-missing-after-ingest.4.query.aql |  31 ++
 ...ta-index-with-missing-after-ingest.5.ddl.aql |  25 ++
 ...eta-pk-in-meta-open-index-in-value.1.ddl.aql |  59 +++
 ...-pk-in-meta-open-index-in-value.2.update.aql |  28 ++
 ...a-pk-in-meta-open-index-in-value.3.query.aql |  29 ++
 ...eta-pk-in-meta-open-index-in-value.4.ddl.aql |  25 ++
 ...pk-in-meta-open-index-with-missing.1.ddl.aql |  59 +++
 ...in-meta-open-index-with-missing.2.update.aql |  28 ++
 ...-in-meta-open-index-with-missing.3.query.aql |  29 ++
 ...pk-in-meta-open-index-with-missing.4.ddl.aql |  25 ++
 ...ge-feed-with-meta-with-mixed-index.1.ddl.aql |  60 +++
 ...feed-with-meta-with-mixed-index.2.update.aql |  28 ++
 ...-feed-with-meta-with-mixed-index.3.query.aql |  30 ++
 ...ge-feed-with-meta-with-mixed-index.4.ddl.aql |  25 ++
 ...ith-meta-pk-in-meta-index-after-ingest.4.adm |   1 +
 ...eed-with-meta-pk-in-meta-index-in-meta.4.adm |   1 +
 ...n-meta-index-with-missing-after-ingest.4.adm |   1 +
 ...th-meta-pk-in-meta-open-index-in-value.4.adm |   1 +
 ...eta-pk-in-meta-open-index-with-missing.4.adm |   1 +
 ...change-feed-with-meta-with-mixed-index.3.adm |   1 +
 .../src/test/resources/runtimets/testsuite.xml  |  36 ++
 .../input/record/reader/kv/KVTestReader.java    |  17 +-
 .../common/statement/CreateIndexStatement.java  |  11 +
 .../metadata/declared/AqlDataSource.java        |  13 +-
 .../metadata/declared/AqlMetadataProvider.java  |  75 +++-
 .../apache/asterix/metadata/entities/Index.java |  11 +
 .../asterix/metadata/utils/DatasetUtils.java    |   8 +-
 .../metadata/utils/KeyFieldTypeUtils.java       |   2 +-
 .../functions/CastRecordDescriptor.java         |   3 +
 ...rixLSMPrimaryUpsertOperatorNodePushable.java |  28 +-
 .../IndexInsertDeleteUpsertOperator.java        |   9 +-
 .../logical/InsertDeleteUpsertOperator.java     |  35 ++
 .../visitors/OperatorDeepCopyVisitor.java       | 109 +++---
 .../IndexInsertDeleteUpsertPOperator.java       |   9 +-
 .../physical/InsertDeleteUpsertPOperator.java   |   5 +-
 .../LogicalOperatorPrettyPrintVisitor.java      |  10 +-
 .../SetAlgebricksPhysicalOperatorsRule.java     |   3 +-
 .../common/tuples/TypeAwareTupleReference.java  |   5 +-
 .../am/lsm/btree/impls/LSMBTreeFileManager.java |   2 +-
 pom.xml                                         |  31 ++
 79 files changed, 2294 insertions(+), 511 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/asterixdb/blob/dfa3c117/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/IntroduceSecondaryIndexInsertDeleteRule.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/IntroduceSecondaryIndexInsertDeleteRule.java
b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/IntroduceSecondaryIndexInsertDeleteRule.java
index 6205962..f5cfa2b 100644
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/IntroduceSecondaryIndexInsertDeleteRule.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/IntroduceSecondaryIndexInsertDeleteRule.java
@@ -102,17 +102,30 @@ public class IntroduceSecondaryIndexInsertDeleteRule implements IAlgebraicRewrit
         InsertDeleteUpsertOperator insertOp = (InsertDeleteUpsertOperator) op1;
         boolean isBulkload = insertOp.isBulkload();
         ILogicalExpression recordExpr = insertOp.getPayloadExpression().getValue();
+        List<Mutable<ILogicalExpression>> metaExprs = insertOp.getAdditionalNonFilteringExpressions();
         LogicalVariable recordVar = null;
+        LogicalVariable metaVar = null;
         List<LogicalVariable> usedRecordVars = new ArrayList<>();
         /** assume the payload is always a single variable expression */
         recordExpr.getUsedVariables(usedRecordVars);
         if (usedRecordVars.size() == 1) {
             recordVar = usedRecordVars.get(0);
         }
+        if (metaExprs != null) {
+            List<LogicalVariable> metaVars = new ArrayList<>();
+            for (Mutable<ILogicalExpression> expr : metaExprs) {
+                expr.getValue().getUsedVariables(metaVars);
+            }
+            if (metaVars.size() > 1) {
+                throw new AlgebricksException(
+                        "Number of meta fields can't be more than 1. Number of meta fields
found = " + metaVars.size());
+            }
+            metaVar = metaVars.get(0);
+        }
 
         /**
-         * op2 is the assign operator which extract primary keys from the record
-         * variable
+         * op2 is the assign operator which extracts primary keys from the input
+         * variables (record or meta)
          */
         AbstractLogicalOperator op2 = (AbstractLogicalOperator) op1.getInputs().get(0).getValue();
 
@@ -169,6 +182,11 @@ public class IntroduceSecondaryIndexInsertDeleteRule implements IAlgebraicRewrit
             throw new AlgebricksException("Only record types can be indexed.");
         }
         ARecordType recType = (ARecordType) itemType;
+        // meta type
+        ARecordType metaType = null;
+        if (dataset.hasMetaPart()) {
+            metaType = (ARecordType) mp.findType(dataset.getMetaItemTypeDataverseName(),
dataset.getMetaItemTypeName());
+        }
         // recType might be replaced with enforced record type and we want to keep a reference
to the original record
         // type
         ARecordType originalRecType = recType;
@@ -228,7 +246,7 @@ public class IntroduceSecondaryIndexInsertDeleteRule implements IAlgebraicRewrit
         if (insertOp.getOperation() == Kind.INSERT || insertOp.getOperation() == Kind.UPSERT)
{
             try {
                 DatasetDataSource ds = (DatasetDataSource) (insertOp.getDataSource());
-                ARecordType insertRecType = (ARecordType) ds.getSchemaTypes()[ds.getSchemaTypes().length
- 1];
+                ARecordType insertRecType = (ARecordType) ds.getItemType();
                 // A new variable which represents the casted record
                 LogicalVariable castedRecVar = context.newVar();
                 // create the expected record type = the original + the optional open field
@@ -281,11 +299,19 @@ public class IntroduceSecondaryIndexInsertDeleteRule implements IAlgebraicRewrit
             List<List<String>> secondaryKeyFields = index.getKeyFieldNames();
             List<IAType> secondaryKeyTypes = index.getKeyFieldTypes();
             List<LogicalVariable> secondaryKeyVars = new ArrayList<LogicalVariable>();
+            List<Integer> indicators = index.getKeyFieldSourceIndicators();
             List<Mutable<ILogicalExpression>> expressions = new ArrayList<Mutable<ILogicalExpression>>();
             List<Mutable<ILogicalExpression>> secondaryExpressions = new ArrayList<Mutable<ILogicalExpression>>();
 
-            for (List<String> secondaryKey : secondaryKeyFields) {
-                prepareVarAndExpression(secondaryKey, recType.getFieldNames(), enforcedRecordVar,
expressions,
+            for (int i = 0; i < secondaryKeyFields.size(); i++) {
+                List<String> secondaryKey = secondaryKeyFields.get(i);
+                ARecordType sourceType = recType;
+                LogicalVariable sourceVar = enforcedRecordVar;
+                if (dataset.hasMetaPart()) {
+                    sourceType = indicators.get(i).intValue() == 0 ? recType : metaType;
+                    sourceVar = indicators.get(i).intValue() == 0 ? enforcedRecordVar : metaVar;
+                }
+                prepareVarAndExpression(secondaryKey, sourceType.getFieldNames(), sourceVar,
expressions,
                         secondaryKeyVars, context);
             }
             // Used with upsert operation
@@ -298,14 +324,18 @@ public class IntroduceSecondaryIndexInsertDeleteRule implements IAlgebraicRewrit
                 prevSecondaryKeyVars = new ArrayList<LogicalVariable>();
                 prevExpressions = new ArrayList<Mutable<ILogicalExpression>>();
                 prevSecondaryExpressions = new ArrayList<Mutable<ILogicalExpression>>();
-                for (List<String> secondaryKey : secondaryKeyFields) {
-                    prepareVarAndExpression(secondaryKey, originalRecType.getFieldNames(),
insertOp.getPrevRecordVar(),
+                for (int i = 0; i < secondaryKeyFields.size(); i++) {
+                    List<String> secondaryKey = secondaryKeyFields.get(i);
+                    prepareVarAndExpression(secondaryKey,
+                            (indicators.get(i).intValue() == 0) ? originalRecType.getFieldNames()
+                                    : metaType.getFieldNames(),
+                            (indicators.get(i).intValue() == 0) ? insertOp.getPrevRecordVar()
+                                    : insertOp.getPrevAdditionalNonFilteringVars().get(0),
                             prevExpressions, prevSecondaryKeyVars, context);
                 }
                 prevSecondaryKeyAssign = new AssignOperator(prevSecondaryKeyVars, prevExpressions);
             }
             AssignOperator assign = new AssignOperator(secondaryKeyVars, expressions);
-
             AssignOperator topAssign = assign;
             if (insertOp.getOperation() == Kind.UPSERT) {
                 prevSecondaryKeyAssign.getInputs().add(new MutableObject<ILogicalOperator>(topAssign));
@@ -402,7 +432,9 @@ public class IntroduceSecondaryIndexInsertDeleteRule implements IAlgebraicRewrit
 
                     IndexInsertDeleteUpsertOperator indexUpdate = new IndexInsertDeleteUpsertOperator(dataSourceIndex,
                             insertOp.getPrimaryKeyExpressions(), tokenizeKeyExprs, filterExpression,
-                            insertOp.getOperation(), insertOp.isBulkload());
+                            insertOp.getOperation(), insertOp.isBulkload(),
+                            insertOp.getAdditionalNonFilteringExpressions() == null ? 0
+                                    : insertOp.getAdditionalNonFilteringExpressions().size());
                     indexUpdate.setAdditionalFilteringExpressions(filteringExpressions);
                     indexUpdate.getInputs().add(new MutableObject<ILogicalOperator>(tokenUpdate));
 
@@ -415,7 +447,9 @@ public class IntroduceSecondaryIndexInsertDeleteRule implements IAlgebraicRewrit
                     // When TokenizeOperator is not needed
                     IndexInsertDeleteUpsertOperator indexUpdate = new IndexInsertDeleteUpsertOperator(dataSourceIndex,
                             insertOp.getPrimaryKeyExpressions(), secondaryExpressions, filterExpression,
-                            insertOp.getOperation(), insertOp.isBulkload());
+                            insertOp.getOperation(), insertOp.isBulkload(),
+                            insertOp.getAdditionalNonFilteringExpressions() == null ? 0
+                                    : insertOp.getAdditionalNonFilteringExpressions().size());
 
                     indexUpdate.setAdditionalFilteringExpressions(filteringExpressions);
                     // We add the necessary expressions for upsert
@@ -482,7 +516,7 @@ public class IntroduceSecondaryIndexInsertDeleteRule implements IAlgebraicRewrit
                 // We do something similar for previous key if the operation is an upsert
                 if (insertOp.getOperation() == Kind.UPSERT) {
                     List<LogicalVariable> originalKeyVarList = new ArrayList<LogicalVariable>();
-                    List<Mutable<ILogicalExpression>> originalKeyExprList = new
ArrayList<Mutable<ILogicalExpression>>();
+                    List<Mutable<ILogicalExpression>> originalKeyExprList = new
ArrayList<>();
                     // we don't do any filtering since nulls are expected here and there
                     for (int i = 0; i < numKeys; i++) {
                         LogicalVariable keyVar = context.newVar();
@@ -522,7 +556,9 @@ public class IntroduceSecondaryIndexInsertDeleteRule implements IAlgebraicRewrit
                 AqlIndex dataSourceIndex = new AqlIndex(index, dataverseName, datasetName,
mp);
                 IndexInsertDeleteUpsertOperator indexUpdate = new IndexInsertDeleteUpsertOperator(dataSourceIndex,
                         insertOp.getPrimaryKeyExpressions(), secondaryExpressions, filterExpression,
-                        insertOp.getOperation(), insertOp.isBulkload());
+                        insertOp.getOperation(), insertOp.isBulkload(),
+                        insertOp.getAdditionalNonFilteringExpressions() == null ? 0
+                                : insertOp.getAdditionalNonFilteringExpressions().size());
                 indexUpdate.setAdditionalFilteringExpressions(filteringExpressions);
                 if (insertOp.getOperation() == Kind.UPSERT) {
                     // set old secondary key expressions
@@ -573,6 +609,9 @@ public class IntroduceSecondaryIndexInsertDeleteRule implements IAlgebraicRewrit
             if (!index.isSecondaryIndex() || !index.isEnforcingKeyFileds()) {
                 continue;
             }
+            if (index.hasMetaFields()) {
+                throw new AlgebricksException("Indexing an open field is only supported on
the record part");
+            }
             for (int i = 0; i < index.getKeyFieldNames().size(); i++) {
                 Stack<Pair<ARecordType, String>> nestedTypeStack = new Stack<Pair<ARecordType,
String>>();
                 List<String> splits = index.getKeyFieldNames().get(i);
@@ -609,14 +648,10 @@ public class IntroduceSecondaryIndexInsertDeleteRule implements IAlgebraicRewrit
                     enforcedType = new ARecordType(bridgeName,
                             ArrayUtils.addAll(parent.getFieldNames(), enforcedType.getTypeName()),
parentFieldTypes,
                             true);
-
                 } else {
                     //Schema is closed all the way to the field
                     //enforced fields are either null or strongly typed
-                    LinkedHashMap<String, IAType> recordNameTypesMap = new LinkedHashMap<String,
IAType>();
-                    for (j = 0; j < nestedFieldType.getFieldNames().length; j++) {
-                        recordNameTypesMap.put(nestedFieldType.getFieldNames()[j], nestedFieldType.getFieldTypes()[j]);
-                    }
+                    LinkedHashMap<String, IAType> recordNameTypesMap = createRecordNameTypeMap(nestedFieldType);
                     // if a an enforced field already exists and the type is correct
                     IAType enforcedFieldType = recordNameTypesMap.get(splits.get(splits.size()
- 1));
                     if (enforcedFieldType != null && enforcedFieldType.getTypeTag()
== ATypeTag.UNION
@@ -654,6 +689,14 @@ public class IntroduceSecondaryIndexInsertDeleteRule implements IAlgebraicRewrit
         return enforcedType;
     }
 
+    private static LinkedHashMap<String, IAType> createRecordNameTypeMap(ARecordType
nestedFieldType) {
+        LinkedHashMap<String, IAType> recordNameTypesMap = new LinkedHashMap<>();
+        for (int j = 0; j < nestedFieldType.getFieldNames().length; j++) {
+            recordNameTypesMap.put(nestedFieldType.getFieldNames()[j], nestedFieldType.getFieldTypes()[j]);
+        }
+        return recordNameTypesMap;
+    }
+
     /***
      * This method takes a list of {fields}: a subset of {recordFields}, the original record
variable
      * and populate expressions with expressions which evaluate to those fields (using field
access functions) and
@@ -675,8 +718,7 @@ public class IntroduceSecondaryIndexInsertDeleteRule implements IAlgebraicRewrit
             List<Mutable<ILogicalExpression>> expressions, List<LogicalVariable>
vars, IOptimizationContext context)
             throws AlgebricksException {
         // Get a reference to the record variable
-        Mutable<ILogicalExpression> varRef = new MutableObject<ILogicalExpression>(
-                new VariableReferenceExpression(recordVar));
+        Mutable<ILogicalExpression> varRef = new MutableObject<>(new VariableReferenceExpression(recordVar));
         // Get the desired field position
         int pos = -1;
         if (fields.size() == 1) {
@@ -730,12 +772,12 @@ public class IntroduceSecondaryIndexInsertDeleteRule implements IAlgebraicRewrit
             if (!NonTaggedFormatUtil.isOptional(secondaryKeyType) && !forceFilter)
{
                 continue;
             }
-            ScalarFunctionCallExpression isNullFuncExpr = new ScalarFunctionCallExpression(
+            ScalarFunctionCallExpression isUnknownFuncExpr = new ScalarFunctionCallExpression(
                     FunctionUtil.getFunctionInfo(AsterixBuiltinFunctions.IS_UNKOWN),
                     new MutableObject<ILogicalExpression>(new VariableReferenceExpression(secondaryKeyVar)));
             ScalarFunctionCallExpression notFuncExpr = new ScalarFunctionCallExpression(
                     FunctionUtil.getFunctionInfo(AsterixBuiltinFunctions.NOT),
-                    new MutableObject<ILogicalExpression>(isNullFuncExpr));
+                    new MutableObject<ILogicalExpression>(isUnknownFuncExpr));
             filterExpressions.add(new MutableObject<ILogicalExpression>(notFuncExpr));
         }
         // No nullable secondary keys.

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/dfa3c117/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/AbstractIntroduceAccessMethodRule.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/AbstractIntroduceAccessMethodRule.java
b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/AbstractIntroduceAccessMethodRule.java
index d17b663..ed1803d 100644
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/AbstractIntroduceAccessMethodRule.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/AbstractIntroduceAccessMethodRule.java
@@ -108,7 +108,7 @@ public abstract class AbstractIntroduceAccessMethodRule implements IAlgebraicRew
 
     protected void fillSubTreeIndexExprs(OptimizableOperatorSubTree subTree,
             Map<IAccessMethod, AccessMethodAnalysisContext> analyzedAMs, IOptimizationContext
context)
-                    throws AlgebricksException {
+            throws AlgebricksException {
         Iterator<Map.Entry<IAccessMethod, AccessMethodAnalysisContext>> amIt
= analyzedAMs.entrySet().iterator();
         // Check applicability of indexes by access method type.
         while (amIt.hasNext()) {
@@ -257,7 +257,7 @@ public abstract class AbstractIntroduceAccessMethodRule implements IAlgebraicRew
                                 @Override
                                 public Object getVarType(LogicalVariable var, List<LogicalVariable>
nonNullVariables,
                                         List<List<LogicalVariable>> correlatedNullableVariableLists)
-                                                throws AlgebricksException {
+                                        throws AlgebricksException {
                                     if (var.equals(optFuncExpr.getSourceVar(exprAndVarIdx.second)))
{
                                         return keyType;
                                     }
@@ -435,7 +435,7 @@ public abstract class AbstractIntroduceAccessMethodRule implements IAlgebraicRew
     protected boolean fillIndexExprs(List<Index> datasetIndexes, List<String>
fieldName, IAType fieldType,
             IOptimizableFuncExpr optFuncExpr, int matchedFuncExprIndex, int varIdx,
             OptimizableOperatorSubTree matchedSubTree, AccessMethodAnalysisContext analysisCtx)
-                    throws AlgebricksException {
+            throws AlgebricksException {
         List<Index> indexCandidates = new ArrayList<Index>();
         // Add an index to the candidates if one of the indexed fields is
         // fieldName
@@ -462,9 +462,18 @@ public abstract class AbstractIntroduceAccessMethodRule implements IAlgebraicRew
             IOptimizationContext context) throws AlgebricksException {
         int optFuncExprIndex = 0;
         List<Index> datasetIndexes = new ArrayList<Index>();
+        LogicalVariable datasetMetaVar = null;
+        LogicalVariable datasetRecordVar = null;
         if (subTree.dataSourceType != DataSourceType.COLLECTION_SCAN) {
             datasetIndexes = metadataProvider.getDatasetIndexes(subTree.dataset.getDataverseName(),
                     subTree.dataset.getDatasetName());
+            List<LogicalVariable> datasetVars = subTree.getDataSourceVariables();
+            if (subTree.dataset.hasMetaPart()) {
+                datasetMetaVar = datasetVars.get(datasetVars.size() - 1);
+                datasetRecordVar = datasetVars.get(datasetVars.size() - 2);
+            } else {
+                datasetRecordVar = datasetVars.get(datasetVars.size() - 1);
+            }
         }
         for (IOptimizableFuncExpr optFuncExpr : analysisCtx.matchedFuncExprs) {
             // Try to match variables from optFuncExpr to assigns or unnests.
@@ -472,73 +481,11 @@ public abstract class AbstractIntroduceAccessMethodRule implements IAlgebraicRew
                     .size(); assignOrUnnestIndex++) {
                 AbstractLogicalOperator op = subTree.assignsAndUnnests.get(assignOrUnnestIndex);
                 if (op.getOperatorTag() == LogicalOperatorTag.ASSIGN) {
-                    AssignOperator assignOp = (AssignOperator) op;
-                    List<LogicalVariable> varList = assignOp.getVariables();
-                    for (int varIndex = 0; varIndex < varList.size(); varIndex++) {
-                        LogicalVariable var = varList.get(varIndex);
-                        int optVarIndex = optFuncExpr.findLogicalVar(var);
-                        // No matching var in optFuncExpr.
-                        if (optVarIndex == -1) {
-                            continue;
-                        }
-                        // At this point we have matched the optimizable func
-                        // expr at optFuncExprIndex to an assigned variable.
-                        // Remember matching subtree.
-                        optFuncExpr.setOptimizableSubTree(optVarIndex, subTree);
-                        List<String> fieldName = getFieldNameFromSubTree(optFuncExpr,
subTree, assignOrUnnestIndex,
-                                varIndex, subTree.recordType, optVarIndex,
-                                optFuncExpr.getFuncExpr().getArguments().get(optVarIndex).getValue());
-                        if (fieldName == null) {
-                            continue;
-                        }
-                        IAType fieldType = (IAType) context.getOutputTypeEnvironment(assignOp)
-                                .getType(optFuncExpr.getLogicalExpr(optVarIndex));
-                        // Set the fieldName in the corresponding matched
-                        // function expression.
-                        optFuncExpr.setFieldName(optVarIndex, fieldName);
-                        optFuncExpr.setFieldType(optVarIndex, fieldType);
-
-                        setTypeTag(context, subTree, optFuncExpr, optVarIndex);
-                        if (subTree.hasDataSource()) {
-                            fillIndexExprs(datasetIndexes, fieldName, fieldType, optFuncExpr,
optFuncExprIndex,
-                                    optVarIndex, subTree, analysisCtx);
-                        }
-                    }
+                    analyzeAssignOp((AssignOperator) op, optFuncExpr, subTree, assignOrUnnestIndex,
datasetRecordVar,
+                            datasetMetaVar, context, datasetIndexes, optFuncExprIndex, analysisCtx);
                 } else {
-                    UnnestOperator unnestOp = (UnnestOperator) op;
-                    LogicalVariable var = unnestOp.getVariable();
-                    int funcVarIndex = optFuncExpr.findLogicalVar(var);
-                    // No matching var in optFuncExpr.
-                    if (funcVarIndex == -1) {
-                        continue;
-                    }
-                    // At this point we have matched the optimizable func expr
-                    // at optFuncExprIndex to an unnest variable.
-                    // Remember matching subtree.
-                    optFuncExpr.setOptimizableSubTree(funcVarIndex, subTree);
-                    List<String> fieldName = null;
-                    if (subTree.dataSourceType == DataSourceType.COLLECTION_SCAN) {
-                        optFuncExpr.setLogicalExpr(funcVarIndex, new VariableReferenceExpression(var));
-                    } else {
-                        fieldName = getFieldNameFromSubTree(optFuncExpr, subTree, assignOrUnnestIndex,
0,
-                                subTree.recordType, funcVarIndex,
-                                optFuncExpr.getFuncExpr().getArguments().get(funcVarIndex).getValue());
-                        if (fieldName == null) {
-                            continue;
-                        }
-                    }
-                    IAType fieldType = (IAType) context.getOutputTypeEnvironment(unnestOp)
-                            .getType(optFuncExpr.getLogicalExpr(funcVarIndex));
-                    // Set the fieldName in the corresponding matched function
-                    // expression.
-                    optFuncExpr.setFieldName(funcVarIndex, fieldName);
-                    optFuncExpr.setFieldType(funcVarIndex, fieldType);
-
-                    setTypeTag(context, subTree, optFuncExpr, funcVarIndex);
-                    if (subTree.hasDataSource()) {
-                        fillIndexExprs(datasetIndexes, fieldName, fieldType, optFuncExpr,
optFuncExprIndex,
-                                funcVarIndex, subTree, analysisCtx);
-                    }
+                    analyzeUnnestOp((UnnestOperator) op, optFuncExpr, subTree, assignOrUnnestIndex,
datasetRecordVar,
+                            datasetMetaVar, context, datasetIndexes, optFuncExprIndex, analysisCtx);
                 }
             }
 
@@ -567,10 +514,88 @@ public abstract class AbstractIntroduceAccessMethodRule implements IAlgebraicRew
         }
     }
 
+    private void analyzeUnnestOp(UnnestOperator unnestOp, IOptimizableFuncExpr optFuncExpr,
+            OptimizableOperatorSubTree subTree, int assignOrUnnestIndex, LogicalVariable
datasetRecordVar,
+            LogicalVariable datasetMetaVar, IOptimizationContext context, List<Index>
datasetIndexes,
+            int optFuncExprIndex, AccessMethodAnalysisContext analysisCtx) throws AlgebricksException
{
+        LogicalVariable var = unnestOp.getVariable();
+        int funcVarIndex = optFuncExpr.findLogicalVar(var);
+        // No matching var in optFuncExpr.
+        if (funcVarIndex == -1) {
+            return;
+        }
+        // At this point we have matched the optimizable func expr
+        // at optFuncExprIndex to an unnest variable.
+        // Remember matching subtree.
+        optFuncExpr.setOptimizableSubTree(funcVarIndex, subTree);
+        List<String> fieldName = null;
+        if (subTree.dataSourceType == DataSourceType.COLLECTION_SCAN) {
+            optFuncExpr.setLogicalExpr(funcVarIndex, new VariableReferenceExpression(var));
+        } else {
+            fieldName = getFieldNameFromSubTree(optFuncExpr, subTree, assignOrUnnestIndex,
0, subTree.recordType,
+                    funcVarIndex, optFuncExpr.getFuncExpr().getArguments().get(funcVarIndex).getValue(),
+                    datasetRecordVar, subTree.metaRecordType, datasetMetaVar);
+            if (fieldName == null) {
+                return;
+            }
+        }
+        IAType fieldType = (IAType) context.getOutputTypeEnvironment(unnestOp)
+                .getType(optFuncExpr.getLogicalExpr(funcVarIndex));
+        // Set the fieldName in the corresponding matched function
+        // expression.
+        optFuncExpr.setFieldName(funcVarIndex, fieldName);
+        optFuncExpr.setFieldType(funcVarIndex, fieldType);
+
+        setTypeTag(context, subTree, optFuncExpr, funcVarIndex);
+        if (subTree.hasDataSource()) {
+            fillIndexExprs(datasetIndexes, fieldName, fieldType, optFuncExpr, optFuncExprIndex,
funcVarIndex, subTree,
+                    analysisCtx);
+        }
+    }
+
+    private void analyzeAssignOp(AssignOperator assignOp, IOptimizableFuncExpr optFuncExpr,
+            OptimizableOperatorSubTree subTree, int assignOrUnnestIndex, LogicalVariable
datasetRecordVar,
+            LogicalVariable datasetMetaVar, IOptimizationContext context, List<Index>
datasetIndexes,
+            int optFuncExprIndex, AccessMethodAnalysisContext analysisCtx) throws AlgebricksException
{
+        List<LogicalVariable> varList = assignOp.getVariables();
+        for (int varIndex = 0; varIndex < varList.size(); varIndex++) {
+            LogicalVariable var = varList.get(varIndex);
+            int optVarIndex = optFuncExpr.findLogicalVar(var);
+            // No matching var in optFuncExpr.
+            if (optVarIndex == -1) {
+                continue;
+            }
+            // At this point we have matched the optimizable func
+            // expr at optFuncExprIndex to an assigned variable.
+            // Remember matching subtree.
+            optFuncExpr.setOptimizableSubTree(optVarIndex, subTree);
+
+            List<String> fieldName = getFieldNameFromSubTree(optFuncExpr, subTree,
assignOrUnnestIndex, varIndex,
+                    subTree.recordType, optVarIndex,
+                    optFuncExpr.getFuncExpr().getArguments().get(optVarIndex).getValue(),
datasetRecordVar,
+                    subTree.metaRecordType, datasetMetaVar);
+            if (fieldName == null) {
+                continue;
+            }
+            IAType fieldType = (IAType) context.getOutputTypeEnvironment(assignOp)
+                    .getType(optFuncExpr.getLogicalExpr(optVarIndex));
+            // Set the fieldName in the corresponding matched
+            // function expression.
+            optFuncExpr.setFieldName(optVarIndex, fieldName);
+            optFuncExpr.setFieldType(optVarIndex, fieldType);
+
+            setTypeTag(context, subTree, optFuncExpr, optVarIndex);
+            if (subTree.hasDataSource()) {
+                fillIndexExprs(datasetIndexes, fieldName, fieldType, optFuncExpr, optFuncExprIndex,
optVarIndex,
+                        subTree, analysisCtx);
+            }
+        }
+    }
+
     private void matchVarsFromOptFuncExprToDataSourceScan(IOptimizableFuncExpr optFuncExpr,
int optFuncExprIndex,
             List<Index> datasetIndexes, List<LogicalVariable> dsVarList, OptimizableOperatorSubTree
subTree,
             AccessMethodAnalysisContext analysisCtx, IOptimizationContext context, boolean
fromAdditionalDataSource)
-                    throws AlgebricksException {
+            throws AlgebricksException {
         for (int varIndex = 0; varIndex < dsVarList.size(); varIndex++) {
             LogicalVariable var = dsVarList.get(varIndex);
             int funcVarIndex = optFuncExpr.findLogicalVar(var);
@@ -640,7 +665,8 @@ public abstract class AbstractIntroduceAccessMethodRule implements IAlgebraicRew
      */
     protected List<String> getFieldNameFromSubTree(IOptimizableFuncExpr optFuncExpr,
OptimizableOperatorSubTree subTree,
             int opIndex, int assignVarIndex, ARecordType recordType, int funcVarIndex,
-            ILogicalExpression parentFuncExpr) throws AlgebricksException {
+            ILogicalExpression parentFuncExpr, LogicalVariable recordVar, ARecordType metaType,
LogicalVariable metaVar)
+            throws AlgebricksException {
         // Get expression corresponding to opVar at varIndex.
         AbstractLogicalExpression expr = null;
         AbstractFunctionCallExpression childFuncExpr = null;
@@ -705,6 +731,8 @@ public abstract class AbstractIntroduceAccessMethodRule implements IAlgebraicRew
             isByName = true;
         }
         if (isFieldAccess) {
+            LogicalVariable sourceVar = ((VariableReferenceExpression) funcExpr.getArguments().get(0).getValue())
+                    .getVariableReference();
             optFuncExpr.setLogicalExpr(funcVarIndex, parentFuncExpr);
             int[] assignAndExpressionIndexes = null;
 
@@ -743,7 +771,7 @@ public abstract class AbstractIntroduceAccessMethodRule implements IAlgebraicRew
                 //Recursive call on nested assign
                 List<String> parentFieldNames = getFieldNameFromSubTree(optFuncExpr,
subTree,
                         assignAndExpressionIndexes[0], assignAndExpressionIndexes[1], recordType,
funcVarIndex,
-                        parentFuncExpr);
+                        parentFuncExpr, recordVar, metaType, metaVar);
 
                 if (parentFieldNames == null) {
                     //Nested assign was not a field access.
@@ -752,8 +780,9 @@ public abstract class AbstractIntroduceAccessMethodRule implements IAlgebraicRew
                 }
 
                 if (!isByName) {
-                    fieldName = ((ARecordType) recordType.getSubFieldType(parentFieldNames))
-                            .getFieldNames()[fieldIndex];
+                    fieldName = sourceVar.equals(metaVar)
+                            ? ((ARecordType) metaType.getSubFieldType(parentFieldNames)).getFieldNames()[fieldIndex]
+                            : ((ARecordType) recordType.getSubFieldType(parentFieldNames)).getFieldNames()[fieldIndex];
                 }
                 optFuncExpr.setSourceVar(funcVarIndex, ((AssignOperator) op).getVariables().get(assignVarIndex));
                 //add fieldName to the nested fieldName, return
@@ -773,9 +802,10 @@ public abstract class AbstractIntroduceAccessMethodRule implements IAlgebraicRew
                 if (nestedAccessFieldName != null) {
                     return nestedAccessFieldName;
                 }
-                return new ArrayList<String>(Arrays.asList(fieldName));
+                return new ArrayList<>(Arrays.asList(fieldName));
             }
-            return new ArrayList<String>(Arrays.asList(recordType.getFieldNames()[fieldIndex]));
+            return new ArrayList<>(Arrays.asList(sourceVar.equals(metaVar) ? metaType.getFieldNames()[fieldIndex]
+                    : recordType.getFieldNames()[fieldIndex]));
 
         }
 
@@ -806,7 +836,7 @@ public abstract class AbstractIntroduceAccessMethodRule implements IAlgebraicRew
                     if (var.equals(curVar)) {
                         optFuncExpr.setSourceVar(funcVarIndex, var);
                         return getFieldNameFromSubTree(optFuncExpr, subTree, assignOrUnnestIndex,
varIndex, recordType,
-                                funcVarIndex, childFuncExpr);
+                                funcVarIndex, childFuncExpr, recordVar, metaType, metaVar);
                     }
                 }
             } else {
@@ -814,7 +844,7 @@ public abstract class AbstractIntroduceAccessMethodRule implements IAlgebraicRew
                 LogicalVariable var = unnestOp.getVariable();
                 if (var.equals(curVar)) {
                     getFieldNameFromSubTree(optFuncExpr, subTree, assignOrUnnestIndex, 0,
recordType, funcVarIndex,
-                            childFuncExpr);
+                            childFuncExpr, recordVar, metaType, metaVar);
                 }
             }
         }


Mime
View raw message