asterixdb-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From mb...@apache.org
Subject [3/3] asterixdb git commit: Cast Record Before Insert for Open Indexes
Date Sun, 24 Jul 2016 11:32:35 GMT
Cast Record Before Insert for Open Indexes

This change modifies the insert plan to perform cast for open
indexes before inserting to primary index. This avoids modification
of indexes and then doing complex work to try and undo the insert.
In addition, feeds can simply log and remove tuples which have
open fields of incompatible type.

Change-Id: I0eef5813ddbfe1b9c518cd7f92d37c95e8486914
Reviewed-on: https://asterix-gerrit.ics.uci.edu/997
Tested-by: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Integration-Tests: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Reviewed-by: Michael Blow <mblow@apache.org>


Project: http://git-wip-us.apache.org/repos/asf/asterixdb/repo
Commit: http://git-wip-us.apache.org/repos/asf/asterixdb/commit/a3603ac9
Tree: http://git-wip-us.apache.org/repos/asf/asterixdb/tree/a3603ac9
Diff: http://git-wip-us.apache.org/repos/asf/asterixdb/diff/a3603ac9

Branch: refs/heads/master
Commit: a3603ac97f47d91ebc2e5ef5d49ebc18091c591c
Parents: 806f7d2
Author: Abdullah Alamoudi <bamousaa@gmail.com>
Authored: Fri Jul 22 23:02:32 2016 +0300
Committer: Michael Blow <mblow@apache.org>
Committed: Sun Jul 24 04:31:53 2016 -0700

----------------------------------------------------------------------
 ...IntroduceSecondaryIndexInsertDeleteRule.java |   75 +-
 .../data/external-parser/dropbox2/jobads3.txt   | 7608 ++++++++++++++++++
 .../api/http/servlet/QueryServiceServlet.java   |   11 +-
 ...th-external-parser-with-open-index.1.ddl.aql |   33 +
 ...th-external-parser-with-open-index.2.lib.aql |   26 +
 ...th-external-parser-with-open-index.3.ddl.aql |   35 +
 ...external-parser-with-open-index.4.update.aql |   31 +
 ...-external-parser-with-open-index.5.query.aql |   30 +
 ...th-external-parser-with-open-index.6.lib.aql |   26 +
 ...th-external-parser-with-open-index.7.ddl.aql |   26 +
 ...ernal-parser-with-two-open-indexes.1.ddl.aql |   33 +
 ...ernal-parser-with-two-open-indexes.2.lib.aql |   26 +
 ...ernal-parser-with-two-open-indexes.3.ddl.aql |   36 +
 ...al-parser-with-two-open-indexes.4.update.aql |   31 +
 ...nal-parser-with-two-open-indexes.5.query.aql |   30 +
 ...ernal-parser-with-two-open-indexes.6.lib.aql |   26 +
 ...ernal-parser-with-two-open-indexes.7.ddl.aql |   26 +
 ...d-with-external-parser-with-open-index.1.adm |    1 +
 ...-external-parser-with-two-open-indexes.1.adm |    1 +
 .../src/test/resources/runtimets/testsuite.xml  |   12 +-
 .../resources/runtimets/testsuite_sqlpp.xml     |   14 +-
 ...erixLSMInsertDeleteOperatorNodePushable.java |   18 +-
 .../asterix/common/exceptions/ErrorCode.java    |   27 +
 .../common/exceptions/FrameDataException.java   |   38 -
 .../feed/dataflow/FeedExceptionHandler.java     |    8 +-
 .../om/pointables/cast/ARecordCaster.java       |   18 +-
 .../rewriter/rules/InlineVariablesRule.java     |  127 +-
 .../operators/std/AssignRuntimeFactory.java     |   53 +-
 .../hyracks/api/exceptions/ErrorCode.java       |   13 +-
 .../api/exceptions/HyracksDataException.java    |   76 +-
 .../control/common/utils/ExceptionUtils.java    |   25 +-
 31 files changed, 8318 insertions(+), 222 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/asterixdb/blob/a3603ac9/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/IntroduceSecondaryIndexInsertDeleteRule.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/IntroduceSecondaryIndexInsertDeleteRule.java
b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/IntroduceSecondaryIndexInsertDeleteRule.java
index f5cfa2b..ae69093 100644
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/IntroduceSecondaryIndexInsertDeleteRule.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/IntroduceSecondaryIndexInsertDeleteRule.java
@@ -146,8 +146,8 @@ public class IntroduceSecondaryIndexInsertDeleteRule implements IAlgebraicRewrit
                 AssignOperator assignOp = (AssignOperator) op2;
                 ILogicalExpression assignExpr = assignOp.getExpressions().get(0).getValue();
                 if (assignExpr.getExpressionTag() == LogicalExpressionTag.FUNCTION_CALL)
{
-                    ScalarFunctionCallExpression funcExpr = (ScalarFunctionCallExpression)
assignOp.getExpressions()
-                            .get(0).getValue();
+                    ScalarFunctionCallExpression funcExpr =
+                            (ScalarFunctionCallExpression) assignOp.getExpressions().get(0).getValue();
                     fid = funcExpr.getFunctionIdentifier();
                 }
             }
@@ -214,10 +214,10 @@ public class IntroduceSecondaryIndexInsertDeleteRule implements IAlgebraicRewrit
         }
 
         // At this point, we have the data type info, and the indexes info as well
-        // Initialize inputs to the SINK operator Op0 (The SINK) is now without input
         if (secondaryIndexTotalCnt > 0) {
             op0.getInputs().clear();
         }
+        // Initialize inputs to the SINK operator Op0 (The SINK) is now without input
 
         // Prepare filtering field information (This is the filter created using the "filter
with" key word in the
         // create dataset ddl)
@@ -247,11 +247,12 @@ public class IntroduceSecondaryIndexInsertDeleteRule implements IAlgebraicRewrit
             try {
                 DatasetDataSource ds = (DatasetDataSource) (insertOp.getDataSource());
                 ARecordType insertRecType = (ARecordType) ds.getItemType();
-                // A new variable which represents the casted record
-                LogicalVariable castedRecVar = context.newVar();
                 // create the expected record type = the original + the optional open field
                 ARecordType enforcedType = createEnforcedType(insertRecType, indexes);
                 if (!enforcedType.equals(insertRecType)) {
+                    // A new variable which represents the casted record
+                    LogicalVariable castedRecVar = context.newVar();
+                    context.addNotToBeInlinedVar(castedRecVar);
                     //introduce casting to enforced type
                     AbstractFunctionCallExpression castFunc = new ScalarFunctionCallExpression(
                             FunctionUtil.getFunctionInfo(AsterixBuiltinFunctions.CAST_RECORD));
@@ -260,14 +261,16 @@ public class IntroduceSecondaryIndexInsertDeleteRule implements IAlgebraicRewrit
                             .add(new MutableObject<ILogicalExpression>(insertOp.getPayloadExpression().getValue()));
                     TypeCastUtils.setRequiredAndInputTypes(castFunc, enforcedType, insertRecType);
                     // AssignOperator puts in the cast var the casted record
-                    AssignOperator castedRecordAssignOperator = new AssignOperator(castedRecVar,
-                            new MutableObject<ILogicalExpression>(castFunc));
+                    AssignOperator castedRecordAssignOperator =
+                            new AssignOperator(castedRecVar, new MutableObject<ILogicalExpression>(castFunc));
                     // Connect the current top of the plan to the cast operator
-                    castedRecordAssignOperator.getInputs().add(new MutableObject<ILogicalOperator>(currentTop));
-                    currentTop = castedRecordAssignOperator;
+                    castedRecordAssignOperator.getInputs().addAll(currentTop.getInputs());
+                    currentTop.getInputs().clear();
+                    currentTop.getInputs().add(new MutableObject<>(castedRecordAssignOperator));
                     enforcedRecordVar = castedRecVar;
                     recType = enforcedType;
                     context.computeAndSetTypeEnvironmentForOperator(castedRecordAssignOperator);
+                    context.computeAndSetTypeEnvironmentForOperator(currentTop);
                     // We don't need to cast the old rec, we just need an assignment function
that extracts the SK
                     // and an expression which reference the new variables.
                 }
@@ -404,8 +407,8 @@ public class IntroduceSecondaryIndexInsertDeleteRule implements IAlgebraicRewrit
 
                     // Check the field type of the secondary key.
                     IAType secondaryKeyType = null;
-                    Pair<IAType, Boolean> keyPairType = Index.getNonNullableKeyFieldType(secondaryKeyFields.get(0),
-                            recType);
+                    Pair<IAType, Boolean> keyPairType =
+                            Index.getNonNullableKeyFieldType(secondaryKeyFields.get(0), recType);
                     secondaryKeyType = keyPairType.first;
 
                     List<Object> varTypes = new ArrayList<Object>();
@@ -430,11 +433,11 @@ public class IntroduceSecondaryIndexInsertDeleteRule implements IAlgebraicRewrit
                     tokenUpdate.getInputs().add(new MutableObject<ILogicalOperator>(assign));
                     context.computeAndSetTypeEnvironmentForOperator(tokenUpdate);
 
-                    IndexInsertDeleteUpsertOperator indexUpdate = new IndexInsertDeleteUpsertOperator(dataSourceIndex,
-                            insertOp.getPrimaryKeyExpressions(), tokenizeKeyExprs, filterExpression,
-                            insertOp.getOperation(), insertOp.isBulkload(),
-                            insertOp.getAdditionalNonFilteringExpressions() == null ? 0
-                                    : insertOp.getAdditionalNonFilteringExpressions().size());
+                    IndexInsertDeleteUpsertOperator indexUpdate =
+                            new IndexInsertDeleteUpsertOperator(dataSourceIndex, insertOp.getPrimaryKeyExpressions(),
+                                    tokenizeKeyExprs, filterExpression, insertOp.getOperation(),
insertOp.isBulkload(),
+                                    insertOp.getAdditionalNonFilteringExpressions() == null
? 0
+                                            : insertOp.getAdditionalNonFilteringExpressions().size());
                     indexUpdate.setAdditionalFilteringExpressions(filteringExpressions);
                     indexUpdate.getInputs().add(new MutableObject<ILogicalOperator>(tokenUpdate));
 
@@ -445,11 +448,11 @@ public class IntroduceSecondaryIndexInsertDeleteRule implements IAlgebraicRewrit
 
                 } else {
                     // When TokenizeOperator is not needed
-                    IndexInsertDeleteUpsertOperator indexUpdate = new IndexInsertDeleteUpsertOperator(dataSourceIndex,
-                            insertOp.getPrimaryKeyExpressions(), secondaryExpressions, filterExpression,
-                            insertOp.getOperation(), insertOp.isBulkload(),
-                            insertOp.getAdditionalNonFilteringExpressions() == null ? 0
-                                    : insertOp.getAdditionalNonFilteringExpressions().size());
+                    IndexInsertDeleteUpsertOperator indexUpdate =
+                            new IndexInsertDeleteUpsertOperator(dataSourceIndex, insertOp.getPrimaryKeyExpressions(),
+                                    secondaryExpressions, filterExpression, insertOp.getOperation(),
+                                    insertOp.isBulkload(), insertOp.getAdditionalNonFilteringExpressions()
== null ? 0
+                                            : insertOp.getAdditionalNonFilteringExpressions().size());
 
                     indexUpdate.setAdditionalFilteringExpressions(filteringExpressions);
                     // We add the necessary expressions for upsert
@@ -473,11 +476,11 @@ public class IntroduceSecondaryIndexInsertDeleteRule implements IAlgebraicRewrit
 
             } else if (index.getIndexType() == IndexType.RTREE) {
                 // Get type, dimensions and number of keys
-                Pair<IAType, Boolean> keyPairType = Index.getNonNullableOpenFieldType(secondaryKeyTypes.get(0),
-                        secondaryKeyFields.get(0), recType);
+                Pair<IAType, Boolean> keyPairType =
+                        Index.getNonNullableOpenFieldType(secondaryKeyTypes.get(0), secondaryKeyFields.get(0),
recType);
                 IAType spatialType = keyPairType.first;
-                boolean isPointMBR = spatialType.getTypeTag() == ATypeTag.POINT
-                        || spatialType.getTypeTag() == ATypeTag.POINT3D;
+                boolean isPointMBR =
+                        spatialType.getTypeTag() == ATypeTag.POINT || spatialType.getTypeTag()
== ATypeTag.POINT3D;
                 int dimension = NonTaggedFormatUtil.getNumDimensions(spatialType.getTypeTag());
                 int numKeys = (isPointMBR && isBulkload) ? dimension : dimension
* 2;
                 // Get variables and expressions
@@ -554,11 +557,11 @@ public class IntroduceSecondaryIndexInsertDeleteRule implements IAlgebraicRewrit
                             context.getOutputTypeEnvironment(assignCoordinates), forceFilter);
                 }
                 AqlIndex dataSourceIndex = new AqlIndex(index, dataverseName, datasetName,
mp);
-                IndexInsertDeleteUpsertOperator indexUpdate = new IndexInsertDeleteUpsertOperator(dataSourceIndex,
-                        insertOp.getPrimaryKeyExpressions(), secondaryExpressions, filterExpression,
-                        insertOp.getOperation(), insertOp.isBulkload(),
-                        insertOp.getAdditionalNonFilteringExpressions() == null ? 0
-                                : insertOp.getAdditionalNonFilteringExpressions().size());
+                IndexInsertDeleteUpsertOperator indexUpdate =
+                        new IndexInsertDeleteUpsertOperator(dataSourceIndex, insertOp.getPrimaryKeyExpressions(),
+                                secondaryExpressions, filterExpression, insertOp.getOperation(),
insertOp.isBulkload(),
+                                insertOp.getAdditionalNonFilteringExpressions() == null ?
0
+                                        : insertOp.getAdditionalNonFilteringExpressions().size());
                 indexUpdate.setAdditionalFilteringExpressions(filteringExpressions);
                 if (insertOp.getOperation() == Kind.UPSERT) {
                     // set old secondary key expressions
@@ -772,12 +775,12 @@ public class IntroduceSecondaryIndexInsertDeleteRule implements IAlgebraicRewrit
             if (!NonTaggedFormatUtil.isOptional(secondaryKeyType) && !forceFilter)
{
                 continue;
             }
-            ScalarFunctionCallExpression isUnknownFuncExpr = new ScalarFunctionCallExpression(
-                    FunctionUtil.getFunctionInfo(AsterixBuiltinFunctions.IS_UNKOWN),
-                    new MutableObject<ILogicalExpression>(new VariableReferenceExpression(secondaryKeyVar)));
-            ScalarFunctionCallExpression notFuncExpr = new ScalarFunctionCallExpression(
-                    FunctionUtil.getFunctionInfo(AsterixBuiltinFunctions.NOT),
-                    new MutableObject<ILogicalExpression>(isUnknownFuncExpr));
+            ScalarFunctionCallExpression isUnknownFuncExpr =
+                    new ScalarFunctionCallExpression(FunctionUtil.getFunctionInfo(AsterixBuiltinFunctions.IS_UNKOWN),
+                            new MutableObject<ILogicalExpression>(new VariableReferenceExpression(secondaryKeyVar)));
+            ScalarFunctionCallExpression notFuncExpr =
+                    new ScalarFunctionCallExpression(FunctionUtil.getFunctionInfo(AsterixBuiltinFunctions.NOT),
+                            new MutableObject<ILogicalExpression>(isUnknownFuncExpr));
             filterExpressions.add(new MutableObject<ILogicalExpression>(notFuncExpr));
         }
         // No nullable secondary keys.


Mime
View raw message