asterixdb-notifications mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From "abdullah alamoudi (Code Review)" <do-not-re...@asterixdb.incubator.apache.org>
Subject Change in asterixdb[master]: ASTERIXDB-1272 Remove ExternalLookupOperator
Date Tue, 16 Feb 2016 17:14:46 GMT
abdullah alamoudi has submitted this change and it was merged.

Change subject: ASTERIXDB-1272 Remove ExternalLookupOperator
......................................................................


ASTERIXDB-1272 Remove ExternalLookupOperator

ExternalLookupOperator has been removed in this change
and UnnestMapOperator was used instead. different visitors
and methods in the UnnestMapOperator were changed to
support both external and internal data.

Change-Id: Ic9a9400586b75fd8cb3c0b8f4aeb45aee76cdb92
Reviewed-on: https://asterix-gerrit.ics.uci.edu/592
Tested-by: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Reviewed-by: Yingyi Bu <buyingyi@gmail.com>
---
M asterix-algebra/src/main/java/org/apache/asterix/algebra/operators/physical/ExternalDataLookupPOperator.java
M asterix-algebra/src/main/java/org/apache/asterix/optimizer/base/AnalysisUtil.java
M asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/IntroduceInstantLockSearchCallbackRule.java
M asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/SweepIllegalNonfunctionalFunctions.java
M asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/AccessMethodUtils.java
M asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/BTreeAccessMethod.java
M asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/IntroduceLSMComponentFilterRule.java
M asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/OptimizableOperatorSubTree.java
M asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/RTreeAccessMethod.java
M asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/subplan/InlineAllNtsInSubplanVisitor.java
M asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/subplan/InlineLeftNtsInSubplanJoinFlatteningVisitor.java
M asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/subplan/SubplanSpecialFlatteningCheckVisitor.java
M asterix-om/src/main/java/org/apache/asterix/om/functions/AsterixBuiltinFunctions.java
13 files changed, 73 insertions(+), 103 deletions(-)

Approvals:
  Yingyi Bu: Looks good to me, approved
  Jenkins: Verified



diff --git a/asterix-algebra/src/main/java/org/apache/asterix/algebra/operators/physical/ExternalDataLookupPOperator.java
b/asterix-algebra/src/main/java/org/apache/asterix/algebra/operators/physical/ExternalDataLookupPOperator.java
index 9b85674..ade7efd 100644
--- a/asterix-algebra/src/main/java/org/apache/asterix/algebra/operators/physical/ExternalDataLookupPOperator.java
+++ b/asterix-algebra/src/main/java/org/apache/asterix/algebra/operators/physical/ExternalDataLookupPOperator.java
@@ -45,8 +45,8 @@
 import org.apache.hyracks.algebricks.core.algebra.functions.FunctionIdentifier;
 import org.apache.hyracks.algebricks.core.algebra.metadata.IDataSourcePropertiesProvider;
 import org.apache.hyracks.algebricks.core.algebra.operators.logical.AbstractScanOperator;
-import org.apache.hyracks.algebricks.core.algebra.operators.logical.ExternalDataLookupOperator;
 import org.apache.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema;
+import org.apache.hyracks.algebricks.core.algebra.operators.logical.UnnestMapOperator;
 import org.apache.hyracks.algebricks.core.algebra.operators.logical.visitors.VariableUtilities;
 import org.apache.hyracks.algebricks.core.algebra.operators.physical.AbstractScanPOperator;
 import org.apache.hyracks.algebricks.core.algebra.properties.BroadcastPartitioningProperty;
@@ -124,8 +124,8 @@
     public void contributeRuntimeOperator(IHyracksJobBuilder builder, JobGenContext context,
ILogicalOperator op,
             IOperatorSchema opSchema, IOperatorSchema[] inputSchemas, IOperatorSchema outerPlanSchema)
                     throws AlgebricksException {
-        ExternalDataLookupOperator edabro = (ExternalDataLookupOperator) op;
-        ILogicalExpression expr = edabro.getExpressionRef().getValue();
+        UnnestMapOperator unnestMap = (UnnestMapOperator) op;
+        ILogicalExpression expr = unnestMap.getExpressionRef().getValue();
         if (expr.getExpressionTag() != LogicalExpressionTag.FUNCTION_CALL) {
             throw new IllegalStateException();
         }
@@ -138,19 +138,19 @@
         IVariableTypeEnvironment typeEnv = context.getTypeEnvironment(op);
         List<LogicalVariable> outputVars = new ArrayList<LogicalVariable>();
         if (retainInput) {
-            VariableUtilities.getLiveVariables(edabro, outputVars);
+            VariableUtilities.getLiveVariables(unnestMap, outputVars);
         } else {
-            VariableUtilities.getProducedVariables(edabro, outputVars);
+            VariableUtilities.getProducedVariables(unnestMap, outputVars);
         }
 
         AqlMetadataProvider metadataProvider = (AqlMetadataProvider) context.getMetadataProvider();
         Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> externalLoopup = AqlMetadataProvider
                 .buildExternalDataLookupRuntime(builder.getJobSpec(), dataset, secondaryIndex,
ridIndexes, retainInput,
                         typeEnv, outputVars, opSchema, context, metadataProvider, retainNull);
-        builder.contributeHyracksOperator(edabro, externalLoopup.first);
+        builder.contributeHyracksOperator(unnestMap, externalLoopup.first);
         builder.contributeAlgebricksPartitionConstraint(externalLoopup.first, externalLoopup.second);
-        ILogicalOperator srcExchange = edabro.getInputs().get(0).getValue();
-        builder.contributeGraphEdge(srcExchange, 0, edabro, 0);
+        ILogicalOperator srcExchange = unnestMap.getInputs().get(0).getValue();
+        builder.contributeGraphEdge(srcExchange, 0, unnestMap, 0);
     }
 
     protected int[] getKeyIndexes(List<LogicalVariable> keyVarList, IOperatorSchema[]
inputSchemas) {
diff --git a/asterix-algebra/src/main/java/org/apache/asterix/optimizer/base/AnalysisUtil.java
b/asterix-algebra/src/main/java/org/apache/asterix/optimizer/base/AnalysisUtil.java
index 0a66013..beefc46 100644
--- a/asterix-algebra/src/main/java/org/apache/asterix/optimizer/base/AnalysisUtil.java
+++ b/asterix-algebra/src/main/java/org/apache/asterix/optimizer/base/AnalysisUtil.java
@@ -23,6 +23,7 @@
 
 import org.apache.asterix.metadata.declared.AqlSourceId;
 import org.apache.asterix.om.functions.AsterixBuiltinFunctions;
+import org.apache.asterix.optimizer.rules.am.AccessMethodUtils;
 import org.apache.commons.lang3.mutable.Mutable;
 import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
 import org.apache.hyracks.algebricks.common.utils.Pair;
@@ -35,6 +36,7 @@
 import org.apache.hyracks.algebricks.core.algebra.functions.FunctionIdentifier;
 import org.apache.hyracks.algebricks.core.algebra.operators.logical.AbstractDataSourceOperator;
 import org.apache.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
+import org.apache.hyracks.algebricks.core.algebra.operators.logical.UnnestMapOperator;
 
 public class AnalysisUtil {
     /*
@@ -130,6 +132,13 @@
         return new Pair<String, String>(srcId.getDataverseName(), srcId.getDatasourceName());
     }
 
+    public static Pair<String, String> getExternalDatasetInfo(UnnestMapOperator op)
throws AlgebricksException {
+        AbstractFunctionCallExpression unnestExpr = (AbstractFunctionCallExpression) op.getExpressionRef().getValue();
+        String dataverseName = AccessMethodUtils.getStringConstant(unnestExpr.getArguments().get(0));
+        String datasetName = AccessMethodUtils.getStringConstant(unnestExpr.getArguments().get(1));
+        return new Pair<String, String>(dataverseName, datasetName);
+    }
+
     private static List<FunctionIdentifier> fieldAccessFunctions = new ArrayList<FunctionIdentifier>();
 
     static {
diff --git a/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/IntroduceInstantLockSearchCallbackRule.java
b/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/IntroduceInstantLockSearchCallbackRule.java
index 54db5d3..90fb137 100644
--- a/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/IntroduceInstantLockSearchCallbackRule.java
+++ b/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/IntroduceInstantLockSearchCallbackRule.java
@@ -23,14 +23,13 @@
 import java.util.Map;
 import java.util.Map.Entry;
 
-import org.apache.commons.lang3.mutable.Mutable;
-
 import org.apache.asterix.algebra.operators.CommitOperator;
 import org.apache.asterix.algebra.operators.physical.BTreeSearchPOperator;
 import org.apache.asterix.metadata.declared.AqlDataSource;
 import org.apache.asterix.metadata.declared.AqlMetadataImplConfig;
 import org.apache.asterix.om.functions.AsterixBuiltinFunctions;
 import org.apache.asterix.optimizer.rules.am.AccessMethodJobGenParams;
+import org.apache.commons.lang3.mutable.Mutable;
 import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
 import org.apache.hyracks.algebricks.common.utils.Triple;
 import org.apache.hyracks.algebricks.core.algebra.base.ILogicalExpression;
@@ -51,7 +50,8 @@
 public class IntroduceInstantLockSearchCallbackRule implements IAlgebraicRewriteRule {
 
     @Override
-    public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext
context) throws AlgebricksException {
+    public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext
context)
+            throws AlgebricksException {
         return false;
     }
 
@@ -67,6 +67,9 @@
                 if (unnestExpr.getExpressionTag() == LogicalExpressionTag.FUNCTION_CALL)
{
                     AbstractFunctionCallExpression f = (AbstractFunctionCallExpression) unnestExpr;
                     FunctionIdentifier fid = f.getFunctionIdentifier();
+                    if (fid.equals(AsterixBuiltinFunctions.EXTERNAL_LOOKUP)) {
+                        return;
+                    }
                     if (!fid.equals(AsterixBuiltinFunctions.INDEX_SEARCH)) {
                         throw new IllegalStateException();
                     }
diff --git a/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/SweepIllegalNonfunctionalFunctions.java
b/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/SweepIllegalNonfunctionalFunctions.java
index 238eaa3..097d192 100644
--- a/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/SweepIllegalNonfunctionalFunctions.java
+++ b/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/SweepIllegalNonfunctionalFunctions.java
@@ -37,7 +37,6 @@
 import org.apache.hyracks.algebricks.core.algebra.operators.logical.EmptyTupleSourceOperator;
 import org.apache.hyracks.algebricks.core.algebra.operators.logical.ExchangeOperator;
 import org.apache.hyracks.algebricks.core.algebra.operators.logical.ExtensionOperator;
-import org.apache.hyracks.algebricks.core.algebra.operators.logical.ExternalDataLookupOperator;
 import org.apache.hyracks.algebricks.core.algebra.operators.logical.GroupByOperator;
 import org.apache.hyracks.algebricks.core.algebra.operators.logical.IndexInsertDeleteUpsertOperator;
 import org.apache.hyracks.algebricks.core.algebra.operators.logical.InnerJoinOperator;
@@ -299,12 +298,6 @@
 
         @Override
         public Void visitTokenizeOperator(TokenizeOperator op, Void tag) throws AlgebricksException
{
-            return null;
-        }
-
-        @Override
-        public Void visitExternalDataLookupOperator(ExternalDataLookupOperator op, Void arg)
-                throws AlgebricksException {
             return null;
         }
     }
diff --git a/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/AccessMethodUtils.java
b/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/AccessMethodUtils.java
index 9165506..f39d190 100644
--- a/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/AccessMethodUtils.java
+++ b/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/AccessMethodUtils.java
@@ -69,7 +69,6 @@
 import org.apache.hyracks.algebricks.core.algebra.operators.logical.AbstractDataSourceOperator;
 import org.apache.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
 import org.apache.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator.ExecutionMode;
-import org.apache.hyracks.algebricks.core.algebra.operators.logical.ExternalDataLookupOperator;
 import org.apache.hyracks.algebricks.core.algebra.operators.logical.GroupByOperator;
 import org.apache.hyracks.algebricks.core.algebra.operators.logical.OrderOperator;
 import org.apache.hyracks.algebricks.core.algebra.operators.logical.OrderOperator.IOrder;
@@ -600,7 +599,7 @@
         funcArgs.add(stringRef);
     }
 
-    public static ExternalDataLookupOperator createExternalDataLookupUnnestMap(AbstractDataSourceOperator
dataSourceOp,
+    public static UnnestMapOperator createExternalDataLookupUnnestMap(AbstractDataSourceOperator
dataSourceOp,
             Dataset dataset, ARecordType recordType, ILogicalOperator inputOp, IOptimizationContext
context,
             Index secondaryIndex, boolean retainInput, boolean retainNull) throws AlgebricksException
{
         List<LogicalVariable> primaryKeyVars = AccessMethodUtils.getPrimaryKeyVarsFromSecondaryUnnestMap(dataset,
@@ -618,37 +617,37 @@
         order.getInputs().add(new MutableObject<ILogicalOperator>(inputOp));
         order.setExecutionMode(ExecutionMode.LOCAL);
         context.computeAndSetTypeEnvironmentForOperator(order);
-        List<Mutable<ILogicalExpression>> externalRIDAccessFuncArgs = new ArrayList<Mutable<ILogicalExpression>>();
-        //Add dataverse and dataset to the arguments
-        AccessMethodUtils.addStringArg(dataset.getDataverseName(), externalRIDAccessFuncArgs);
-        AccessMethodUtils.addStringArg(dataset.getDatasetName(), externalRIDAccessFuncArgs);
-        AccessMethodUtils.writeVarList(primaryKeyVars, externalRIDAccessFuncArgs);
+        List<Mutable<ILogicalExpression>> externalLookupArgs = new ArrayList<Mutable<ILogicalExpression>>();
+        //Add dataverse to the arguments
+        AccessMethodUtils.addStringArg(dataset.getDataverseName(), externalLookupArgs);
+        //Add dataset to the arguments
+        AccessMethodUtils.addStringArg(dataset.getDatasetName(), externalLookupArgs);
+        //Add PK vars to the arguments
+        AccessMethodUtils.writeVarList(primaryKeyVars, externalLookupArgs);
 
         // Variables and types coming out of the external access.
-        List<LogicalVariable> externalAccessByRIDVars = new ArrayList<LogicalVariable>();
-        List<Object> externalAccessOutputTypes = new ArrayList<Object>();
+        List<LogicalVariable> externalUnnestVars = new ArrayList<LogicalVariable>();
+        List<Object> outputTypes = new ArrayList<Object>();
         // Append output variables/types generated by the data scan (not forwarded from input).
-        externalAccessByRIDVars.addAll(dataSourceOp.getVariables());
-        appendExternalRecTypes(dataset, recordType, externalAccessOutputTypes);
+        externalUnnestVars.addAll(dataSourceOp.getVariables());
+        appendExternalRecTypes(dataset, recordType, outputTypes);
 
-        IFunctionInfo externalAccessByRID = FunctionUtil.getFunctionInfo(AsterixBuiltinFunctions.EXTERNAL_LOOKUP);
-        AbstractFunctionCallExpression externalAccessFunc = new ScalarFunctionCallExpression(externalAccessByRID,
-                externalRIDAccessFuncArgs);
-
-        ExternalDataLookupOperator externalLookupOp = new ExternalDataLookupOperator(externalAccessByRIDVars,
-                new MutableObject<ILogicalExpression>(externalAccessFunc), externalAccessOutputTypes,
retainInput,
-                dataSourceOp.getDataSource());
+        IFunctionInfo externalLookup = FunctionUtil.getFunctionInfo(AsterixBuiltinFunctions.EXTERNAL_LOOKUP);
+        AbstractFunctionCallExpression externalLookupFunc = new ScalarFunctionCallExpression(externalLookup,
+                externalLookupArgs);
+        UnnestMapOperator unnestOp = new UnnestMapOperator(externalUnnestVars,
+                new MutableObject<ILogicalExpression>(externalLookupFunc), outputTypes,
retainInput);
         // Fed by the order operator or the secondaryIndexUnnestOp.
-        externalLookupOp.getInputs().add(new MutableObject<ILogicalOperator>(order));
+        unnestOp.getInputs().add(new MutableObject<ILogicalOperator>(order));
 
-        context.computeAndSetTypeEnvironmentForOperator(externalLookupOp);
-        externalLookupOp.setExecutionMode(ExecutionMode.PARTITIONED);
+        context.computeAndSetTypeEnvironmentForOperator(unnestOp);
+        unnestOp.setExecutionMode(ExecutionMode.PARTITIONED);
 
         //set the physical operator
         AqlSourceId dataSourceId = new AqlSourceId(dataset.getDataverseName(), dataset.getDatasetName());
-        externalLookupOp.setPhysicalOperator(new ExternalDataLookupPOperator(dataSourceId,
dataset, recordType,
-                secondaryIndex, primaryKeyVars, false, retainInput, retainNull));
-        return externalLookupOp;
+        unnestOp.setPhysicalOperator(new ExternalDataLookupPOperator(dataSourceId, dataset,
recordType, secondaryIndex,
+                primaryKeyVars, false, retainInput, retainNull));
+        return unnestOp;
     }
 
     //If the expression is constant at runtime, runturn the type
diff --git a/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/BTreeAccessMethod.java
b/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/BTreeAccessMethod.java
index 370b90d..3fdbc5c 100644
--- a/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/BTreeAccessMethod.java
+++ b/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/BTreeAccessMethod.java
@@ -60,7 +60,6 @@
 import org.apache.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator.ExecutionMode;
 import org.apache.hyracks.algebricks.core.algebra.operators.logical.AssignOperator;
 import org.apache.hyracks.algebricks.core.algebra.operators.logical.EmptyTupleSourceOperator;
-import org.apache.hyracks.algebricks.core.algebra.operators.logical.ExternalDataLookupOperator;
 import org.apache.hyracks.algebricks.core.algebra.operators.logical.SelectOperator;
 import org.apache.hyracks.algebricks.core.algebra.operators.logical.UnnestMapOperator;
 import org.apache.hyracks.algebricks.core.algebra.util.OperatorManipulationUtil;
@@ -489,9 +488,9 @@
         boolean isPrimaryIndex = chosenIndex.isPrimaryIndex();
         if (dataset.getDatasetType() == DatasetType.EXTERNAL) {
             // External dataset
-            ExternalDataLookupOperator externalDataAccessOp = AccessMethodUtils.createExternalDataLookupUnnestMap(
-                    dataSourceOp, dataset, recordType, secondaryIndexUnnestOp, context, chosenIndex,
retainInput,
-                    retainNull);
+            UnnestMapOperator externalDataAccessOp = AccessMethodUtils.createExternalDataLookupUnnestMap(dataSourceOp,
+                    dataset, recordType, secondaryIndexUnnestOp, context, chosenIndex, retainInput,
retainNull);
+            indexSubTree.dataSourceRef.setValue(externalDataAccessOp);
             return externalDataAccessOp;
         } else if (!isPrimaryIndex) {
             primaryIndexUnnestOp = AccessMethodUtils.createPrimaryIndexUnnestMap(dataSourceOp,
dataset, recordType,
diff --git a/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/IntroduceLSMComponentFilterRule.java
b/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/IntroduceLSMComponentFilterRule.java
index 55ba426..94dd84e 100644
--- a/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/IntroduceLSMComponentFilterRule.java
+++ b/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/IntroduceLSMComponentFilterRule.java
@@ -230,13 +230,21 @@
                 if (unnestExpr.getExpressionTag() == LogicalExpressionTag.FUNCTION_CALL)
{
                     AbstractFunctionCallExpression f = (AbstractFunctionCallExpression) unnestExpr;
                     FunctionIdentifier fid = f.getFunctionIdentifier();
-                    if (!fid.equals(AsterixBuiltinFunctions.INDEX_SEARCH)) {
-                        throw new IllegalStateException();
+                    String dataverseName = null;
+                    String datasetName = null;
+                    if (AsterixBuiltinFunctions.EXTERNAL_LOOKUP.equals(fid)) {
+                        dataverseName = AccessMethodUtils.getStringConstant(f.getArguments().get(0));
+                        datasetName = AccessMethodUtils.getStringConstant(f.getArguments().get(1));
+                    } else if (fid.equals(AsterixBuiltinFunctions.INDEX_SEARCH)) {
+                        AccessMethodJobGenParams jobGenParams = new AccessMethodJobGenParams();
+                        jobGenParams.readFromFuncArgs(f.getArguments());
+                        dataverseName = jobGenParams.dataverseName;
+                        datasetName = jobGenParams.datasetName;
+                    } else {
+                        throw new AlgebricksException("Unexpected function for Unnest Map:
" + fid);
                     }
-                    AccessMethodJobGenParams jobGenParams = new AccessMethodJobGenParams();
-                    jobGenParams.readFromFuncArgs(f.getArguments());
-                    return ((AqlMetadataProvider) context.getMetadataProvider()).findDataset(jobGenParams.dataverseName,
-                            jobGenParams.datasetName);
+                    return ((AqlMetadataProvider) context.getMetadataProvider()).findDataset(dataverseName,
+                            datasetName);
                 }
             }
             if (descendantOp.getInputs().isEmpty()) {
diff --git a/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/OptimizableOperatorSubTree.java
b/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/OptimizableOperatorSubTree.java
index 52582ba..39a9ab8 100644
--- a/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/OptimizableOperatorSubTree.java
+++ b/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/OptimizableOperatorSubTree.java
@@ -42,7 +42,6 @@
 import org.apache.hyracks.algebricks.core.algebra.operators.logical.AbstractScanOperator;
 import org.apache.hyracks.algebricks.core.algebra.operators.logical.AbstractUnnestOperator;
 import org.apache.hyracks.algebricks.core.algebra.operators.logical.DataSourceScanOperator;
-import org.apache.hyracks.algebricks.core.algebra.operators.logical.ExternalDataLookupOperator;
 import org.apache.hyracks.algebricks.core.algebra.operators.logical.UnnestMapOperator;
 
 /**
@@ -230,9 +229,8 @@
                     dataverseName = jobGenParams.getDataverseName();
                     break;
                 case EXTERNAL_SCAN:
-                    ExternalDataLookupOperator externalScan = (ExternalDataLookupOperator)
sourceOpRefs.get(i)
-                            .getValue();
-                    datasetInfo = AnalysisUtil.getDatasetInfo(externalScan);
+                    UnnestMapOperator externalScan = (UnnestMapOperator) sourceOpRefs.get(i).getValue();
+                    datasetInfo = AnalysisUtil.getExternalDatasetInfo(externalScan);
                     dataverseName = datasetInfo.first;
                     datasetName = datasetInfo.second;
                     break;
diff --git a/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/RTreeAccessMethod.java
b/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/RTreeAccessMethod.java
index c5bd8af..3c925e2 100644
--- a/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/RTreeAccessMethod.java
+++ b/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/RTreeAccessMethod.java
@@ -51,7 +51,6 @@
 import org.apache.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
 import org.apache.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator.ExecutionMode;
 import org.apache.hyracks.algebricks.core.algebra.operators.logical.AssignOperator;
-import org.apache.hyracks.algebricks.core.algebra.operators.logical.ExternalDataLookupOperator;
 import org.apache.hyracks.algebricks.core.algebra.operators.logical.SelectOperator;
 import org.apache.hyracks.algebricks.core.algebra.operators.logical.UnnestMapOperator;
 import org.apache.hyracks.algebricks.core.algebra.util.OperatorManipulationUtil;
@@ -244,9 +243,8 @@
 
         // Generate the rest of the upstream plan which feeds the search results into the
primary index.
         if (dataset.getDatasetType() == DatasetType.EXTERNAL) {
-            ExternalDataLookupOperator externalDataAccessOp = AccessMethodUtils.createExternalDataLookupUnnestMap(
-                    dataSourceOp, dataset, recordType, secondaryIndexUnnestOp, context, chosenIndex,
retainInput,
-                    retainNull);
+            UnnestMapOperator externalDataAccessOp = AccessMethodUtils.createExternalDataLookupUnnestMap(dataSourceOp,
+                    dataset, recordType, secondaryIndexUnnestOp, context, chosenIndex, retainInput,
retainNull);
             return externalDataAccessOp;
         } else {
             UnnestMapOperator primaryIndexUnnestOp = AccessMethodUtils.createPrimaryIndexUnnestMap(dataSourceOp,
diff --git a/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/subplan/InlineAllNtsInSubplanVisitor.java
b/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/subplan/InlineAllNtsInSubplanVisitor.java
index 0715d46..b3e07e5 100644
--- a/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/subplan/InlineAllNtsInSubplanVisitor.java
+++ b/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/subplan/InlineAllNtsInSubplanVisitor.java
@@ -54,7 +54,6 @@
 import org.apache.hyracks.algebricks.core.algebra.operators.logical.EmptyTupleSourceOperator;
 import org.apache.hyracks.algebricks.core.algebra.operators.logical.ExchangeOperator;
 import org.apache.hyracks.algebricks.core.algebra.operators.logical.ExtensionOperator;
-import org.apache.hyracks.algebricks.core.algebra.operators.logical.ExternalDataLookupOperator;
 import org.apache.hyracks.algebricks.core.algebra.operators.logical.GroupByOperator;
 import org.apache.hyracks.algebricks.core.algebra.operators.logical.InnerJoinOperator;
 import org.apache.hyracks.algebricks.core.algebra.operators.logical.IntersectOperator;
@@ -586,19 +585,6 @@
     @Override
     public ILogicalOperator visitExchangeOperator(ExchangeOperator op, Void arg) throws AlgebricksException
{
         return visitSingleInputOperator(op);
-    }
-
-    @Override
-    public ILogicalOperator visitExternalDataLookupOperator(ExternalDataLookupOperator op,
Void arg)
-            throws AlgebricksException {
-        visitSingleInputOperator(op);
-        Set<LogicalVariable> liveVars = new HashSet<>();
-        VariableUtilities.getLiveVariables(op, liveVars);
-        if (!liveVars.containsAll(correlatedKeyVars)) {
-            op.setPropagateInput(true);
-        }
-        context.computeAndSetTypeEnvironmentForOperator(op);
-        return op;
     }
 
     @Override
diff --git a/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/subplan/InlineLeftNtsInSubplanJoinFlatteningVisitor.java
b/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/subplan/InlineLeftNtsInSubplanJoinFlatteningVisitor.java
index 5aaa59f..33ed9f9 100644
--- a/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/subplan/InlineLeftNtsInSubplanJoinFlatteningVisitor.java
+++ b/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/subplan/InlineLeftNtsInSubplanJoinFlatteningVisitor.java
@@ -42,7 +42,6 @@
 import org.apache.hyracks.algebricks.core.algebra.operators.logical.EmptyTupleSourceOperator;
 import org.apache.hyracks.algebricks.core.algebra.operators.logical.ExchangeOperator;
 import org.apache.hyracks.algebricks.core.algebra.operators.logical.ExtensionOperator;
-import org.apache.hyracks.algebricks.core.algebra.operators.logical.ExternalDataLookupOperator;
 import org.apache.hyracks.algebricks.core.algebra.operators.logical.GroupByOperator;
 import org.apache.hyracks.algebricks.core.algebra.operators.logical.InnerJoinOperator;
 import org.apache.hyracks.algebricks.core.algebra.operators.logical.IntersectOperator;
@@ -362,22 +361,6 @@
     @Override
     public ILogicalOperator visitExchangeOperator(ExchangeOperator op, Void arg) throws AlgebricksException
{
         return visitSingleInputOperator(op);
-    }
-
-    @Override
-    public ILogicalOperator visitExternalDataLookupOperator(ExternalDataLookupOperator op,
Void arg)
-            throws AlgebricksException {
-        visitSingleInputOperator(op);
-        if (!rewritten) {
-            return op;
-        }
-        Set<LogicalVariable> liveVars = new HashSet<>();
-        VariableUtilities.getLiveVariables(op, liveVars);
-        if (!liveVars.containsAll(liveVarsFromSubplanInput)) {
-            op.setPropagateInput(true);
-        }
-        context.computeAndSetTypeEnvironmentForOperator(op);
-        return op;
     }
 
     @Override
diff --git a/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/subplan/SubplanSpecialFlatteningCheckVisitor.java
b/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/subplan/SubplanSpecialFlatteningCheckVisitor.java
index c691365..d2fe52f 100644
--- a/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/subplan/SubplanSpecialFlatteningCheckVisitor.java
+++ b/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/subplan/SubplanSpecialFlatteningCheckVisitor.java
@@ -28,7 +28,6 @@
 import org.apache.hyracks.algebricks.core.algebra.operators.logical.EmptyTupleSourceOperator;
 import org.apache.hyracks.algebricks.core.algebra.operators.logical.ExchangeOperator;
 import org.apache.hyracks.algebricks.core.algebra.operators.logical.ExtensionOperator;
-import org.apache.hyracks.algebricks.core.algebra.operators.logical.ExternalDataLookupOperator;
 import org.apache.hyracks.algebricks.core.algebra.operators.logical.GroupByOperator;
 import org.apache.hyracks.algebricks.core.algebra.operators.logical.InnerJoinOperator;
 import org.apache.hyracks.algebricks.core.algebra.operators.logical.IntersectOperator;
@@ -216,11 +215,6 @@
 
     @Override
     public Boolean visitExchangeOperator(ExchangeOperator op, Void arg) throws AlgebricksException
{
-        return visitInputs(op);
-    }
-
-    @Override
-    public Boolean visitExternalDataLookupOperator(ExternalDataLookupOperator op, Void arg)
throws AlgebricksException {
         return visitInputs(op);
     }
 
diff --git a/asterix-om/src/main/java/org/apache/asterix/om/functions/AsterixBuiltinFunctions.java
b/asterix-om/src/main/java/org/apache/asterix/om/functions/AsterixBuiltinFunctions.java
index f1b0e67..a1c4e33 100644
--- a/asterix-om/src/main/java/org/apache/asterix/om/functions/AsterixBuiltinFunctions.java
+++ b/asterix-om/src/main/java/org/apache/asterix/om/functions/AsterixBuiltinFunctions.java
@@ -18,6 +18,12 @@
  */
 package org.apache.asterix.om.functions;
 
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
 import org.apache.asterix.common.functions.FunctionConstants;
 import org.apache.asterix.common.functions.FunctionSignature;
 import org.apache.asterix.om.typecomputer.base.IResultTypeComputer;
@@ -112,12 +118,6 @@
 import org.apache.hyracks.algebricks.core.algebra.functions.AlgebricksBuiltinFunctions;
 import org.apache.hyracks.algebricks.core.algebra.functions.FunctionIdentifier;
 import org.apache.hyracks.algebricks.core.algebra.functions.IFunctionInfo;
-
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
 
 public class AsterixBuiltinFunctions {
 

-- 
To view, visit https://asterix-gerrit.ics.uci.edu/592
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings

Gerrit-MessageType: merged
Gerrit-Change-Id: Ic9a9400586b75fd8cb3c0b8f4aeb45aee76cdb92
Gerrit-PatchSet: 12
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: abdullah alamoudi <bamousaa@gmail.com>
Gerrit-Reviewer: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Gerrit-Reviewer: Till Westmann <tillw@apache.org>
Gerrit-Reviewer: Yingyi Bu <buyingyi@gmail.com>
Gerrit-Reviewer: Yingyi Bu <yingyib@google.com>
Gerrit-Reviewer: abdullah alamoudi <bamousaa@gmail.com>

Mime
View raw message