asterixdb-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From buyin...@apache.org
Subject [14/16] asterixdb git commit: Add Asterix Extension Manager
Date Sat, 20 Aug 2016 06:15:56 GMT
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/OptimizableOperatorSubTree.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/OptimizableOperatorSubTree.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/OptimizableOperatorSubTree.java
index d5a6f2b..f483d70 100644
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/OptimizableOperatorSubTree.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/OptimizableOperatorSubTree.java
@@ -21,6 +21,8 @@ package org.apache.asterix.optimizer.rules.am;
 import java.util.ArrayList;
 import java.util.List;
 
+import org.apache.asterix.metadata.declared.AqlDataSource;
+import org.apache.asterix.metadata.declared.AqlDataSource.AqlDataSourceType;
 import org.apache.asterix.metadata.declared.AqlMetadataProvider;
 import org.apache.asterix.metadata.entities.Dataset;
 import org.apache.asterix.metadata.utils.DatasetUtils;
@@ -38,6 +40,7 @@ import org.apache.hyracks.algebricks.core.algebra.base.LogicalExpressionTag;
 import org.apache.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
 import org.apache.hyracks.algebricks.core.algebra.base.LogicalVariable;
 import org.apache.hyracks.algebricks.core.algebra.expressions.AbstractFunctionCallExpression;
+import org.apache.hyracks.algebricks.core.algebra.metadata.IDataSource;
 import org.apache.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
 import org.apache.hyracks.algebricks.core.algebra.operators.logical.AbstractScanOperator;
 import org.apache.hyracks.algebricks.core.algebra.operators.logical.AbstractUnnestOperator;
@@ -59,24 +62,24 @@ public class OptimizableOperatorSubTree {
         NO_DATASOURCE
     }
 
-    public ILogicalOperator root = null;
-    public Mutable<ILogicalOperator> rootRef = null;
-    public final List<Mutable<ILogicalOperator>> assignsAndUnnestsRefs = new ArrayList<Mutable<ILogicalOperator>>();
-    public final List<AbstractLogicalOperator> assignsAndUnnests = new ArrayList<AbstractLogicalOperator>();
-    public Mutable<ILogicalOperator> dataSourceRef = null;
-    public DataSourceType dataSourceType = DataSourceType.NO_DATASOURCE;
+    private ILogicalOperator root = null;
+    private Mutable<ILogicalOperator> rootRef = null;
+    private final List<Mutable<ILogicalOperator>> assignsAndUnnestsRefs = new ArrayList<>();
+    private final List<AbstractLogicalOperator> assignsAndUnnests = new ArrayList<>();
+    private Mutable<ILogicalOperator> dataSourceRef = null;
+    private DataSourceType dataSourceType = DataSourceType.NO_DATASOURCE;
 
     // Dataset and type metadata. Set in setDatasetAndTypeMetadata().
-    public Dataset dataset = null;
-    public ARecordType recordType = null;
-    public ARecordType metaRecordType = null;
+    private Dataset dataset = null;
+    private ARecordType recordType = null;
+    private ARecordType metaRecordType = null;
 
     // Additional datasources can exist if IntroduceJoinAccessMethodRule has been applied.
     // (E.g. There are index-nested-loop-joins in the plan.)
-    public List<Mutable<ILogicalOperator>> ixJoinOuterAdditionalDataSourceRefs = null;
-    public List<DataSourceType> ixJoinOuterAdditionalDataSourceTypes = null;
-    public List<Dataset> ixJoinOuterAdditionalDatasets = null;
-    public List<ARecordType> ixJoinOuterAdditionalRecordTypes = null;
+    private List<Mutable<ILogicalOperator>> ixJoinOuterAdditionalDataSourceRefs = null;
+    private List<DataSourceType> ixJoinOuterAdditionalDataSourceTypes = null;
+    private List<Dataset> ixJoinOuterAdditionalDatasets = null;
+    private List<ARecordType> ixJoinOuterAdditionalRecordTypes = null;
 
     public boolean initFromSubTree(Mutable<ILogicalOperator> subTreeOpRef) throws AlgebricksException {
         reset();
@@ -102,12 +105,12 @@ public class OptimizableOperatorSubTree {
                 if (!OperatorPropertiesUtil.isMovable(subTreeOp)) {
                     return false;
                 } else {
-                    assignsAndUnnestsRefs.add(subTreeOpRef);
-                    assignsAndUnnests.add(subTreeOp);
+                    getAssignsAndUnnestsRefs().add(subTreeOpRef);
+                    getAssignsAndUnnests().add(subTreeOp);
                 }
                 subTreeOpRef = subTreeOp.getInputs().get(0);
                 subTreeOp = (AbstractLogicalOperator) subTreeOpRef.getValue();
-            };
+            }
         } while (subTreeOp.getOperatorTag() == LogicalOperatorTag.SELECT);
 
         // Match data source (datasource scan or primary index search).
@@ -118,12 +121,12 @@ public class OptimizableOperatorSubTree {
         AbstractLogicalOperator subTreeOp = (AbstractLogicalOperator) subTreeOpRef.getValue();
 
         if (subTreeOp.getOperatorTag() == LogicalOperatorTag.DATASOURCESCAN) {
-            dataSourceType = DataSourceType.DATASOURCE_SCAN;
-            dataSourceRef = subTreeOpRef;
+            setDataSourceType(DataSourceType.DATASOURCE_SCAN);
+            setDataSourceRef(subTreeOpRef);
             return true;
         } else if (subTreeOp.getOperatorTag() == LogicalOperatorTag.EMPTYTUPLESOURCE) {
-            dataSourceType = DataSourceType.COLLECTION_SCAN;
-            dataSourceRef = subTreeOpRef;
+            setDataSourceType(DataSourceType.COLLECTION_SCAN);
+            setDataSourceRef(subTreeOpRef);
             return true;
         } else if (subTreeOp.getOperatorTag() == LogicalOperatorTag.UNNEST_MAP) {
             // There can be multiple unnest-map or datasource-scan operators
@@ -141,40 +144,40 @@ public class OptimizableOperatorSubTree {
                             AccessMethodJobGenParams jobGenParams = new AccessMethodJobGenParams();
                             jobGenParams.readFromFuncArgs(f.getArguments());
                             if (jobGenParams.isPrimaryIndex()) {
-                                if (dataSourceRef == null) {
-                                    dataSourceRef = subTreeOpRef;
-                                    dataSourceType = DataSourceType.PRIMARY_INDEX_LOOKUP;
+                                if (getDataSourceRef() == null) {
+                                    setDataSourceRef(subTreeOpRef);
+                                    setDataSourceType(DataSourceType.PRIMARY_INDEX_LOOKUP);
                                 } else {
                                     // One datasource already exists. This is an additional datasource.
                                     initializeIxJoinOuterAddtionalDataSourcesIfEmpty();
-                                    ixJoinOuterAdditionalDataSourceTypes.add(DataSourceType.PRIMARY_INDEX_LOOKUP);
-                                    ixJoinOuterAdditionalDataSourceRefs.add(subTreeOpRef);
+                                    getIxJoinOuterAdditionalDataSourceTypes().add(DataSourceType.PRIMARY_INDEX_LOOKUP);
+                                    getIxJoinOuterAdditionalDataSourceRefs().add(subTreeOpRef);
                                 }
                                 dataSourceFound = true;
                             }
                         } else if (f.getFunctionIdentifier().equals(AsterixBuiltinFunctions.EXTERNAL_LOOKUP)) {
                             // External lookup case
-                            if (dataSourceRef == null) {
-                                dataSourceRef = subTreeOpRef;
-                                dataSourceType = DataSourceType.EXTERNAL_SCAN;
+                            if (getDataSourceRef() == null) {
+                                setDataSourceRef(subTreeOpRef);
+                                setDataSourceType(DataSourceType.EXTERNAL_SCAN);
                             } else {
                                 // One datasource already exists. This is an additional datasource.
                                 initializeIxJoinOuterAddtionalDataSourcesIfEmpty();
-                                ixJoinOuterAdditionalDataSourceTypes.add(DataSourceType.EXTERNAL_SCAN);
-                                ixJoinOuterAdditionalDataSourceRefs.add(subTreeOpRef);
+                                getIxJoinOuterAdditionalDataSourceTypes().add(DataSourceType.EXTERNAL_SCAN);
+                                getIxJoinOuterAdditionalDataSourceRefs().add(subTreeOpRef);
                             }
                             dataSourceFound = true;
                         }
                     }
                 } else if (subTreeOp.getOperatorTag() == LogicalOperatorTag.DATASOURCESCAN) {
                     initializeIxJoinOuterAddtionalDataSourcesIfEmpty();
-                    ixJoinOuterAdditionalDataSourceTypes.add(DataSourceType.DATASOURCE_SCAN);
-                    ixJoinOuterAdditionalDataSourceRefs.add(subTreeOpRef);
+                    getIxJoinOuterAdditionalDataSourceTypes().add(DataSourceType.DATASOURCE_SCAN);
+                    getIxJoinOuterAdditionalDataSourceRefs().add(subTreeOpRef);
                     dataSourceFound = true;
                 } else if (subTreeOp.getOperatorTag() == LogicalOperatorTag.EMPTYTUPLESOURCE) {
                     initializeIxJoinOuterAddtionalDataSourcesIfEmpty();
-                    ixJoinOuterAdditionalDataSourceTypes.add(DataSourceType.COLLECTION_SCAN);
-                    ixJoinOuterAdditionalDataSourceRefs.add(subTreeOpRef);
+                    getIxJoinOuterAdditionalDataSourceTypes().add(DataSourceType.COLLECTION_SCAN);
+                    getIxJoinOuterAdditionalDataSourceRefs().add(subTreeOpRef);
                 }
 
                 // Traverse the subtree while there are operators in the path.
@@ -208,14 +211,14 @@ public class OptimizableOperatorSubTree {
         List<Mutable<ILogicalOperator>> sourceOpRefs = new ArrayList<Mutable<ILogicalOperator>>();
         List<DataSourceType> dsTypes = new ArrayList<DataSourceType>();
 
-        sourceOpRefs.add(dataSourceRef);
-        dsTypes.add(dataSourceType);
+        sourceOpRefs.add(getDataSourceRef());
+        dsTypes.add(getDataSourceType());
 
         // If there are multiple datasources in the subtree, we need to find the dataset for these.
-        if (ixJoinOuterAdditionalDataSourceRefs != null) {
-            for (int i = 0; i < ixJoinOuterAdditionalDataSourceRefs.size(); i++) {
-                sourceOpRefs.add(ixJoinOuterAdditionalDataSourceRefs.get(i));
-                dsTypes.add(ixJoinOuterAdditionalDataSourceTypes.get(i));
+        if (getIxJoinOuterAdditionalDataSourceRefs() != null) {
+            for (int i = 0; i < getIxJoinOuterAdditionalDataSourceRefs().size(); i++) {
+                sourceOpRefs.add(getIxJoinOuterAdditionalDataSourceRefs().get(i));
+                dsTypes.add(getIxJoinOuterAdditionalDataSourceTypes().get(i));
             }
         }
 
@@ -223,6 +226,14 @@ public class OptimizableOperatorSubTree {
             switch (dsTypes.get(i)) {
                 case DATASOURCE_SCAN:
                     DataSourceScanOperator dataSourceScan = (DataSourceScanOperator) sourceOpRefs.get(i).getValue();
+                    IDataSource<?> datasource = dataSourceScan.getDataSource();
+                    if (datasource instanceof AqlDataSource) {
+                        AqlDataSourceType dsType = ((AqlDataSource) datasource).getDatasourceType();
+                        if (dsType != AqlDataSourceType.INTERNAL_DATASET
+                                && dsType != AqlDataSourceType.EXTERNAL_DATASET) {
+                            return false;
+                        }
+                    }
                     Pair<String, String> datasetInfo = AnalysisUtil.getDatasetInfo(dataSourceScan);
                     dataverseName = datasetInfo.first;
                     datasetName = datasetInfo.second;
@@ -244,8 +255,8 @@ public class OptimizableOperatorSubTree {
                     break;
                 case COLLECTION_SCAN:
                     if (i != 0) {
-                        ixJoinOuterAdditionalDatasets.add(null);
-                        ixJoinOuterAdditionalRecordTypes.add(null);
+                        getIxJoinOuterAdditionalDatasets().add(null);
+                        getIxJoinOuterAdditionalRecordTypes().add(null);
                     }
                     continue;
                 case NO_DATASOURCE:
@@ -266,24 +277,24 @@ public class OptimizableOperatorSubTree {
                 if (i == 0) {
                     return false;
                 } else {
-                    ixJoinOuterAdditionalDatasets.add(null);
-                    ixJoinOuterAdditionalRecordTypes.add(null);
+                    getIxJoinOuterAdditionalDatasets().add(null);
+                    getIxJoinOuterAdditionalRecordTypes().add(null);
                 }
             }
             rType = (ARecordType) itemType;
 
             // Get the meta record type for that dataset.
-            IAType metaItemType = metadataProvider.findType(ds.getMetaItemTypeDataverseName(),
-                    ds.getMetaItemTypeName());
+            IAType metaItemType =
+                    metadataProvider.findType(ds.getMetaItemTypeDataverseName(), ds.getMetaItemTypeName());
 
             // First index is always the primary datasource in this subtree.
             if (i == 0) {
-                dataset = ds;
-                recordType = rType;
-                metaRecordType = (ARecordType) metaItemType;
+                setDataset(ds);
+                setRecordType(rType);
+                setMetaRecordType((ARecordType) metaItemType);
             } else {
-                ixJoinOuterAdditionalDatasets.add(ds);
-                ixJoinOuterAdditionalRecordTypes.add(rType);
+                getIxJoinOuterAdditionalDatasets().add(ds);
+                getIxJoinOuterAdditionalRecordTypes().add(rType);
             }
 
             dataverseName = null;
@@ -296,14 +307,14 @@ public class OptimizableOperatorSubTree {
     }
 
     public boolean hasDataSource() {
-        return dataSourceType != DataSourceType.NO_DATASOURCE;
+        return getDataSourceType() != DataSourceType.NO_DATASOURCE;
     }
 
     public boolean hasIxJoinOuterAdditionalDataSource() {
         boolean dataSourceFound = false;
-        if (ixJoinOuterAdditionalDataSourceTypes != null) {
-            for (int i = 0; i < ixJoinOuterAdditionalDataSourceTypes.size(); i++) {
-                if (ixJoinOuterAdditionalDataSourceTypes.get(i) != DataSourceType.NO_DATASOURCE) {
+        if (getIxJoinOuterAdditionalDataSourceTypes() != null) {
+            for (int i = 0; i < getIxJoinOuterAdditionalDataSourceTypes().size(); i++) {
+                if (getIxJoinOuterAdditionalDataSourceTypes().get(i) != DataSourceType.NO_DATASOURCE) {
                     dataSourceFound = true;
                     break;
                 }
@@ -313,13 +324,13 @@ public class OptimizableOperatorSubTree {
     }
 
     public boolean hasDataSourceScan() {
-        return dataSourceType == DataSourceType.DATASOURCE_SCAN;
+        return getDataSourceType() == DataSourceType.DATASOURCE_SCAN;
     }
 
     public boolean hasIxJoinOuterAdditionalDataSourceScan() {
-        if (ixJoinOuterAdditionalDataSourceTypes != null) {
-            for (int i = 0; i < ixJoinOuterAdditionalDataSourceTypes.size(); i++) {
-                if (ixJoinOuterAdditionalDataSourceTypes.get(i) == DataSourceType.DATASOURCE_SCAN) {
+        if (getIxJoinOuterAdditionalDataSourceTypes() != null) {
+            for (int i = 0; i < getIxJoinOuterAdditionalDataSourceTypes().size(); i++) {
+                if (getIxJoinOuterAdditionalDataSourceTypes().get(i) == DataSourceType.DATASOURCE_SCAN) {
                     return true;
                 }
             }
@@ -328,33 +339,33 @@ public class OptimizableOperatorSubTree {
     }
 
     public void reset() {
-        root = null;
-        rootRef = null;
-        assignsAndUnnestsRefs.clear();
-        assignsAndUnnests.clear();
-        dataSourceRef = null;
-        dataSourceType = DataSourceType.NO_DATASOURCE;
-        ixJoinOuterAdditionalDataSourceRefs = null;
-        ixJoinOuterAdditionalDataSourceTypes = null;
-        dataset = null;
-        ixJoinOuterAdditionalDatasets = null;
-        recordType = null;
-        ixJoinOuterAdditionalRecordTypes = null;
+        setRoot(null);
+        setRootRef(null);
+        getAssignsAndUnnestsRefs().clear();
+        getAssignsAndUnnests().clear();
+        setDataSourceRef(null);
+        setDataSourceType(DataSourceType.NO_DATASOURCE);
+        setIxJoinOuterAdditionalDataSourceRefs(null);
+        setIxJoinOuterAdditionalDataSourceTypes(null);
+        setDataset(null);
+        setIxJoinOuterAdditionalDatasets(null);
+        setRecordType(null);
+        setIxJoinOuterAdditionalRecordTypes(null);
     }
 
     public void getPrimaryKeyVars(List<LogicalVariable> target) throws AlgebricksException {
-        switch (dataSourceType) {
+        switch (getDataSourceType()) {
             case DATASOURCE_SCAN:
-                DataSourceScanOperator dataSourceScan = (DataSourceScanOperator) dataSourceRef.getValue();
-                int numPrimaryKeys = DatasetUtils.getPartitioningKeys(dataset).size();
+                DataSourceScanOperator dataSourceScan = (DataSourceScanOperator) getDataSourceRef().getValue();
+                int numPrimaryKeys = DatasetUtils.getPartitioningKeys(getDataset()).size();
                 for (int i = 0; i < numPrimaryKeys; i++) {
                     target.add(dataSourceScan.getVariables().get(i));
                 }
                 break;
             case PRIMARY_INDEX_LOOKUP:
-                UnnestMapOperator unnestMapOp = (UnnestMapOperator) dataSourceRef.getValue();
+                UnnestMapOperator unnestMapOp = (UnnestMapOperator) getDataSourceRef().getValue();
                 List<LogicalVariable> primaryKeys = null;
-                primaryKeys = AccessMethodUtils.getPrimaryKeyVarsFromPrimaryUnnestMap(dataset, unnestMapOp);
+                primaryKeys = AccessMethodUtils.getPrimaryKeyVarsFromPrimaryUnnestMap(getDataset(), unnestMapOp);
                 target.addAll(primaryKeys);
                 break;
             case NO_DATASOURCE:
@@ -364,14 +375,14 @@ public class OptimizableOperatorSubTree {
     }
 
     public List<LogicalVariable> getDataSourceVariables() throws AlgebricksException {
-        switch (dataSourceType) {
+        switch (getDataSourceType()) {
             case DATASOURCE_SCAN:
             case EXTERNAL_SCAN:
             case PRIMARY_INDEX_LOOKUP:
-                AbstractScanOperator scanOp = (AbstractScanOperator) dataSourceRef.getValue();
+                AbstractScanOperator scanOp = (AbstractScanOperator) getDataSourceRef().getValue();
                 return scanOp.getVariables();
             case COLLECTION_SCAN:
-                return new ArrayList<LogicalVariable>();
+                return new ArrayList<>();
             case NO_DATASOURCE:
             default:
                 throw new AlgebricksException("The subtree does not have any data source.");
@@ -379,16 +390,16 @@ public class OptimizableOperatorSubTree {
     }
 
     public List<LogicalVariable> getIxJoinOuterAdditionalDataSourceVariables(int idx) throws AlgebricksException {
-        if (ixJoinOuterAdditionalDataSourceRefs != null && ixJoinOuterAdditionalDataSourceRefs.size() > idx) {
-            switch (ixJoinOuterAdditionalDataSourceTypes.get(idx)) {
+        if (getIxJoinOuterAdditionalDataSourceRefs() != null && getIxJoinOuterAdditionalDataSourceRefs().size() > idx) {
+            switch (getIxJoinOuterAdditionalDataSourceTypes().get(idx)) {
                 case DATASOURCE_SCAN:
                 case EXTERNAL_SCAN:
                 case PRIMARY_INDEX_LOOKUP:
-                    AbstractScanOperator scanOp = (AbstractScanOperator) ixJoinOuterAdditionalDataSourceRefs.get(idx)
-                            .getValue();
+                    AbstractScanOperator scanOp =
+                            (AbstractScanOperator) getIxJoinOuterAdditionalDataSourceRefs().get(idx).getValue();
                     return scanOp.getVariables();
                 case COLLECTION_SCAN:
-                    return new ArrayList<LogicalVariable>();
+                    return new ArrayList<>();
                 case NO_DATASOURCE:
                 default:
                     throw new AlgebricksException("The subtree does not have any additional data sources.");
@@ -399,12 +410,109 @@ public class OptimizableOperatorSubTree {
     }
 
     public void initializeIxJoinOuterAddtionalDataSourcesIfEmpty() {
-        if (ixJoinOuterAdditionalDataSourceRefs == null) {
-            ixJoinOuterAdditionalDataSourceRefs = new ArrayList<Mutable<ILogicalOperator>>();
-            ixJoinOuterAdditionalDataSourceTypes = new ArrayList<DataSourceType>();
-            ixJoinOuterAdditionalDatasets = new ArrayList<Dataset>();
-            ixJoinOuterAdditionalRecordTypes = new ArrayList<ARecordType>();
+        if (getIxJoinOuterAdditionalDataSourceRefs() == null) {
+            setIxJoinOuterAdditionalDataSourceRefs(new ArrayList<Mutable<ILogicalOperator>>());
+            setIxJoinOuterAdditionalDataSourceTypes(new ArrayList<DataSourceType>());
+            setIxJoinOuterAdditionalDatasets(new ArrayList<Dataset>());
+            setIxJoinOuterAdditionalRecordTypes(new ArrayList<ARecordType>());
         }
     }
 
+    public ILogicalOperator getRoot() {
+        return root;
+    }
+
+    public void setRoot(ILogicalOperator root) {
+        this.root = root;
+    }
+
+    public Mutable<ILogicalOperator> getRootRef() {
+        return rootRef;
+    }
+
+    public void setRootRef(Mutable<ILogicalOperator> rootRef) {
+        this.rootRef = rootRef;
+    }
+
+    public List<Mutable<ILogicalOperator>> getAssignsAndUnnestsRefs() {
+        return assignsAndUnnestsRefs;
+    }
+
+    public List<AbstractLogicalOperator> getAssignsAndUnnests() {
+        return assignsAndUnnests;
+    }
+
+    public Mutable<ILogicalOperator> getDataSourceRef() {
+        return dataSourceRef;
+    }
+
+    public void setDataSourceRef(Mutable<ILogicalOperator> dataSourceRef) {
+        this.dataSourceRef = dataSourceRef;
+    }
+
+    public DataSourceType getDataSourceType() {
+        return dataSourceType;
+    }
+
+    public void setDataSourceType(DataSourceType dataSourceType) {
+        this.dataSourceType = dataSourceType;
+    }
+
+    public Dataset getDataset() {
+        return dataset;
+    }
+
+    public void setDataset(Dataset dataset) {
+        this.dataset = dataset;
+    }
+
+    public ARecordType getRecordType() {
+        return recordType;
+    }
+
+    public void setRecordType(ARecordType recordType) {
+        this.recordType = recordType;
+    }
+
+    public ARecordType getMetaRecordType() {
+        return metaRecordType;
+    }
+
+    public void setMetaRecordType(ARecordType metaRecordType) {
+        this.metaRecordType = metaRecordType;
+    }
+
+    public List<Mutable<ILogicalOperator>> getIxJoinOuterAdditionalDataSourceRefs() {
+        return ixJoinOuterAdditionalDataSourceRefs;
+    }
+
+    public void setIxJoinOuterAdditionalDataSourceRefs(
+            List<Mutable<ILogicalOperator>> ixJoinOuterAdditionalDataSourceRefs) {
+        this.ixJoinOuterAdditionalDataSourceRefs = ixJoinOuterAdditionalDataSourceRefs;
+    }
+
+    public List<DataSourceType> getIxJoinOuterAdditionalDataSourceTypes() {
+        return ixJoinOuterAdditionalDataSourceTypes;
+    }
+
+    public void setIxJoinOuterAdditionalDataSourceTypes(List<DataSourceType> ixJoinOuterAdditionalDataSourceTypes) {
+        this.ixJoinOuterAdditionalDataSourceTypes = ixJoinOuterAdditionalDataSourceTypes;
+    }
+
+    public List<Dataset> getIxJoinOuterAdditionalDatasets() {
+        return ixJoinOuterAdditionalDatasets;
+    }
+
+    public void setIxJoinOuterAdditionalDatasets(List<Dataset> ixJoinOuterAdditionalDatasets) {
+        this.ixJoinOuterAdditionalDatasets = ixJoinOuterAdditionalDatasets;
+    }
+
+    public List<ARecordType> getIxJoinOuterAdditionalRecordTypes() {
+        return ixJoinOuterAdditionalRecordTypes;
+    }
+
+    public void setIxJoinOuterAdditionalRecordTypes(List<ARecordType> ixJoinOuterAdditionalRecordTypes) {
+        this.ixJoinOuterAdditionalRecordTypes = ixJoinOuterAdditionalRecordTypes;
+    }
+
 }

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/RTreeAccessMethod.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/RTreeAccessMethod.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/RTreeAccessMethod.java
index eba2d33..c3c162e 100644
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/RTreeAccessMethod.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/RTreeAccessMethod.java
@@ -50,10 +50,8 @@ import org.apache.hyracks.algebricks.core.algebra.operators.logical.AbstractBina
 import org.apache.hyracks.algebricks.core.algebra.operators.logical.AbstractDataSourceOperator;
 import org.apache.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
 import org.apache.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator.ExecutionMode;
-import org.apache.hyracks.algebricks.core.algebra.operators.logical.AbstractUnnestMapOperator;
 import org.apache.hyracks.algebricks.core.algebra.operators.logical.AssignOperator;
 import org.apache.hyracks.algebricks.core.algebra.operators.logical.SelectOperator;
-import org.apache.hyracks.algebricks.core.algebra.operators.logical.UnnestMapOperator;
 import org.apache.hyracks.algebricks.core.algebra.util.OperatorManipulationUtil;
 
 /**
@@ -61,13 +59,13 @@ import org.apache.hyracks.algebricks.core.algebra.util.OperatorManipulationUtil;
  */
 public class RTreeAccessMethod implements IAccessMethod {
 
-    private static List<FunctionIdentifier> funcIdents = new ArrayList<FunctionIdentifier>();
+    private static List<FunctionIdentifier> funcIdents = new ArrayList<>();
 
     static {
         funcIdents.add(AsterixBuiltinFunctions.SPATIAL_INTERSECT);
     }
 
-    public static RTreeAccessMethod INSTANCE = new RTreeAccessMethod();
+    public static final RTreeAccessMethod INSTANCE = new RTreeAccessMethod();
 
     @Override
     public List<FunctionIdentifier> getOptimizableFunctions() {
@@ -107,7 +105,7 @@ public class RTreeAccessMethod implements IAccessMethod {
             return false;
         }
         // Replace the datasource scan with the new plan rooted at primaryIndexUnnestMap.
-        subTree.dataSourceRef.setValue(primaryIndexUnnestOp);
+        subTree.getDataSourceRef().setValue(primaryIndexUnnestOp);
         return true;
     }
 
@@ -125,16 +123,17 @@ public class RTreeAccessMethod implements IAccessMethod {
             OptimizableOperatorSubTree leftSubTree, OptimizableOperatorSubTree rightSubTree, Index chosenIndex,
             AccessMethodAnalysisContext analysisCtx, IOptimizationContext context, boolean isLeftOuterJoin,
             boolean hasGroupBy) throws AlgebricksException {
-        // Determine if the index is applicable on the left or right side (if both, we arbitrarily prefer the left side).
+        // Determine if the index is applicable on the left or right side (if both, we arbitrarily prefer the left
+        // side).
         Dataset dataset = analysisCtx.indexDatasetMap.get(chosenIndex);
-        OptimizableOperatorSubTree indexSubTree = null;
-        OptimizableOperatorSubTree probeSubTree = null;
+        OptimizableOperatorSubTree indexSubTree;
+        OptimizableOperatorSubTree probeSubTree;
 
         // We assume that the left subtree is the outer branch and the right subtree is the inner branch.
         // This assumption holds true since we only use an index from the right subtree.
         // The following is just a sanity check.
         if (rightSubTree.hasDataSourceScan()
-                && dataset.getDatasetName().equals(rightSubTree.dataset.getDatasetName())) {
+                && dataset.getDatasetName().equals(rightSubTree.getDataset().getDatasetName())) {
             indexSubTree = rightSubTree;
             probeSubTree = leftSubTree;
         } else {
@@ -143,8 +142,8 @@ public class RTreeAccessMethod implements IAccessMethod {
 
         LogicalVariable newNullPlaceHolderVar = null;
         if (isLeftOuterJoin) {
-            //get a new null place holder variable that is the first field variable of the primary key
-            //from the indexSubTree's datasourceScanOp
+            // get a new null place holder variable that is the first field variable of the primary key
+            // from the indexSubTree's datasourceScanOp
             newNullPlaceHolderVar = indexSubTree.getDataSourceVariables().get(0);
         }
 
@@ -156,15 +155,15 @@ public class RTreeAccessMethod implements IAccessMethod {
         }
 
         if (isLeftOuterJoin && hasGroupBy) {
-            //reset the null place holder variable
+            // reset the null place holder variable
             AccessMethodUtils.resetLOJNullPlaceholderVariableInGroupByOp(analysisCtx, newNullPlaceHolderVar, context);
         }
 
-        indexSubTree.dataSourceRef.setValue(primaryIndexUnnestOp);
+        indexSubTree.getDataSourceRef().setValue(primaryIndexUnnestOp);
         // Change join into a select with the same condition.
         AbstractBinaryJoinOperator joinOp = (AbstractBinaryJoinOperator) joinRef.getValue();
         SelectOperator topSelect = new SelectOperator(joinOp.getCondition(), isLeftOuterJoin, newNullPlaceHolderVar);
-        topSelect.getInputs().add(indexSubTree.rootRef);
+        topSelect.getInputs().add(indexSubTree.getRootRef());
         topSelect.setExecutionMode(ExecutionMode.LOCAL);
         context.computeAndSetTypeEnvironmentForOperator(topSelect);
         // Replace the original join with the new subtree rooted at the select op.
@@ -175,12 +174,12 @@ public class RTreeAccessMethod implements IAccessMethod {
     private ILogicalOperator createSecondaryToPrimaryPlan(OptimizableOperatorSubTree indexSubTree,
             OptimizableOperatorSubTree probeSubTree, Index chosenIndex, AccessMethodAnalysisContext analysisCtx,
             boolean retainInput, boolean retainNull, boolean requiresBroadcast, IOptimizationContext context)
-                    throws AlgebricksException {
+            throws AlgebricksException {
 
         IOptimizableFuncExpr optFuncExpr = AccessMethodUtils.chooseFirstOptFuncExpr(chosenIndex, analysisCtx);
-        Dataset dataset = indexSubTree.dataset;
-        ARecordType recordType = indexSubTree.recordType;
-        ARecordType metaRecordType = indexSubTree.metaRecordType;
+        Dataset dataset = indexSubTree.getDataset();
+        ARecordType recordType = indexSubTree.getRecordType();
+        ARecordType metaRecordType = indexSubTree.getMetaRecordType();
 
         int optFieldIdx = AccessMethodUtils.chooseFirstOptFuncVar(chosenIndex, analysisCtx);
         Pair<IAType, Boolean> keyPairType = Index.getNonNullableOpenFieldType(optFuncExpr.getFieldType(optFieldIdx),
@@ -194,16 +193,18 @@ public class RTreeAccessMethod implements IAccessMethod {
         int numDimensions = NonTaggedFormatUtil.getNumDimensions(spatialType.getTypeTag());
         int numSecondaryKeys = numDimensions * 2;
         // we made sure indexSubTree has datasource scan
-        AbstractDataSourceOperator dataSourceOp = (AbstractDataSourceOperator) indexSubTree.dataSourceRef.getValue();
+        AbstractDataSourceOperator dataSourceOp = (AbstractDataSourceOperator) indexSubTree.getDataSourceRef()
+                .getValue();
         RTreeJobGenParams jobGenParams = new RTreeJobGenParams(chosenIndex.getIndexName(), IndexType.RTREE,
                 dataset.getDataverseName(), dataset.getDatasetName(), retainInput, requiresBroadcast);
         // A spatial object is serialized in the constant of the func expr we are optimizing.
         // The R-Tree expects as input an MBR represented with 1 field per dimension.
-        // Here we generate vars and funcs for extracting MBR fields from the constant into fields of a tuple (as the R-Tree expects them).
+        // Here we generate vars and funcs for extracting MBR fields from the constant into fields of a tuple (as the
+        // R-Tree expects them).
         // List of variables for the assign.
-        ArrayList<LogicalVariable> keyVarList = new ArrayList<LogicalVariable>();
+        ArrayList<LogicalVariable> keyVarList = new ArrayList<>();
         // List of expressions for the assign.
-        ArrayList<Mutable<ILogicalExpression>> keyExprList = new ArrayList<Mutable<ILogicalExpression>>();
+        ArrayList<Mutable<ILogicalExpression>> keyExprList = new ArrayList<>();
         Pair<ILogicalExpression, Boolean> returnedSearchKeyExpr = AccessMethodUtils.createSearchKeyExpr(optFuncExpr,
                 indexSubTree, probeSubTree);
         ILogicalExpression searchKeyExpr = returnedSearchKeyExpr.first;
@@ -237,23 +238,18 @@ public class RTreeAccessMethod implements IAccessMethod {
             assignSearchKeys.setExecutionMode(dataSourceOp.getExecutionMode());
         } else {
             // We are optimizing a join, place the assign op top of the probe subtree.
-            assignSearchKeys.getInputs().add(probeSubTree.rootRef);
+            assignSearchKeys.getInputs().add(probeSubTree.getRootRef());
         }
 
         ILogicalOperator secondaryIndexUnnestOp = AccessMethodUtils.createSecondaryIndexUnnestMap(dataset, recordType,
                 metaRecordType, chosenIndex, assignSearchKeys, jobGenParams, context, false, retainInput, retainNull);
 
         // Generate the rest of the upstream plan which feeds the search results into the primary index.
-        if (dataset.getDatasetType() == DatasetType.EXTERNAL) {
-            UnnestMapOperator externalDataAccessOp = AccessMethodUtils.createExternalDataLookupUnnestMap(dataSourceOp,
-                    dataset, recordType, secondaryIndexUnnestOp, context, chosenIndex, retainInput, retainNull);
-            return externalDataAccessOp;
-        } else {
-            AbstractUnnestMapOperator primaryIndexUnnestOp = AccessMethodUtils.createPrimaryIndexUnnestMap(dataSourceOp,
-                    dataset, recordType, metaRecordType, secondaryIndexUnnestOp, context, true, retainInput, false,
-                    false);
-            return primaryIndexUnnestOp;
-        }
+        return dataset.getDatasetType() == DatasetType.EXTERNAL
+                ? AccessMethodUtils.createExternalDataLookupUnnestMap(dataSourceOp, dataset, recordType,
+                        secondaryIndexUnnestOp, context, chosenIndex, retainInput, retainNull)
+                : AccessMethodUtils.createPrimaryIndexUnnestMap(dataSourceOp, dataset, recordType, metaRecordType,
+                        secondaryIndexUnnestOp, context, true, retainInput, false, false);
     }
 
     @Override

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/AbstractLangTranslator.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/AbstractLangTranslator.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/AbstractLangTranslator.java
index 9030da9..bf7b975 100644
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/AbstractLangTranslator.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/AbstractLangTranslator.java
@@ -24,18 +24,18 @@ import java.util.logging.Level;
 import java.util.logging.Logger;
 
 import org.apache.asterix.common.api.IClusterManagementWork.ClusterState;
-import org.apache.asterix.common.config.MetadataConstants;
 import org.apache.asterix.common.exceptions.AsterixException;
 import org.apache.asterix.lang.common.base.Statement;
 import org.apache.asterix.lang.common.statement.DatasetDecl;
 import org.apache.asterix.lang.common.statement.DataverseDropStatement;
 import org.apache.asterix.lang.common.statement.DeleteStatement;
-import org.apache.asterix.lang.common.statement.DropStatement;
+import org.apache.asterix.lang.common.statement.DropDatasetStatement;
 import org.apache.asterix.lang.common.statement.InsertStatement;
 import org.apache.asterix.lang.common.statement.NodeGroupDropStatement;
 import org.apache.asterix.metadata.dataset.hints.DatasetHints;
 import org.apache.asterix.metadata.entities.AsterixBuiltinTypeMap;
 import org.apache.asterix.metadata.entities.Dataverse;
+import org.apache.asterix.metadata.utils.MetadataConstants;
 import org.apache.asterix.om.types.BuiltinType;
 import org.apache.asterix.om.util.AsterixAppContextInfo;
 import org.apache.asterix.om.util.AsterixClusterProperties;
@@ -149,7 +149,7 @@ public abstract class AbstractLangTranslator {
                 break;
 
             case Statement.Kind.DATASET_DROP:
-                DropStatement dropStmt = (DropStatement) stmt;
+                DropDatasetStatement dropStmt = (DropDatasetStatement) stmt;
                 if (dropStmt.getDataverseName() != null) {
                     dataverse = dropStmt.getDataverseName().getValue();
                 }

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/AqlExpressionToPlanTranslator.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/AqlExpressionToPlanTranslator.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/AqlExpressionToPlanTranslator.java
index 79342c9..5c09978 100644
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/AqlExpressionToPlanTranslator.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/AqlExpressionToPlanTranslator.java
@@ -30,8 +30,8 @@ import org.apache.asterix.lang.aql.expression.UnionExpr;
 import org.apache.asterix.lang.aql.visitor.base.IAQLVisitor;
 import org.apache.asterix.lang.common.base.Clause;
 import org.apache.asterix.lang.common.base.Expression;
-import org.apache.asterix.lang.common.base.ILangExpression;
 import org.apache.asterix.lang.common.base.Expression.Kind;
+import org.apache.asterix.lang.common.base.ILangExpression;
 import org.apache.asterix.lang.common.expression.VariableExpr;
 import org.apache.asterix.lang.common.statement.Query;
 import org.apache.asterix.metadata.declared.AqlMetadataProvider;

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/AqlPlusExpressionToPlanTranslator.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/AqlPlusExpressionToPlanTranslator.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/AqlPlusExpressionToPlanTranslator.java
index 334436a..05bd343 100644
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/AqlPlusExpressionToPlanTranslator.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/AqlPlusExpressionToPlanTranslator.java
@@ -81,7 +81,7 @@ import org.apache.asterix.lang.common.statement.DataverseDecl;
 import org.apache.asterix.lang.common.statement.DataverseDropStatement;
 import org.apache.asterix.lang.common.statement.DeleteStatement;
 import org.apache.asterix.lang.common.statement.DisconnectFeedStatement;
-import org.apache.asterix.lang.common.statement.DropStatement;
+import org.apache.asterix.lang.common.statement.DropDatasetStatement;
 import org.apache.asterix.lang.common.statement.FeedDropStatement;
 import org.apache.asterix.lang.common.statement.FeedPolicyDropStatement;
 import org.apache.asterix.lang.common.statement.FunctionDecl;
@@ -1218,7 +1218,7 @@ public class AqlPlusExpressionToPlanTranslator extends AbstractLangTranslator
     }
 
     @Override
-    public Pair<ILogicalOperator, LogicalVariable> visit(DropStatement del, Mutable<ILogicalOperator> arg)
+    public Pair<ILogicalOperator, LogicalVariable> visit(DropDatasetStatement del, Mutable<ILogicalOperator> arg)
             throws AsterixException {
         // TODO Auto-generated method stub
         return null;

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/IStatementExecutor.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/IStatementExecutor.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/IStatementExecutor.java
new file mode 100644
index 0000000..3c24cbc
--- /dev/null
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/IStatementExecutor.java
@@ -0,0 +1,129 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.translator;
+
+import java.rmi.RemoteException;
+
+import org.apache.asterix.common.exceptions.ACIDException;
+import org.apache.asterix.common.exceptions.AsterixException;
+import org.apache.asterix.lang.common.statement.Query;
+import org.apache.asterix.metadata.declared.AqlMetadataProvider;
+import org.apache.asterix.translator.CompiledStatements.ICompiledDmlStatement;
+import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
+import org.apache.hyracks.api.client.IHyracksClientConnection;
+import org.apache.hyracks.api.dataset.IHyracksDataset;
+import org.apache.hyracks.api.job.JobSpecification;
+import org.json.JSONException;
+
+/**
+ * An interface that takes care of executing a list of statements that are submitted through an Asterix API
+ */
+public interface IStatementExecutor {
+
+    /**
+     * Specifies result delivery of executed statements
+     */
+    public enum ResultDelivery {
+        /**
+         * Wait for results to be read
+         */
+        SYNC,
+        /**
+         * Flush out result handle beofre waiting for the result
+         */
+        ASYNC,
+        /**
+         * Return result handle and don't wait for the result
+         */
+        ASYNC_DEFERRED
+    }
+
+    public static class Stats {
+        private long count;
+        private long size;
+
+        public long getCount() {
+            return count;
+        }
+
+        public void setCount(long count) {
+            this.count = count;
+        }
+
+        public long getSize() {
+            return size;
+        }
+
+        public void setSize(long size) {
+            this.size = size;
+        }
+
+    }
+
+    /**
+     * Compiles and execute a list of statements.
+     *
+     * @param hcc
+     *            A Hyracks client connection that is used to submit a jobspec to Hyracks.
+     * @param hdc
+     *            A Hyracks dataset client object that is used to read the results.
+     * @param resultDelivery
+     *            The {@code ResultDelivery} kind required for queries in the list of statements
+     * @throws Exception
+     */
+    void compileAndExecute(IHyracksClientConnection hcc, IHyracksDataset hdc, ResultDelivery resultDelivery)
+            throws Exception;
+
+    /**
+     * Compiles and execute a list of statements.
+     *
+     * @param hcc
+     *            A Hyracks client connection that is used to submit a jobspec to Hyracks.
+     * @param hdc
+     *            A Hyracks dataset client object that is used to read the results.
+     * @param resultDelivery
+     *            The {@code ResultDelivery} kind required for queries in the list of statements
+     * @param stats
+     *            a reference to write the stats of executed queries
+     * @throws Exception
+     */
+    void compileAndExecute(IHyracksClientConnection hcc, IHyracksDataset hdc, ResultDelivery resultDelivery,
+            Stats stats) throws Exception;
+
+    /**
+     * rewrites and compiles query into a hyracks job specifications
+     *
+     * @param metadataProvider
+     *            The metadataProvider used to access metadata and build runtimes
+     * @param query
+     *            The query to be compiled
+     * @param dmlStatement
+     *            The data modification statement when the query results in a modification to a dataset
+     * @return the compiled {@code JobSpecification}
+     * @throws AsterixException
+     * @throws RemoteException
+     * @throws AlgebricksException
+     * @throws JSONException
+     * @throws ACIDException
+     */
+    JobSpecification rewriteCompileQuery(AqlMetadataProvider metadataProvider, Query query,
+            ICompiledDmlStatement dmlStatement)
+            throws AsterixException, RemoteException, AlgebricksException, JSONException, ACIDException;
+
+}

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/IStatementExecutorFactory.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/IStatementExecutorFactory.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/IStatementExecutorFactory.java
new file mode 100644
index 0000000..9addc87
--- /dev/null
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/IStatementExecutorFactory.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.translator;
+
+import java.util.List;
+
+import org.apache.asterix.common.app.SessionConfig;
+import org.apache.asterix.compiler.provider.ILangCompilationProvider;
+import org.apache.asterix.lang.common.base.Statement;
+
+/**
+ * An interface for creating a statement executor which can be provided by
+ * implementing {@code IStatementExecutorExtension}
+ */
+@FunctionalInterface
+public interface IStatementExecutorFactory {
+
+    /**
+     * create a statement executor
+     *
+     * @param statements
+     *            Statements to execute
+     * @param conf
+     *            request configuration
+     * @param compilationProvider
+     * @return an implementation of {@code IStatementExecutor} that is used to execute the passed list of statements
+     */
+    IStatementExecutor create(List<Statement> statements, SessionConfig conf,
+            ILangCompilationProvider compilationProvider);
+}

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/LangExpressionToPlanTranslator.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/LangExpressionToPlanTranslator.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/LangExpressionToPlanTranslator.java
index bd87e6e..ed9c1e6 100644
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/LangExpressionToPlanTranslator.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/LangExpressionToPlanTranslator.java
@@ -18,8 +18,6 @@
  */
 package org.apache.asterix.translator;
 
-import static java.util.logging.Logger.global;
-
 import java.io.File;
 import java.io.IOException;
 import java.util.ArrayList;
@@ -29,8 +27,8 @@ import java.util.HashSet;
 import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
-import java.util.Set;
 import java.util.Map.Entry;
+import java.util.Set;
 import java.util.concurrent.atomic.AtomicLong;
 
 import org.apache.asterix.algebra.base.ILangExpressionToPlanTranslator;
@@ -41,15 +39,15 @@ import org.apache.asterix.common.functions.FunctionConstants;
 import org.apache.asterix.common.functions.FunctionSignature;
 import org.apache.asterix.lang.aql.util.RangeMapBuilder;
 import org.apache.asterix.lang.common.base.Expression;
+import org.apache.asterix.lang.common.base.Expression.Kind;
 import org.apache.asterix.lang.common.base.ILangExpression;
 import org.apache.asterix.lang.common.base.Statement;
-import org.apache.asterix.lang.common.base.Expression.Kind;
 import org.apache.asterix.lang.common.clause.GroupbyClause;
 import org.apache.asterix.lang.common.clause.LetClause;
 import org.apache.asterix.lang.common.clause.LimitClause;
 import org.apache.asterix.lang.common.clause.OrderbyClause;
-import org.apache.asterix.lang.common.clause.WhereClause;
 import org.apache.asterix.lang.common.clause.OrderbyClause.OrderModifier;
+import org.apache.asterix.lang.common.clause.WhereClause;
 import org.apache.asterix.lang.common.expression.CallExpr;
 import org.apache.asterix.lang.common.expression.FieldAccessor;
 import org.apache.asterix.lang.common.expression.FieldBinding;
@@ -57,14 +55,14 @@ import org.apache.asterix.lang.common.expression.GbyVariableExpressionPair;
 import org.apache.asterix.lang.common.expression.IfExpr;
 import org.apache.asterix.lang.common.expression.IndexAccessor;
 import org.apache.asterix.lang.common.expression.ListConstructor;
+import org.apache.asterix.lang.common.expression.ListConstructor.Type;
 import org.apache.asterix.lang.common.expression.LiteralExpr;
 import org.apache.asterix.lang.common.expression.OperatorExpr;
 import org.apache.asterix.lang.common.expression.QuantifiedExpression;
+import org.apache.asterix.lang.common.expression.QuantifiedExpression.Quantifier;
 import org.apache.asterix.lang.common.expression.RecordConstructor;
 import org.apache.asterix.lang.common.expression.UnaryExpr;
 import org.apache.asterix.lang.common.expression.VariableExpr;
-import org.apache.asterix.lang.common.expression.ListConstructor.Type;
-import org.apache.asterix.lang.common.expression.QuantifiedExpression.Quantifier;
 import org.apache.asterix.lang.common.literal.StringLiteral;
 import org.apache.asterix.lang.common.statement.FunctionDecl;
 import org.apache.asterix.lang.common.statement.Query;
@@ -75,13 +73,13 @@ import org.apache.asterix.lang.common.util.FunctionUtil;
 import org.apache.asterix.lang.common.visitor.base.AbstractQueryExpressionVisitor;
 import org.apache.asterix.metadata.MetadataException;
 import org.apache.asterix.metadata.MetadataManager;
+import org.apache.asterix.metadata.declared.AqlDataSource.AqlDataSourceType;
 import org.apache.asterix.metadata.declared.AqlMetadataProvider;
 import org.apache.asterix.metadata.declared.AqlSourceId;
 import org.apache.asterix.metadata.declared.DatasetDataSource;
 import org.apache.asterix.metadata.declared.LoadableDataSource;
 import org.apache.asterix.metadata.declared.ResultSetDataSink;
 import org.apache.asterix.metadata.declared.ResultSetSinkId;
-import org.apache.asterix.metadata.declared.AqlDataSource.AqlDataSourceType;
 import org.apache.asterix.metadata.entities.Dataset;
 import org.apache.asterix.metadata.entities.Function;
 import org.apache.asterix.metadata.entities.InternalDatasetDetails;
@@ -117,15 +115,15 @@ import org.apache.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
 import org.apache.hyracks.algebricks.core.algebra.base.LogicalVariable;
 import org.apache.hyracks.algebricks.core.algebra.base.OperatorAnnotations;
 import org.apache.hyracks.algebricks.core.algebra.expressions.AbstractFunctionCallExpression;
+import org.apache.hyracks.algebricks.core.algebra.expressions.AbstractFunctionCallExpression.FunctionKind;
 import org.apache.hyracks.algebricks.core.algebra.expressions.AggregateFunctionCallExpression;
 import org.apache.hyracks.algebricks.core.algebra.expressions.BroadcastExpressionAnnotation;
+import org.apache.hyracks.algebricks.core.algebra.expressions.BroadcastExpressionAnnotation.BroadcastSide;
 import org.apache.hyracks.algebricks.core.algebra.expressions.ConstantExpression;
 import org.apache.hyracks.algebricks.core.algebra.expressions.IExpressionAnnotation;
 import org.apache.hyracks.algebricks.core.algebra.expressions.ScalarFunctionCallExpression;
 import org.apache.hyracks.algebricks.core.algebra.expressions.UnnestingFunctionCallExpression;
 import org.apache.hyracks.algebricks.core.algebra.expressions.VariableReferenceExpression;
-import org.apache.hyracks.algebricks.core.algebra.expressions.AbstractFunctionCallExpression.FunctionKind;
-import org.apache.hyracks.algebricks.core.algebra.expressions.BroadcastExpressionAnnotation.BroadcastSide;
 import org.apache.hyracks.algebricks.core.algebra.functions.AlgebricksBuiltinFunctions;
 import org.apache.hyracks.algebricks.core.algebra.functions.FunctionIdentifier;
 import org.apache.hyracks.algebricks.core.algebra.functions.IFunctionInfo;
@@ -141,13 +139,13 @@ import org.apache.hyracks.algebricks.core.algebra.operators.logical.InsertDelete
 import org.apache.hyracks.algebricks.core.algebra.operators.logical.LimitOperator;
 import org.apache.hyracks.algebricks.core.algebra.operators.logical.NestedTupleSourceOperator;
 import org.apache.hyracks.algebricks.core.algebra.operators.logical.OrderOperator;
+import org.apache.hyracks.algebricks.core.algebra.operators.logical.OrderOperator.IOrder.OrderKind;
 import org.apache.hyracks.algebricks.core.algebra.operators.logical.ProjectOperator;
 import org.apache.hyracks.algebricks.core.algebra.operators.logical.SelectOperator;
 import org.apache.hyracks.algebricks.core.algebra.operators.logical.SinkOperator;
 import org.apache.hyracks.algebricks.core.algebra.operators.logical.SubplanOperator;
 import org.apache.hyracks.algebricks.core.algebra.operators.logical.UnionAllOperator;
 import org.apache.hyracks.algebricks.core.algebra.operators.logical.UnnestOperator;
-import org.apache.hyracks.algebricks.core.algebra.operators.logical.OrderOperator.IOrder.OrderKind;
 import org.apache.hyracks.algebricks.core.algebra.operators.logical.visitors.LogicalOperatorDeepCopyWithNewVariablesVisitor;
 import org.apache.hyracks.algebricks.core.algebra.operators.logical.visitors.VariableUtilities;
 import org.apache.hyracks.algebricks.core.algebra.plan.ALogicalPlanImpl;
@@ -198,10 +196,10 @@ class LangExpressionToPlanTranslator
                     "Unable to load dataset " + clffs.getDatasetName() + " since it does not exist");
         }
         IAType itemType = metadataProvider.findType(dataset.getItemTypeDataverseName(), dataset.getItemTypeName());
-        IAType metaItemType =
-                metadataProvider.findType(dataset.getMetaItemTypeDataverseName(), dataset.getMetaItemTypeName());
-        DatasetDataSource targetDatasource =
-                validateDatasetInfo(metadataProvider, stmt.getDataverseName(), stmt.getDatasetName());
+        IAType metaItemType = metadataProvider.findType(dataset.getMetaItemTypeDataverseName(),
+                dataset.getMetaItemTypeName());
+        DatasetDataSource targetDatasource = validateDatasetInfo(metadataProvider, stmt.getDataverseName(),
+                stmt.getDatasetName());
         List<List<String>> partitionKeys = DatasetUtils.getPartitioningKeys(targetDatasource.getDataset());
         if (dataset.hasMetaPart()) {
             throw new AlgebricksException(
@@ -264,8 +262,8 @@ class LangExpressionToPlanTranslator
             additionalFilteringExpressions = new ArrayList<>();
             PlanTranslationUtil.prepareVarAndExpression(additionalFilteringField, payloadVar, additionalFilteringVars,
                     additionalFilteringAssignExpressions, additionalFilteringExpressions, context);
-            additionalFilteringAssign =
-                    new AssignOperator(additionalFilteringVars, additionalFilteringAssignExpressions);
+            additionalFilteringAssign = new AssignOperator(additionalFilteringVars,
+                    additionalFilteringAssignExpressions);
         }
 
         InsertDeleteUpsertOperator insertOp = new InsertDeleteUpsertOperator(targetDatasource, payloadRef,
@@ -284,12 +282,11 @@ class LangExpressionToPlanTranslator
         return new ALogicalPlanImpl(new MutableObject<>(leafOperator));
     }
 
-    @SuppressWarnings("unchecked")
     @Override
     public ILogicalPlan translate(Query expr, String outputDatasetName, ICompiledDmlStatement stmt)
             throws AlgebricksException {
-        Pair<ILogicalOperator, LogicalVariable> p =
-                expr.accept(this, new MutableObject<>(new EmptyTupleSourceOperator()));
+        Pair<ILogicalOperator, LogicalVariable> p = expr.accept(this,
+                new MutableObject<>(new EmptyTupleSourceOperator()));
         ArrayList<Mutable<ILogicalOperator>> globalPlanRoots = new ArrayList<>();
         ILogicalOperator topOp = p.first;
         ProjectOperator project = (ProjectOperator) topOp;
@@ -322,7 +319,10 @@ class LangExpressionToPlanTranslator
              * because dataset only accept non-collection records
              */
             LogicalVariable seqVar = context.newVar();
-            /** This assign adds a marker function collection-to-sequence: if the input is a singleton collection, unnest it; otherwise do nothing. */
+            /**
+             * This assign adds a marker function collection-to-sequence: if the input is a singleton collection, unnest
+             * it; otherwise do nothing.
+             */
             AssignOperator assignCollectionToSequence = new AssignOperator(seqVar,
                     new MutableObject<>(new ScalarFunctionCallExpression(
                             FunctionUtil.getFunctionInfo(AsterixBuiltinFunctions.COLLECTION_TO_SEQUENCE),
@@ -331,11 +331,10 @@ class LangExpressionToPlanTranslator
             project.getInputs().get(0).setValue(assignCollectionToSequence);
             project.getVariables().set(0, seqVar);
             resVar = seqVar;
-            DatasetDataSource targetDatasource =
-                    validateDatasetInfo(metadataProvider, stmt.getDataverseName(), stmt.getDatasetName());
-            List<Integer> keySourceIndicator =
-                    ((InternalDatasetDetails) targetDatasource.getDataset().getDatasetDetails())
-                            .getKeySourceIndicator();
+            DatasetDataSource targetDatasource = validateDatasetInfo(metadataProvider, stmt.getDataverseName(),
+                    stmt.getDatasetName());
+            List<Integer> keySourceIndicator = ((InternalDatasetDetails) targetDatasource.getDataset()
+                    .getDatasetDetails()).getKeySourceIndicator();
             ArrayList<LogicalVariable> vars = new ArrayList<>();
             ArrayList<Mutable<ILogicalExpression>> exprs = new ArrayList<>();
             List<Mutable<ILogicalExpression>> varRefsForLoading = new ArrayList<>();
@@ -367,8 +366,8 @@ class LangExpressionToPlanTranslator
                 PlanTranslationUtil.prepareVarAndExpression(additionalFilteringField, resVar, additionalFilteringVars,
                         additionalFilteringAssignExpressions, additionalFilteringExpressions, context);
 
-                additionalFilteringAssign =
-                        new AssignOperator(additionalFilteringVars, additionalFilteringAssignExpressions);
+                additionalFilteringAssign = new AssignOperator(additionalFilteringVars,
+                        additionalFilteringAssignExpressions);
                 additionalFilteringAssign.getInputs().add(new MutableObject<>(project));
                 assign.getInputs().add(new MutableObject<>(additionalFilteringAssign));
             } else {
@@ -384,7 +383,8 @@ class LangExpressionToPlanTranslator
                     break;
                 case Statement.Kind.UPSERT:
                     leafOperator = translateUpsert(targetDatasource, varRef, varRefsForLoading,
-                            additionalFilteringExpressions, assign, additionalFilteringField);
+                            additionalFilteringExpressions, assign, additionalFilteringField, unnestVar, project, exprs,
+                            resVar, additionalFilteringAssign);
                     break;
                 case Statement.Kind.DELETE:
                     leafOperator = translateDelete(targetDatasource, varRef, varRefsForLoading,
@@ -451,8 +451,8 @@ class LangExpressionToPlanTranslator
         List<LogicalVariable> metaAndKeysVars = null;
         List<Mutable<ILogicalExpression>> metaAndKeysExprs = null;
         List<Mutable<ILogicalExpression>> metaExpSingletonList = null;
-        boolean isChangeFeed =
-                FeedMetadataUtil.isChangeFeed(metadataProvider, sfs.getDataverseName(), sfs.getFeedName());
+        boolean isChangeFeed = FeedMetadataUtil.isChangeFeed(metadataProvider, sfs.getDataverseName(),
+                sfs.getFeedName());
         if (targetDatasource.getDataset().hasMetaPart() || isChangeFeed) {
             metaAndKeysVars = new ArrayList<>();
             metaAndKeysExprs = new ArrayList<>();
@@ -527,26 +527,94 @@ class LangExpressionToPlanTranslator
     private ILogicalOperator translateUpsert(DatasetDataSource targetDatasource, Mutable<ILogicalExpression> varRef,
             List<Mutable<ILogicalExpression>> varRefsForLoading,
             List<Mutable<ILogicalExpression>> additionalFilteringExpressions, ILogicalOperator assign,
-            List<String> additionalFilteringField) throws AlgebricksException {
-        if (targetDatasource.getDataset().hasMetaPart()) {
+            List<String> additionalFilteringField, LogicalVariable unnestVar, ProjectOperator project,
+            List<Mutable<ILogicalExpression>> exprs, LogicalVariable resVar, AssignOperator additionalFilteringAssign)
+            throws AlgebricksException {
+        if (!targetDatasource.getDataset().allow(project, Dataset.OP_UPSERT)) {
             throw new AlgebricksException(targetDatasource.getDataset().getDatasetName()
                     + ": upsert into dataset is not supported on Datasets with Meta records");
         }
-        InsertDeleteUpsertOperator upsertOp = new InsertDeleteUpsertOperator(targetDatasource, varRef,
-                varRefsForLoading, InsertDeleteUpsertOperator.Kind.UPSERT, false);
-        upsertOp.setAdditionalFilteringExpressions(additionalFilteringExpressions);
-        upsertOp.getInputs().add(new MutableObject<>(assign));
-        // Create and add a new variable used for representing the original record
-        ARecordType recordType = (ARecordType) targetDatasource.getItemType();
-        upsertOp.setPrevRecordVar(context.newVar());
-        upsertOp.setPrevRecordType(recordType);
-        if (additionalFilteringField != null) {
-            upsertOp.setPrevFilterVar(context.newVar());
-            upsertOp.setPrevFilterType(recordType.getFieldType(additionalFilteringField.get(0)));
+        if (targetDatasource.getDataset().hasMetaPart()) {
+            InsertDeleteUpsertOperator feedModificationOp;
+            AssignOperator metaAndKeysAssign;
+            List<LogicalVariable> metaAndKeysVars;
+            List<Mutable<ILogicalExpression>> metaAndKeysExprs;
+            List<Mutable<ILogicalExpression>> metaExpSingletonList;
+            metaAndKeysVars = new ArrayList<>();
+            metaAndKeysExprs = new ArrayList<>();
+            // add the meta function
+            IFunctionInfo finfoMeta = FunctionUtil.getFunctionInfo(AsterixBuiltinFunctions.META);
+            ScalarFunctionCallExpression metaFunction = new ScalarFunctionCallExpression(finfoMeta,
+                    new MutableObject<>(new VariableReferenceExpression(unnestVar)));
+            // create assign for the meta part
+            LogicalVariable metaVar = context.newVar();
+            metaExpSingletonList = new ArrayList<>(1);
+            metaExpSingletonList.add(new MutableObject<>(new VariableReferenceExpression(metaVar)));
+            metaAndKeysVars.add(metaVar);
+            metaAndKeysExprs.add(new MutableObject<>(metaFunction));
+            project.getVariables().add(metaVar);
+            varRefsForLoading.clear();
+            for (Mutable<ILogicalExpression> assignExpr : exprs) {
+                if (assignExpr.getValue().getExpressionTag() == LogicalExpressionTag.FUNCTION_CALL) {
+                    AbstractFunctionCallExpression funcCall = (AbstractFunctionCallExpression) assignExpr.getValue();
+                    funcCall.substituteVar(resVar, unnestVar);
+                    LogicalVariable pkVar = context.newVar();
+                    metaAndKeysVars.add(pkVar);
+                    metaAndKeysExprs.add(new MutableObject<>(assignExpr.getValue()));
+                    project.getVariables().add(pkVar);
+                    varRefsForLoading.add(new MutableObject<>(new VariableReferenceExpression(pkVar)));
+                }
+            }
+            // A change feed, we don't need the assign to access PKs
+            feedModificationOp = new InsertDeleteUpsertOperator(targetDatasource, varRef, varRefsForLoading,
+                    metaExpSingletonList, InsertDeleteUpsertOperator.Kind.UPSERT, false);
+            // Create and add a new variable used for representing the original record
+            feedModificationOp.setPrevRecordVar(context.newVar());
+            feedModificationOp.setPrevRecordType(targetDatasource.getItemType());
+            if (targetDatasource.getDataset().hasMetaPart()) {
+                List<LogicalVariable> metaVars = new ArrayList<>();
+                metaVars.add(context.newVar());
+                feedModificationOp.setPrevAdditionalNonFilteringVars(metaVars);
+                List<Object> metaTypes = new ArrayList<>();
+                metaTypes.add(targetDatasource.getMetaItemType());
+                feedModificationOp.setPrevAdditionalNonFilteringTypes(metaTypes);
+            }
+
+            if (additionalFilteringField != null) {
+                feedModificationOp.setPrevFilterVar(context.newVar());
+                feedModificationOp.setPrevFilterType(
+                        ((ARecordType) targetDatasource.getItemType()).getFieldType(additionalFilteringField.get(0)));
+                additionalFilteringAssign.getInputs().clear();
+                additionalFilteringAssign.getInputs().add(assign.getInputs().get(0));
+                feedModificationOp.getInputs().add(new MutableObject<>(additionalFilteringAssign));
+            } else {
+                feedModificationOp.getInputs().add(assign.getInputs().get(0));
+            }
+            metaAndKeysAssign = new AssignOperator(metaAndKeysVars, metaAndKeysExprs);
+            metaAndKeysAssign.getInputs().add(project.getInputs().get(0));
+            project.getInputs().set(0, new MutableObject<>(metaAndKeysAssign));
+            feedModificationOp.setAdditionalFilteringExpressions(additionalFilteringExpressions);
+            SinkOperator leafOperator = new SinkOperator();
+            leafOperator.getInputs().add(new MutableObject<>(feedModificationOp));
+            return leafOperator;
+        } else {
+            InsertDeleteUpsertOperator feedModificationOp;
+            feedModificationOp = new InsertDeleteUpsertOperator(targetDatasource, varRef, varRefsForLoading,
+                    InsertDeleteUpsertOperator.Kind.UPSERT, false);
+            feedModificationOp.setAdditionalFilteringExpressions(additionalFilteringExpressions);
+            feedModificationOp.getInputs().add(new MutableObject<>(assign));
+            // Create and add a new variable used for representing the original record
+            ARecordType recordType = (ARecordType) targetDatasource.getItemType();
+            feedModificationOp.setPrevRecordVar(context.newVar());
+            feedModificationOp.setPrevRecordType(recordType);
+            if (additionalFilteringField != null) {
+                feedModificationOp.setPrevFilterVar(context.newVar());
+                feedModificationOp.setPrevFilterType(recordType.getFieldType(additionalFilteringField.get(0)));
+            }
+            SinkOperator leafOperator = new SinkOperator();
+            leafOperator.getInputs().add(new MutableObject<>(feedModificationOp));
+            return leafOperator;
         }
-        SinkOperator leafOperator = new SinkOperator();
-        leafOperator.getInputs().add(new MutableObject<>(upsertOp));
-        return leafOperator;
     }
 
     private ILogicalOperator translateInsert(DatasetDataSource targetDatasource, Mutable<ILogicalExpression> varRef,
@@ -577,8 +645,8 @@ class LangExpressionToPlanTranslator
         }
         AqlSourceId sourceId = new AqlSourceId(dataverseName, datasetName);
         IAType itemType = metadataProvider.findType(dataset.getItemTypeDataverseName(), dataset.getItemTypeName());
-        IAType metaItemType =
-                metadataProvider.findType(dataset.getMetaItemTypeDataverseName(), dataset.getMetaItemTypeName());
+        IAType metaItemType = metadataProvider.findType(dataset.getMetaItemTypeDataverseName(),
+                dataset.getMetaItemTypeName());
         INodeDomain domain = metadataProvider.findNodeDomain(dataset.getNodeGroupName());
         return new DatasetDataSource(sourceId, dataset, itemType, metaItemType, AqlDataSourceType.INTERNAL_DATASET,
                 dataset.getDatasetDetails(), domain);
@@ -586,8 +654,8 @@ class LangExpressionToPlanTranslator
 
     private FileSplit getDefaultOutputFileLocation() throws MetadataException {
         String outputDir = System.getProperty("java.io.tmpDir");
-        String filePath =
-                outputDir + System.getProperty("file.separator") + OUTPUT_FILE_PREFIX + outputFileID.incrementAndGet();
+        String filePath = outputDir + System.getProperty("file.separator") + OUTPUT_FILE_PREFIX
+                + outputFileID.incrementAndGet();
         AsterixMetadataProperties metadataProperties = AsterixAppContextInfo.getInstance().getMetadataProperties();
         return new FileSplit(metadataProperties.getMetadataNodeName(), new FileReference(new File(filePath)));
     }
@@ -604,8 +672,8 @@ class LangExpressionToPlanTranslator
             returnedOp.getInputs().add(tupSource);
         } else {
             v = context.newVar(lc.getVarExpr());
-            Pair<ILogicalExpression, Mutable<ILogicalOperator>> eo =
-                    langExprToAlgExpression(lc.getBindingExpr(), tupSource);
+            Pair<ILogicalExpression, Mutable<ILogicalOperator>> eo = langExprToAlgExpression(lc.getBindingExpr(),
+                    tupSource);
             returnedOp = new AssignOperator(v, new MutableObject<>(eo.first));
             returnedOp.getInputs().add(eo.second);
         }
@@ -620,8 +688,8 @@ class LangExpressionToPlanTranslator
         AbstractFunctionCallExpression fldAccess = new ScalarFunctionCallExpression(
                 FunctionUtil.getFunctionInfo(AsterixBuiltinFunctions.FIELD_ACCESS_BY_NAME));
         fldAccess.getArguments().add(new MutableObject<>(p.first));
-        ILogicalExpression faExpr =
-                new ConstantExpression(new AsterixConstantValue(new AString(fa.getIdent().getValue())));
+        ILogicalExpression faExpr = new ConstantExpression(
+                new AsterixConstantValue(new AString(fa.getIdent().getValue())));
         fldAccess.getArguments().add(new MutableObject<>(faExpr));
         AssignOperator a = new AssignOperator(v, new MutableObject<>(fldAccess));
         a.getInputs().add(p.second);
@@ -639,8 +707,8 @@ class LangExpressionToPlanTranslator
                     FunctionUtil.getFunctionInfo(AsterixBuiltinFunctions.ANY_COLLECTION_MEMBER));
             f.getArguments().add(new MutableObject<>(p.first));
         } else {
-            Pair<ILogicalExpression, Mutable<ILogicalOperator>> indexPair =
-                    langExprToAlgExpression(ia.getIndexExpr(), tupSource);
+            Pair<ILogicalExpression, Mutable<ILogicalOperator>> indexPair = langExprToAlgExpression(ia.getIndexExpr(),
+                    tupSource);
             f = new ScalarFunctionCallExpression(FunctionUtil.getFunctionInfo(AsterixBuiltinFunctions.GET_ITEM));
             f.getArguments().add(new MutableObject<>(p.first));
             f.getArguments().add(new MutableObject<>(indexPair.first));
@@ -747,8 +815,8 @@ class LangExpressionToPlanTranslator
         if (AsterixBuiltinFunctions.isBuiltinAggregateFunction(fi)) {
             f = AsterixBuiltinFunctions.makeAggregateFunctionExpression(fi, args);
         } else if (AsterixBuiltinFunctions.isBuiltinUnnestingFunction(fi)) {
-            UnnestingFunctionCallExpression ufce =
-                    new UnnestingFunctionCallExpression(FunctionUtil.getFunctionInfo(fi), args);
+            UnnestingFunctionCallExpression ufce = new UnnestingFunctionCallExpression(FunctionUtil.getFunctionInfo(fi),
+                    args);
             ufce.setReturnsUniqueValues(AsterixBuiltinFunctions.returnsUniqueValues(fi));
             f = ufce;
         } else {
@@ -762,7 +830,6 @@ class LangExpressionToPlanTranslator
         throw new IllegalStateException("Function declarations should be inlined at AST rewriting phase.");
     }
 
-    @SuppressWarnings("unchecked")
     @Override
     public Pair<ILogicalOperator, LogicalVariable> visit(GroupbyClause gc, Mutable<ILogicalOperator> tupSource)
             throws AsterixException {
@@ -771,9 +838,8 @@ class LangExpressionToPlanTranslator
             List<Pair<Expression, Identifier>> groupFieldList = gc.getGroupFieldList();
             List<Mutable<ILogicalExpression>> groupRecordConstructorArgList = new ArrayList<>();
             for (Pair<Expression, Identifier> groupField : groupFieldList) {
-                ILogicalExpression groupFieldNameExpr =
-                        langExprToAlgExpression(new LiteralExpr(new StringLiteral(groupField.second.getValue())),
-                                topOp).first;
+                ILogicalExpression groupFieldNameExpr = langExprToAlgExpression(
+                        new LiteralExpr(new StringLiteral(groupField.second.getValue())), topOp).first;
                 groupRecordConstructorArgList.add(new MutableObject<>(groupFieldNameExpr));
                 ILogicalExpression groupFieldExpr = langExprToAlgExpression(groupField.first, topOp).first;
                 groupRecordConstructorArgList.add(new MutableObject<>(groupFieldExpr));
@@ -802,14 +868,13 @@ class LangExpressionToPlanTranslator
                 // Calls the first-element aggregate function on a decoration variable to make sure the value
                 // is propagated through a global aggregation.
                 AggregateFunctionCallExpression firstElementAgg = AsterixBuiltinFunctions
-                        .makeAggregateFunctionExpression(
-                        AsterixBuiltinFunctions.FIRST_ELEMENT, flArgs);
+                        .makeAggregateFunctionExpression(AsterixBuiltinFunctions.FIRST_ELEMENT, flArgs);
                 aggVars.add(decorVar);
                 aggFuncs.add(new MutableObject<>(firstElementAgg));
             }
             for (Entry<Expression, VariableExpr> entry : gc.getWithVarMap().entrySet()) {
-                Pair<ILogicalExpression, Mutable<ILogicalOperator>> listifyInput =
-                        langExprToAlgExpression(entry.getKey(), topOp);
+                Pair<ILogicalExpression, Mutable<ILogicalOperator>> listifyInput = langExprToAlgExpression(
+                        entry.getKey(), topOp);
                 topOp = listifyInput.second;
                 List<Mutable<ILogicalExpression>> flArgs = new ArrayList<>();
                 flArgs.add(new MutableObject<>(listifyInput.first));
@@ -892,7 +957,7 @@ class LangExpressionToPlanTranslator
         Pair<ILogicalOperator, LogicalVariable> pCond = ifexpr.getCondExpr().accept(this, tupSource);
         LogicalVariable varCond = pCond.second;
 
-        //Creates a subplan for the "then" branch.
+        // Creates a subplan for the "then" branch.
         Pair<ILogicalOperator, LogicalVariable> opAndVarForThen = constructSubplanOperatorForBranch(pCond.first,
                 new MutableObject<>(new VariableReferenceExpression(varCond)), ifexpr.getThenExpr());
 
@@ -925,8 +990,8 @@ class LangExpressionToPlanTranslator
 
         // Produces the final result.
         LogicalVariable resultVar = context.newVar();
-        AssignOperator finalAssignOp =
-                new AssignOperator(resultVar, new MutableObject<>(new VariableReferenceExpression(unnestVar)));
+        AssignOperator finalAssignOp = new AssignOperator(resultVar,
+                new MutableObject<>(new VariableReferenceExpression(unnestVar)));
         finalAssignOp.getInputs().add(new MutableObject<>(unnestOp));
         return new Pair<>(finalAssignOp, resultVar);
     }
@@ -1222,29 +1287,22 @@ class LangExpressionToPlanTranslator
         return new ScalarFunctionCallExpression(finfo);
     }
 
-    private FunctionIdentifier operatorTypeToFunctionIdentifier(OperatorType t) {
+    private static FunctionIdentifier operatorTypeToFunctionIdentifier(OperatorType t) {
         switch (t) {
-            case EQ: {
+            case EQ:
                 return AlgebricksBuiltinFunctions.EQ;
-            }
-            case NEQ: {
+            case NEQ:
                 return AlgebricksBuiltinFunctions.NEQ;
-            }
-            case GT: {
+            case GT:
                 return AlgebricksBuiltinFunctions.GT;
-            }
-            case GE: {
+            case GE:
                 return AlgebricksBuiltinFunctions.GE;
-            }
-            case LT: {
+            case LT:
                 return AlgebricksBuiltinFunctions.LT;
-            }
-            case LE: {
+            case LE:
                 return AlgebricksBuiltinFunctions.LE;
-            }
-            default: {
+            default:
                 throw new IllegalStateException();
-            }
         }
     }
 
@@ -1300,8 +1358,8 @@ class LangExpressionToPlanTranslator
             Mutable<ILogicalOperator> topOpRef) throws AsterixException {
         switch (expr.getKind()) {
             case VARIABLE_EXPRESSION:
-                VariableReferenceExpression ve =
-                        new VariableReferenceExpression(context.getVar(((VariableExpr) expr).getVar().getId()));
+                VariableReferenceExpression ve = new VariableReferenceExpression(
+                        context.getVar(((VariableExpr) expr).getVar().getId()));
                 return new Pair<>(ve, topOpRef);
             case LITERAL_EXPRESSION:
                 LiteralExpr val = (LiteralExpr) expr;
@@ -1344,7 +1402,7 @@ class LangExpressionToPlanTranslator
 
         LogicalVariable varListified = context.newSubplanOutputVar();
         AggregateOperator agg = new AggregateOperator(mkSingletonArrayList(varListified),
-                (List) mkSingletonArrayList(new MutableObject<>(funAgg)));
+                mkSingletonArrayList(new MutableObject<>(funAgg)));
         agg.getInputs().add(opRef);
         ILogicalOperator res;
         if (bProject) {
@@ -1510,8 +1568,8 @@ class LangExpressionToPlanTranslator
 
                 // Recursively eliminate shared operator reference for the operator subtree,
                 // even if it is a deep copy of some other one.
-                Map<LogicalVariable, LogicalVariable> childVarMap =
-                        eliminateSharedOperatorReference(childRef, opRefSet);
+                Map<LogicalVariable, LogicalVariable> childVarMap = eliminateSharedOperatorReference(childRef,
+                        opRefSet);
                 // Substitute variables according to the new subtree.
                 VariableUtilities.substituteVariables(currentOperator, childVarMap, null);
 
@@ -1554,8 +1612,8 @@ class LangExpressionToPlanTranslator
         context.enterSubplan();
         SubplanOperator subplanOp = new SubplanOperator();
         subplanOp.getInputs().add(new MutableObject<>(inputOp));
-        Mutable<ILogicalOperator> nestedSource =
-                new MutableObject<>(new NestedTupleSourceOperator(new MutableObject<>(subplanOp)));
+        Mutable<ILogicalOperator> nestedSource = new MutableObject<>(
+                new NestedTupleSourceOperator(new MutableObject<>(subplanOp)));
         SelectOperator select = new SelectOperator(selectExpr, false, null);
         // The select operator cannot be moved up and down, otherwise it will cause typing issues (ASTERIXDB-1203).
         OperatorPropertiesUtil.markMovable(select, false);
@@ -1575,8 +1633,8 @@ class LangExpressionToPlanTranslator
 
     // Processes EXISTS and NOT EXISTS.
     private AssignOperator processExists(ILogicalExpression inputExpr, LogicalVariable v1, boolean not) {
-        AbstractFunctionCallExpression count =
-                new ScalarFunctionCallExpression(FunctionUtil.getFunctionInfo(AsterixBuiltinFunctions.SCALAR_COUNT));
+        AbstractFunctionCallExpression count = new ScalarFunctionCallExpression(
+                FunctionUtil.getFunctionInfo(AsterixBuiltinFunctions.SCALAR_COUNT));
         count.getArguments().add(new MutableObject<>(inputExpr));
         AbstractFunctionCallExpression comparison = new ScalarFunctionCallExpression(
                 FunctionUtil.getFunctionInfo(not ? AsterixBuiltinFunctions.EQ : AsterixBuiltinFunctions.NEQ));
@@ -1648,8 +1706,8 @@ class LangExpressionToPlanTranslator
         while (inputOpRefIterator.hasNext()) {
             // Generates the variable triple <leftVar, rightVar, outputVar> .
             topUnionVar = context.newVar();
-            Triple<LogicalVariable, LogicalVariable, LogicalVariable> varTriple =
-                    new Triple<>(leftInputVar, inputVarIterator.next(), topUnionVar);
+            Triple<LogicalVariable, LogicalVariable, LogicalVariable> varTriple = new Triple<>(leftInputVar,
+                    inputVarIterator.next(), topUnionVar);
             List<Triple<LogicalVariable, LogicalVariable, LogicalVariable>> varTriples = new ArrayList<>();
             varTriples.add(varTriple);
 

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/util/PlanTranslationUtil.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/util/PlanTranslationUtil.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/util/PlanTranslationUtil.java
index 63d1908..ea3ec2c 100644
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/util/PlanTranslationUtil.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/util/PlanTranslationUtil.java
@@ -71,7 +71,6 @@ public class PlanTranslationUtil {
         }
     }
 
-    @SuppressWarnings("unchecked")
     private static ScalarFunctionCallExpression createFieldAccessExpression(ILogicalExpression target,
             List<String> field) {
         FunctionIdentifier functionIdentifier;

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/asterixdb/asterix-algebra/src/main/javacc/AQLPlus.jj
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-algebra/src/main/javacc/AQLPlus.jj b/asterixdb/asterix-algebra/src/main/javacc/AQLPlus.jj
index e0e9388..725f50a 100644
--- a/asterixdb/asterix-algebra/src/main/javacc/AQLPlus.jj
+++ b/asterixdb/asterix-algebra/src/main/javacc/AQLPlus.jj
@@ -41,7 +41,6 @@ import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 
-import org.apache.asterix.common.config.MetadataConstants;
 import org.apache.asterix.common.exceptions.AsterixException;
 import org.apache.asterix.common.functions.FunctionSignature;
 import org.apache.asterix.lang.aql.clause.DistinctClause;
@@ -100,6 +99,7 @@ import org.apache.asterix.lang.common.statement.WriteStatement;
 import org.apache.asterix.lang.common.struct.Identifier;
 import org.apache.asterix.lang.common.struct.QuantifiedPair;
 import org.apache.asterix.lang.common.struct.VarIdentifier;
+import org.apache.asterix.metadata.utils.MetadataConstants;
 import org.apache.hyracks.algebricks.core.algebra.expressions.IndexedNLJoinExpressionAnnotation;
 
 


Mime
View raw message