asterixdb-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From buyin...@apache.org
Subject [13/16] asterixdb git commit: Add Asterix Extension Manager
Date Sat, 20 Aug 2016 06:15:55 GMT
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/APIFramework.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/APIFramework.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/APIFramework.java
index cd088c1..5728947 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/APIFramework.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/APIFramework.java
@@ -27,6 +27,9 @@ import java.util.List;
 import org.apache.asterix.algebra.base.ILangExpressionToPlanTranslator;
 import org.apache.asterix.algebra.base.ILangExpressionToPlanTranslatorFactory;
 import org.apache.asterix.api.common.Job.SubmissionMode;
+import org.apache.asterix.app.cc.CompilerExtensionManager;
+import org.apache.asterix.app.result.ResultUtil;
+import org.apache.asterix.common.app.SessionConfig;
 import org.apache.asterix.common.config.AsterixCompilerProperties;
 import org.apache.asterix.common.config.AsterixExternalProperties;
 import org.apache.asterix.common.config.OptimizationConfUtil;
@@ -50,10 +53,10 @@ import org.apache.asterix.lang.common.statement.Query;
 import org.apache.asterix.metadata.declared.AqlMetadataProvider;
 import org.apache.asterix.om.util.AsterixAppContextInfo;
 import org.apache.asterix.optimizer.base.RuleCollections;
-import org.apache.asterix.result.ResultUtils;
 import org.apache.asterix.runtime.job.listener.JobEventListenerFactory;
 import org.apache.asterix.transaction.management.service.transaction.JobIdFactory;
 import org.apache.asterix.translator.CompiledStatements.ICompiledDmlStatement;
+import org.apache.asterix.translator.IStatementExecutor.Stats;
 import org.apache.hyracks.algebricks.common.constraints.AlgebricksPartitionConstraint;
 import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
 import org.apache.hyracks.algebricks.common.utils.Pair;
@@ -89,19 +92,21 @@ import org.json.JSONException;
  * to Hyracks through the Hyracks client interface.
  */
 public class APIFramework {
-    public static final String HTML_STATEMENT_SEPARATOR = "<!-- BEGIN -->";
 
     private final IRewriterFactory rewriterFactory;
     private final IAstPrintVisitorFactory astPrintVisitorFactory;
     private final ILangExpressionToPlanTranslatorFactory translatorFactory;
+    private final CompilerExtensionManager cExtensionManager;
 
-    public APIFramework(ILangCompilationProvider compilationProvider) {
+    public APIFramework(ILangCompilationProvider compilationProvider, CompilerExtensionManager cExtensionManager) {
         this.rewriterFactory = compilationProvider.getRewriterFactory();
         this.astPrintVisitorFactory = compilationProvider.getAstPrintVisitorFactory();
         this.translatorFactory = compilationProvider.getExpressionToPlanTranslatorFactory();
+        this.cExtensionManager = cExtensionManager;
     }
 
-    private static List<Pair<AbstractRuleController, List<IAlgebraicRewriteRule>>> buildDefaultLogicalRewrites() {
+    private static List<Pair<AbstractRuleController, List<IAlgebraicRewriteRule>>>
+            buildDefaultLogicalRewrites(CompilerExtensionManager ccExtensionManager) {
         List<Pair<AbstractRuleController, List<IAlgebraicRewriteRule>>> defaultLogicalRewrites = new ArrayList<>();
         SequentialFixpointRuleController seqCtrlNoDfs = new SequentialFixpointRuleController(false);
         SequentialFixpointRuleController seqCtrlFullDfs = new SequentialFixpointRuleController(true);
@@ -109,14 +114,16 @@ public class APIFramework {
         defaultLogicalRewrites.add(new Pair<>(seqOnceCtrl, RuleCollections.buildInitialTranslationRuleCollection()));
         defaultLogicalRewrites.add(new Pair<>(seqOnceCtrl, RuleCollections.buildTypeInferenceRuleCollection()));
         defaultLogicalRewrites.add(new Pair<>(seqOnceCtrl, RuleCollections.buildAutogenerateIDRuleCollection()));
-        defaultLogicalRewrites.add(new Pair<>(seqCtrlFullDfs, RuleCollections.buildNormalizationRuleCollection()));
+        defaultLogicalRewrites
+                .add(new Pair<>(seqCtrlFullDfs, RuleCollections.buildNormalizationRuleCollection(ccExtensionManager)));
         defaultLogicalRewrites
                 .add(new Pair<>(seqCtrlNoDfs, RuleCollections.buildCondPushDownAndJoinInferenceRuleCollection()));
         defaultLogicalRewrites.add(new Pair<>(seqCtrlFullDfs, RuleCollections.buildLoadFieldsRuleCollection()));
         // fj
         defaultLogicalRewrites.add(new Pair<>(seqCtrlFullDfs, RuleCollections.buildFuzzyJoinRuleCollection()));
         //
-        defaultLogicalRewrites.add(new Pair<>(seqCtrlFullDfs, RuleCollections.buildNormalizationRuleCollection()));
+        defaultLogicalRewrites
+                .add(new Pair<>(seqCtrlFullDfs, RuleCollections.buildNormalizationRuleCollection(ccExtensionManager)));
         defaultLogicalRewrites
                 .add(new Pair<>(seqCtrlNoDfs, RuleCollections.buildCondPushDownAndJoinInferenceRuleCollection()));
         defaultLogicalRewrites.add(new Pair<>(seqCtrlFullDfs, RuleCollections.buildLoadFieldsRuleCollection()));
@@ -251,7 +258,7 @@ public class APIFramework {
         HeuristicCompilerFactoryBuilder builder =
                 new HeuristicCompilerFactoryBuilder(AqlOptimizationContextFactory.INSTANCE);
         builder.setPhysicalOptimizationConfig(OptimizationConfUtil.getPhysicalOptimizationConfig());
-        builder.setLogicalRewrites(buildDefaultLogicalRewrites());
+        builder.setLogicalRewrites(buildDefaultLogicalRewrites(cExtensionManager));
         builder.setPhysicalRewrites(buildDefaultPhysicalRewrites());
         IDataFormat format = queryMetadataProvider.getFormat();
         ICompilerFactory compilerFactory = builder.create();
@@ -289,7 +296,7 @@ public class APIFramework {
             try {
                 LogicalOperatorPrettyPrintVisitor pvisitor = new LogicalOperatorPrettyPrintVisitor();
                 PlanPrettyPrinter.printPlan(plan, pvisitor, 0);
-                ResultUtils.displayResults(pvisitor.get().toString(), conf, new ResultUtils.Stats(), null);
+                ResultUtil.displayResults(pvisitor.get().toString(), conf, new Stats(), null);
                 return null;
             } catch (IOException e) {
                 throw new AlgebricksException(e);

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/AsterixAppRuntimeContext.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/AsterixAppRuntimeContext.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/AsterixAppRuntimeContext.java
deleted file mode 100644
index 4bb09d4..0000000
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/AsterixAppRuntimeContext.java
+++ /dev/null
@@ -1,465 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.asterix.api.common;
-
-import java.io.IOException;
-import java.rmi.RemoteException;
-import java.rmi.server.UnicastRemoteObject;
-import java.util.Set;
-import java.util.logging.Level;
-import java.util.logging.Logger;
-
-import org.apache.asterix.active.ActiveManager;
-import org.apache.asterix.common.api.AsterixThreadExecutor;
-import org.apache.asterix.common.api.IAsterixAppRuntimeContext;
-import org.apache.asterix.common.api.IDatasetLifecycleManager;
-import org.apache.asterix.common.cluster.ClusterPartition;
-import org.apache.asterix.common.config.AsterixBuildProperties;
-import org.apache.asterix.common.config.AsterixCompilerProperties;
-import org.apache.asterix.common.config.AsterixExternalProperties;
-import org.apache.asterix.common.config.AsterixFeedProperties;
-import org.apache.asterix.common.config.AsterixMetadataProperties;
-import org.apache.asterix.common.config.AsterixPropertiesAccessor;
-import org.apache.asterix.common.config.AsterixReplicationProperties;
-import org.apache.asterix.common.config.AsterixStorageProperties;
-import org.apache.asterix.common.config.AsterixTransactionProperties;
-import org.apache.asterix.common.config.IAsterixPropertiesProvider;
-import org.apache.asterix.common.context.AsterixFileMapManager;
-import org.apache.asterix.common.context.DatasetLifecycleManager;
-import org.apache.asterix.common.exceptions.ACIDException;
-import org.apache.asterix.common.exceptions.AsterixException;
-import org.apache.asterix.common.library.ILibraryManager;
-import org.apache.asterix.common.replication.IRemoteRecoveryManager;
-import org.apache.asterix.common.replication.IReplicaResourcesManager;
-import org.apache.asterix.common.replication.IReplicationChannel;
-import org.apache.asterix.common.replication.IReplicationManager;
-import org.apache.asterix.common.transactions.IAsterixAppRuntimeContextProvider;
-import org.apache.asterix.common.transactions.IRecoveryManager;
-import org.apache.asterix.common.transactions.IRecoveryManager.SystemState;
-import org.apache.asterix.common.transactions.ITransactionSubsystem;
-import org.apache.asterix.external.library.ExternalLibraryManager;
-import org.apache.asterix.metadata.MetadataManager;
-import org.apache.asterix.metadata.MetadataNode;
-import org.apache.asterix.metadata.api.IAsterixStateProxy;
-import org.apache.asterix.metadata.api.IMetadataNode;
-import org.apache.asterix.metadata.bootstrap.MetadataBootstrap;
-import org.apache.asterix.metadata.bootstrap.MetadataIndexImmutableProperties;
-import org.apache.asterix.om.util.AsterixClusterProperties;
-import org.apache.asterix.replication.management.ReplicationChannel;
-import org.apache.asterix.replication.management.ReplicationManager;
-import org.apache.asterix.replication.recovery.RemoteRecoveryManager;
-import org.apache.asterix.replication.storage.ReplicaResourcesManager;
-import org.apache.asterix.transaction.management.resource.GlobalResourceIdFactoryProvider;
-import org.apache.asterix.transaction.management.resource.PersistentLocalResourceRepository;
-import org.apache.asterix.transaction.management.resource.PersistentLocalResourceRepositoryFactory;
-import org.apache.asterix.transaction.management.service.transaction.TransactionSubsystem;
-import org.apache.hyracks.api.application.IApplicationConfig;
-import org.apache.hyracks.api.application.INCApplicationContext;
-import org.apache.hyracks.api.exceptions.HyracksDataException;
-import org.apache.hyracks.api.io.IIOManager;
-import org.apache.hyracks.api.lifecycle.ILifeCycleComponent;
-import org.apache.hyracks.api.lifecycle.ILifeCycleComponentManager;
-import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationScheduler;
-import org.apache.hyracks.storage.am.lsm.common.api.ILSMMergePolicyFactory;
-import org.apache.hyracks.storage.am.lsm.common.api.ILSMOperationTracker;
-import org.apache.hyracks.storage.am.lsm.common.impls.AsynchronousScheduler;
-import org.apache.hyracks.storage.am.lsm.common.impls.PrefixMergePolicyFactory;
-import org.apache.hyracks.storage.common.buffercache.BufferCache;
-import org.apache.hyracks.storage.common.buffercache.ClockPageReplacementStrategy;
-import org.apache.hyracks.storage.common.buffercache.DelayPageCleanerPolicy;
-import org.apache.hyracks.storage.common.buffercache.HeapBufferAllocator;
-import org.apache.hyracks.storage.common.buffercache.IBufferCache;
-import org.apache.hyracks.storage.common.buffercache.ICacheMemoryAllocator;
-import org.apache.hyracks.storage.common.buffercache.IPageCleanerPolicy;
-import org.apache.hyracks.storage.common.buffercache.IPageReplacementStrategy;
-import org.apache.hyracks.storage.common.file.IFileMapManager;
-import org.apache.hyracks.storage.common.file.IFileMapProvider;
-import org.apache.hyracks.storage.common.file.ILocalResourceRepository;
-import org.apache.hyracks.storage.common.file.ILocalResourceRepositoryFactory;
-import org.apache.hyracks.storage.common.file.IResourceIdFactory;
-
-public class AsterixAppRuntimeContext implements IAsterixAppRuntimeContext, IAsterixPropertiesProvider {
-    private static final Logger LOGGER = Logger.getLogger(AsterixAppRuntimeContext.class.getName());
-
-    private ILSMMergePolicyFactory metadataMergePolicyFactory;
-    private final INCApplicationContext ncApplicationContext;
-
-    private AsterixCompilerProperties compilerProperties;
-    private AsterixExternalProperties externalProperties;
-    private AsterixMetadataProperties metadataProperties;
-    private AsterixStorageProperties storageProperties;
-    private AsterixTransactionProperties txnProperties;
-    private AsterixFeedProperties feedProperties;
-    private AsterixBuildProperties buildProperties;
-    private AsterixReplicationProperties replicationProperties;
-
-    private AsterixThreadExecutor threadExecutor;
-    private IDatasetLifecycleManager datasetLifecycleManager;
-    private IFileMapManager fileMapManager;
-    private IBufferCache bufferCache;
-    private ITransactionSubsystem txnSubsystem;
-
-    private ILSMIOOperationScheduler lsmIOScheduler;
-    private PersistentLocalResourceRepository localResourceRepository;
-    private IResourceIdFactory resourceIdFactory;
-    private IIOManager ioManager;
-    private boolean isShuttingdown;
-
-    private ActiveManager activeManager;
-
-    private IReplicationChannel replicationChannel;
-    private IReplicationManager replicationManager;
-    private IRemoteRecoveryManager remoteRecoveryManager;
-    private IReplicaResourcesManager replicaResourcesManager;
-    private final int metadataRmiPort;
-
-    private ILibraryManager libraryManager;
-
-    public AsterixAppRuntimeContext(INCApplicationContext ncApplicationContext, int metadataRmiPort)
-            throws AsterixException {
-        this.ncApplicationContext = ncApplicationContext;
-        // Determine whether to use old-style asterix-configuration.xml or new-style configuration.
-        // QQQ strip this out eventually
-        AsterixPropertiesAccessor propertiesAccessor;
-        IApplicationConfig cfg = ncApplicationContext.getAppConfig();
-        // QQQ this is NOT a good way to determine whether the config is valid
-        if (cfg.getString("cc", "cluster.address") != null) {
-            propertiesAccessor = new AsterixPropertiesAccessor(cfg);
-        } else {
-            propertiesAccessor = new AsterixPropertiesAccessor();
-        }
-        compilerProperties = new AsterixCompilerProperties(propertiesAccessor);
-        externalProperties = new AsterixExternalProperties(propertiesAccessor);
-        metadataProperties = new AsterixMetadataProperties(propertiesAccessor);
-        storageProperties = new AsterixStorageProperties(propertiesAccessor);
-        txnProperties = new AsterixTransactionProperties(propertiesAccessor);
-        feedProperties = new AsterixFeedProperties(propertiesAccessor);
-        buildProperties = new AsterixBuildProperties(propertiesAccessor);
-        replicationProperties =
-                new AsterixReplicationProperties(propertiesAccessor, AsterixClusterProperties.INSTANCE.getCluster());
-        this.metadataRmiPort = metadataRmiPort;
-    }
-
-    @Override
-    public void initialize(boolean initialRun) throws IOException, ACIDException {
-        Logger.getLogger("org.apache").setLevel(externalProperties.getLogLevel());
-
-        threadExecutor = new AsterixThreadExecutor(ncApplicationContext.getThreadFactory());
-        fileMapManager = new AsterixFileMapManager();
-        ICacheMemoryAllocator allocator = new HeapBufferAllocator();
-        IPageCleanerPolicy pcp = new DelayPageCleanerPolicy(600000);
-        ioManager = ncApplicationContext.getRootContext().getIOManager();
-        IPageReplacementStrategy prs = new ClockPageReplacementStrategy(allocator,
-                storageProperties.getBufferCachePageSize(), storageProperties.getBufferCacheNumPages());
-
-        AsynchronousScheduler.INSTANCE.init(ncApplicationContext.getThreadFactory());
-        lsmIOScheduler = AsynchronousScheduler.INSTANCE;
-
-        metadataMergePolicyFactory = new PrefixMergePolicyFactory();
-
-        ILocalResourceRepositoryFactory persistentLocalResourceRepositoryFactory =
-                new PersistentLocalResourceRepositoryFactory(ioManager, ncApplicationContext.getNodeId(),
-                        metadataProperties);
-
-        localResourceRepository =
-                (PersistentLocalResourceRepository) persistentLocalResourceRepositoryFactory.createRepository();
-
-        IAsterixAppRuntimeContextProvider asterixAppRuntimeContextProvider =
-                new AsterixAppRuntimeContextProviderForRecovery(this);
-        txnSubsystem = new TransactionSubsystem(ncApplicationContext.getNodeId(), asterixAppRuntimeContextProvider,
-                txnProperties);
-
-        IRecoveryManager recoveryMgr = txnSubsystem.getRecoveryManager();
-        SystemState systemState = recoveryMgr.getSystemState();
-        if (initialRun || systemState == SystemState.NEW_UNIVERSE) {
-            //delete any storage data before the resource factory is initialized
-            localResourceRepository.deleteStorageData(true);
-        }
-        initializeResourceIdFactory();
-
-        datasetLifecycleManager = new DatasetLifecycleManager(storageProperties, localResourceRepository,
-                MetadataIndexImmutableProperties.FIRST_AVAILABLE_USER_DATASET_ID, txnSubsystem.getLogManager(),
-                ioManager.getIODevices().size());
-
-        isShuttingdown = false;
-
-        activeManager = new ActiveManager(ncApplicationContext.getNodeId(),
-                feedProperties.getMemoryComponentGlobalBudget(), compilerProperties.getFrameSize());
-
-        if (replicationProperties.isReplicationEnabled()) {
-            String nodeId = ncApplicationContext.getNodeId();
-
-            replicaResourcesManager = new ReplicaResourcesManager(localResourceRepository, metadataProperties);
-
-            replicationManager = new ReplicationManager(nodeId, replicationProperties, replicaResourcesManager,
-                    txnSubsystem.getLogManager(), asterixAppRuntimeContextProvider);
-
-            //pass replication manager to replication required object
-            //LogManager to replicate logs
-            txnSubsystem.getLogManager().setReplicationManager(replicationManager);
-
-            //PersistentLocalResourceRepository to replicate metadata files and delete backups on drop index
-            localResourceRepository.setReplicationManager(replicationManager);
-
-            /**
-             * add the partitions that will be replicated in this node as inactive partitions
-             */
-            //get nodes which replicate to this node
-            Set<String> replicationClients = replicationProperties.getNodeReplicationClients(nodeId);
-            //remove the node itself
-            replicationClients.remove(nodeId);
-            for (String clientId : replicationClients) {
-                //get the partitions of each client
-                ClusterPartition[] clientPartitions = metadataProperties.getNodePartitions().get(clientId);
-                for (ClusterPartition partition : clientPartitions) {
-                    localResourceRepository.addInactivePartition(partition.getPartitionId());
-                }
-            }
-
-            //initialize replication channel
-            replicationChannel = new ReplicationChannel(nodeId, replicationProperties, txnSubsystem.getLogManager(),
-                    replicaResourcesManager, replicationManager, ncApplicationContext,
-                    asterixAppRuntimeContextProvider);
-
-            remoteRecoveryManager = new RemoteRecoveryManager(replicationManager, this, replicationProperties);
-
-            bufferCache =
-                    new BufferCache(ioManager, prs, pcp, fileMapManager, storageProperties.getBufferCacheMaxOpenFiles(),
-                            ncApplicationContext.getThreadFactory(), replicationManager);
-        } else {
-            bufferCache = new BufferCache(ioManager, prs, pcp, fileMapManager,
-                    storageProperties.getBufferCacheMaxOpenFiles(), ncApplicationContext.getThreadFactory());
-        }
-
-        // The order of registration is important. The buffer cache must registered before recovery and transaction managers.
-        //Notes: registered components are stopped in reversed order
-        ILifeCycleComponentManager lccm = ncApplicationContext.getLifeCycleComponentManager();
-        lccm.register((ILifeCycleComponent) bufferCache);
-        /**
-         * LogManager must be stopped after RecoveryManager, DatasetLifeCycleManager, and ReplicationManager
-         * to process any logs that might be generated during stopping these components
-         */
-        lccm.register((ILifeCycleComponent) txnSubsystem.getLogManager());
-        /**
-         * ReplicationManager must be stopped after indexLifecycleManager and recovery manager
-         * so that any logs/files generated during closing datasets or checkpoints are sent to remote replicas
-         */
-        if (replicationManager != null) {
-            lccm.register(replicationManager);
-        }
-        lccm.register((ILifeCycleComponent) txnSubsystem.getRecoveryManager());
-        /**
-         * Stopping indexLifecycleManager will flush and close all datasets.
-         */
-        lccm.register((ILifeCycleComponent) datasetLifecycleManager);
-        lccm.register((ILifeCycleComponent) txnSubsystem.getTransactionManager());
-        lccm.register((ILifeCycleComponent) txnSubsystem.getLockManager());
-
-        /**
-         * Initializes the library manager.
-         */
-        libraryManager = new ExternalLibraryManager();
-    }
-
-    @Override
-    public boolean isShuttingdown() {
-        return isShuttingdown;
-    }
-
-    @Override
-    public void setShuttingdown(boolean isShuttingdown) {
-        this.isShuttingdown = isShuttingdown;
-    }
-
-    @Override
-    public void deinitialize() throws HyracksDataException {
-    }
-
-    @Override
-    public IBufferCache getBufferCache() {
-        return bufferCache;
-    }
-
-    @Override
-    public IFileMapProvider getFileMapManager() {
-        return fileMapManager;
-    }
-
-    @Override
-    public ITransactionSubsystem getTransactionSubsystem() {
-        return txnSubsystem;
-    }
-
-    @Override
-    public IDatasetLifecycleManager getDatasetLifecycleManager() {
-        return datasetLifecycleManager;
-    }
-
-    @Override
-    public double getBloomFilterFalsePositiveRate() {
-        return storageProperties.getBloomFilterFalsePositiveRate();
-    }
-
-    @Override
-    public ILSMIOOperationScheduler getLSMIOScheduler() {
-        return lsmIOScheduler;
-    }
-
-    @Override
-    public ILocalResourceRepository getLocalResourceRepository() {
-        return localResourceRepository;
-    }
-
-    @Override
-    public IResourceIdFactory getResourceIdFactory() {
-        return resourceIdFactory;
-    }
-
-    @Override
-    public IIOManager getIOManager() {
-        return ioManager;
-    }
-
-    @Override
-    public AsterixStorageProperties getStorageProperties() {
-        return storageProperties;
-    }
-
-    @Override
-    public AsterixTransactionProperties getTransactionProperties() {
-        return txnProperties;
-    }
-
-    @Override
-    public AsterixCompilerProperties getCompilerProperties() {
-        return compilerProperties;
-    }
-
-    @Override
-    public AsterixMetadataProperties getMetadataProperties() {
-        return metadataProperties;
-    }
-
-    @Override
-    public AsterixExternalProperties getExternalProperties() {
-        return externalProperties;
-    }
-
-    @Override
-    public AsterixFeedProperties getFeedProperties() {
-        return feedProperties;
-    }
-
-    @Override
-    public AsterixBuildProperties getBuildProperties() {
-        return buildProperties;
-    }
-
-    @Override
-    public ILSMOperationTracker getLSMBTreeOperationTracker(int datasetID) {
-        return datasetLifecycleManager.getOperationTracker(datasetID);
-    }
-
-    @Override
-    public AsterixThreadExecutor getThreadExecutor() {
-        return threadExecutor;
-    }
-
-    @Override
-    public ILSMMergePolicyFactory getMetadataMergePolicyFactory() {
-        return metadataMergePolicyFactory;
-    }
-
-    @Override
-    public ActiveManager getFeedManager() {
-        return activeManager;
-    }
-
-    @Override
-    public AsterixReplicationProperties getReplicationProperties() {
-        return replicationProperties;
-    }
-
-    @Override
-    public IReplicationChannel getReplicationChannel() {
-        return replicationChannel;
-    }
-
-    @Override
-    public IReplicaResourcesManager getReplicaResourcesManager() {
-        return replicaResourcesManager;
-    }
-
-    @Override
-    public IRemoteRecoveryManager getRemoteRecoveryManager() {
-        return remoteRecoveryManager;
-    }
-
-    @Override
-    public IReplicationManager getReplicationManager() {
-        return replicationManager;
-    }
-
-    @Override
-    public ILibraryManager getLibraryManager() {
-        return libraryManager;
-    }
-
-    @Override
-    public void initializeResourceIdFactory() throws HyracksDataException {
-        resourceIdFactory = new GlobalResourceIdFactoryProvider(ncApplicationContext).createResourceIdFactory();
-    }
-
-    @Override
-    public void initializeMetadata(boolean newUniverse) throws Exception {
-        IAsterixStateProxy proxy;
-        if (LOGGER.isLoggable(Level.INFO)) {
-            LOGGER.info("Bootstrapping metadata");
-        }
-        MetadataNode.INSTANCE.initialize(this);
-
-        proxy = (IAsterixStateProxy) ncApplicationContext.getDistributedState();
-        if (proxy == null) {
-            throw new IllegalStateException("Metadata node cannot access distributed state");
-        }
-
-        // This is a special case, we just give the metadataNode directly.
-        // This way we can delay the registration of the metadataNode until
-        // it is completely initialized.
-        MetadataManager.INSTANCE = new MetadataManager(proxy, MetadataNode.INSTANCE);
-        MetadataBootstrap.startUniverse(this, ncApplicationContext, newUniverse);
-        MetadataBootstrap.startDDLRecovery();
-
-        if (LOGGER.isLoggable(Level.INFO)) {
-            LOGGER.info("Metadata node bound");
-        }
-    }
-
-    @Override
-    public void exportMetadataNodeStub() throws RemoteException {
-        IMetadataNode stub = (IMetadataNode) UnicastRemoteObject.exportObject(MetadataNode.INSTANCE, metadataRmiPort);
-        ((IAsterixStateProxy) ncApplicationContext.getDistributedState()).setMetadataNode(stub);
-    }
-
-    @Override
-    public void unexportMetadataNodeStub() throws RemoteException {
-        UnicastRemoteObject.unexportObject(MetadataNode.INSTANCE, false);
-    }
-
-}

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/AsterixAppRuntimeContextProviderForRecovery.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/AsterixAppRuntimeContextProviderForRecovery.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/AsterixAppRuntimeContextProviderForRecovery.java
index 7ac5036..265025f 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/AsterixAppRuntimeContextProviderForRecovery.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/AsterixAppRuntimeContextProviderForRecovery.java
@@ -18,6 +18,7 @@
  */
 package org.apache.asterix.api.common;
 
+import org.apache.asterix.app.nc.AsterixNCAppRuntimeContext;
 import org.apache.asterix.common.api.AsterixThreadExecutor;
 import org.apache.asterix.common.api.IAsterixAppRuntimeContext;
 import org.apache.asterix.common.api.IDatasetLifecycleManager;
@@ -33,9 +34,9 @@ import org.apache.hyracks.storage.common.file.IResourceIdFactory;
 
 public class AsterixAppRuntimeContextProviderForRecovery implements IAsterixAppRuntimeContextProvider {
 
-    private final AsterixAppRuntimeContext asterixAppRuntimeContext;
+    private final AsterixNCAppRuntimeContext asterixAppRuntimeContext;
 
-    public AsterixAppRuntimeContextProviderForRecovery(AsterixAppRuntimeContext asterixAppRuntimeContext) {
+    public AsterixAppRuntimeContextProviderForRecovery(AsterixNCAppRuntimeContext asterixAppRuntimeContext) {
         this.asterixAppRuntimeContext = asterixAppRuntimeContext;
     }
 

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/AsterixHyracksIntegrationUtil.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/AsterixHyracksIntegrationUtil.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/AsterixHyracksIntegrationUtil.java
index 4b00195..b1bf418 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/AsterixHyracksIntegrationUtil.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/AsterixHyracksIntegrationUtil.java
@@ -24,6 +24,8 @@ import java.util.ArrayList;
 import java.util.Collections;
 import java.util.EnumSet;
 import java.util.List;
+import java.util.logging.Level;
+import java.util.logging.Logger;
 
 import org.apache.asterix.common.config.AsterixPropertiesAccessor;
 import org.apache.asterix.common.config.GlobalConfig;
@@ -42,7 +44,7 @@ import org.apache.hyracks.control.common.controllers.NCConfig;
 import org.apache.hyracks.control.nc.NodeControllerService;
 
 public class AsterixHyracksIntegrationUtil {
-
+    private static final Logger LOGGER = Logger.getLogger(AsterixHyracksIntegrationUtil.class.getName());
     private static final String IO_DIR_KEY = "java.io.tmpdir";
     public static final int DEFAULT_HYRACKS_CC_CLIENT_PORT = 1098;
     public static final int DEFAULT_HYRACKS_CC_CLUSTER_PORT = 1099;
@@ -75,7 +77,7 @@ public class AsterixHyracksIntegrationUtil {
                     try {
                         nodeControllerService.start();
                     } catch (Exception e) {
-                        e.printStackTrace();
+                        LOGGER.log(Level.SEVERE, e.getMessage(), e);
                     }
                 }
             };

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/SessionConfig.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/SessionConfig.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/SessionConfig.java
deleted file mode 100644
index ba205d1..0000000
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/SessionConfig.java
+++ /dev/null
@@ -1,263 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.asterix.api.common;
-
-import java.io.PrintWriter;
-import java.util.HashMap;
-import java.util.Map;
-
-import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
-import org.apache.hyracks.algebricks.core.algebra.prettyprint.AlgebricksAppendable;
-
-/**
- * SessionConfig captures several different parameters for controlling
- * the execution of an APIFramework call.
- * <li>It specifies how the execution will proceed (for instance,
- * whether to optimize, or whether to execute at all).
- * <li>It allows you specify where the primary execution output will
- * be sent.
- * <li>It also allows you to request additional output for optional
- * out-of-band data about the execution (query plan, etc).
- * <li>It allows you to specify the output format for the primary
- * execution output - LOSSLESS_JSON, CSV, etc.
- * <li>It allows you to specify output format-specific parameters.
- */
-
-public class SessionConfig {
-    /**
-     * Used to specify the output format for the primary execution.
-     */
-    public enum OutputFormat {
-        ADM,
-        CSV,
-        CLEAN_JSON,
-        LOSSLESS_JSON
-    };
-
-    /**
-     * Produce out-of-band output for Hyracks Job.
-     */
-    public static final String OOB_HYRACKS_JOB = "oob-hyracks-job";
-
-    /**
-     * Produce out-of-band output for Expression Tree.
-     */
-    public static final String OOB_EXPR_TREE = "oob-expr-tree";
-
-    /**
-     * Produce out-of-band output for Rewritten Expression Tree.
-     */
-    public static final String OOB_REWRITTEN_EXPR_TREE = "oob-rewritten-expr-tree";
-
-    /**
-     * Produce out-of-band output for Logical Plan.
-     */
-    public static final String OOB_LOGICAL_PLAN = "oob-logical-plan";
-
-    /**
-     * Produce out-of-band output for Optimized Logical Plan.
-     */
-    public static final String OOB_OPTIMIZED_LOGICAL_PLAN = "oob-optimized-logical-plan";
-
-    /**
-     * Format flag: print only physical ops (for optimizer tests).
-     */
-    public static final String FORMAT_ONLY_PHYSICAL_OPS = "format-only-physical-ops";
-
-    /**
-     * Format flag: wrap out-of-band data in HTML.
-     */
-    public static final String FORMAT_HTML = "format-html";
-
-    /**
-     * Format flag: print CSV header line.
-     */
-    public static final String FORMAT_CSV_HEADER = "format-csv-header";
-
-    /**
-     * Format flag: wrap results in outer array brackets (JSON or ADM).
-     */
-    public static final String FORMAT_WRAPPER_ARRAY = "format-wrapper-array";
-
-    /**
-     * Format flag: indent JSON results.
-     */
-    public static final String FORMAT_INDENT_JSON = "indent-json";
-
-    /**
-     * Format flag: quote records in the results array.
-     */
-    public static final String FORMAT_QUOTE_RECORD = "quote-record";
-
-    public interface ResultDecorator {
-        AlgebricksAppendable append(AlgebricksAppendable app) throws AlgebricksException;
-    }
-
-    // Standard execution flags.
-    private final boolean executeQuery;
-    private final boolean generateJobSpec;
-    private final boolean optimize;
-
-    // Output path for primary execution.
-    private final PrintWriter out;
-
-    // Output format.
-    private final OutputFormat fmt;
-
-    private final ResultDecorator preResultDecorator;
-    private final ResultDecorator postResultDecorator;
-
-    // Flags.
-    private final Map<String, Boolean> flags;
-
-    /**
-     * Create a SessionConfig object with all default values:
-     * - All format flags set to "false".
-     * - All out-of-band outputs set to "null".
-     * - "Optimize" set to "true".
-     * - "Execute Query" set to "true".
-     * - "Generate Job Spec" set to "true".
-     *
-     * @param out
-     *            PrintWriter for execution output.
-     * @param fmt
-     *            Output format for execution output.
-     */
-    public SessionConfig(PrintWriter out, OutputFormat fmt) {
-        this(out, fmt, null, null, true, true, true);
-    }
-
-    public SessionConfig(PrintWriter out, OutputFormat fmt, ResultDecorator preResultDecorator,
-            ResultDecorator postResultDecorator) {
-        this(out, fmt, preResultDecorator, postResultDecorator, true, true, true);
-    }
-
-    public SessionConfig(PrintWriter out, OutputFormat fmt, boolean optimize, boolean executeQuery,
-            boolean generateJobSpec) {
-        this(out, fmt, null, null, optimize, executeQuery, generateJobSpec);
-    }
-
-    /**
-     * Create a SessionConfig object with all optional values set to defaults:
-     * - All format flags set to "false".
-     * - All out-of-band outputs set to "false".
-     *
-     * @param out
-     *            PrintWriter for execution output.
-     * @param fmt
-     *            Output format for execution output.
-     * @param optimize
-     *            Whether to optimize the execution.
-     * @param executeQuery
-     *            Whether to execute the query or not.
-     * @param generateJobSpec
-     *            Whether to generate the Hyracks job specification (if
-     *            false, job cannot be executed).
-     */
-    public SessionConfig(PrintWriter out, OutputFormat fmt, ResultDecorator preResultDecorator,
-            ResultDecorator postResultDecorator, boolean optimize, boolean executeQuery, boolean generateJobSpec) {
-        this.out = out;
-        this.fmt = fmt;
-        this.preResultDecorator = preResultDecorator;
-        this.postResultDecorator = postResultDecorator;
-        this.optimize = optimize;
-        this.executeQuery = executeQuery;
-        this.generateJobSpec = generateJobSpec;
-        this.flags = new HashMap<String, Boolean>();
-    }
-
-    /**
-     * Retrieve the PrintWriter to produce output to.
-     */
-    public PrintWriter out() {
-        return this.out;
-    }
-
-    /**
-     * Retrieve the OutputFormat for this execution.
-     */
-    public OutputFormat fmt() {
-        return this.fmt;
-    }
-
-    public AlgebricksAppendable resultPrefix(AlgebricksAppendable app) throws AlgebricksException {
-        return this.preResultDecorator != null ? this.preResultDecorator.append(app) : app;
-    };
-
-    public AlgebricksAppendable resultPostfix(AlgebricksAppendable app) throws AlgebricksException {
-        return this.postResultDecorator != null ? this.postResultDecorator.append(app) : app;
-    };
-
-    /**
-     * Retrieve the value of the "execute query" flag.
-     */
-    public boolean isExecuteQuery() {
-        return executeQuery;
-    }
-
-    /**
-     * Retrieve the value of the "optimize" flag.
-     */
-    public boolean isOptimize() {
-        return optimize;
-    }
-
-    /**
-     * Retrieve the value of the "generate job spec" flag.
-     */
-    public boolean isGenerateJobSpec() {
-        return generateJobSpec;
-    }
-
-    /**
-     * Specify all out-of-band settings at once. For convenience of older code.
-     */
-    public void setOOBData(boolean expr_tree, boolean rewritten_expr_tree, boolean logical_plan,
-            boolean optimized_logical_plan, boolean hyracks_job) {
-        this.set(OOB_EXPR_TREE, expr_tree);
-        this.set(OOB_REWRITTEN_EXPR_TREE, rewritten_expr_tree);
-        this.set(OOB_LOGICAL_PLAN, logical_plan);
-        this.set(OOB_OPTIMIZED_LOGICAL_PLAN, optimized_logical_plan);
-        this.set(OOB_HYRACKS_JOB, hyracks_job);
-    }
-
-    /**
-     * Specify a flag.
-     *
-     * @param flag
-     *            One of the OOB_ or FORMAT_ constants from this class.
-     * @param value
-     *            Value for the flag (all flags default to "false").
-     */
-    public void set(String flag, boolean value) {
-        flags.put(flag, Boolean.valueOf(value));
-    }
-
-    /**
-     * Retrieve the setting of a format-specific flag.
-     *
-     * @param flag
-     *            One of the FORMAT_ constants from this class.
-     * @returns true or false (all flags default to "false").
-     */
-    public boolean is(String flag) {
-        Boolean value = flags.get(flag);
-        return value == null ? false : value.booleanValue();
-    }
-}

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/APIServlet.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/APIServlet.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/APIServlet.java
index 204ff5c..b811172 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/APIServlet.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/APIServlet.java
@@ -18,6 +18,9 @@
  */
 package org.apache.asterix.api.http.servlet;
 
+import static org.apache.asterix.api.http.servlet.ServletConstants.HYRACKS_CONNECTION_ATTR;
+import static org.apache.asterix.api.http.servlet.ServletConstants.HYRACKS_DATASET_ATTR;
+
 import java.awt.image.BufferedImage;
 import java.io.BufferedReader;
 import java.io.IOException;
@@ -35,40 +38,38 @@ import javax.servlet.http.HttpServlet;
 import javax.servlet.http.HttpServletRequest;
 import javax.servlet.http.HttpServletResponse;
 
-import org.apache.asterix.api.common.APIFramework;
-import org.apache.asterix.api.common.SessionConfig;
-import org.apache.asterix.api.common.SessionConfig.OutputFormat;
-import org.apache.asterix.aql.translator.QueryTranslator;
+import org.apache.asterix.app.result.ResultReader;
+import org.apache.asterix.app.result.ResultUtil;
+import org.apache.asterix.common.app.SessionConfig;
+import org.apache.asterix.common.app.SessionConfig.OutputFormat;
 import org.apache.asterix.common.config.GlobalConfig;
 import org.apache.asterix.common.exceptions.AsterixException;
-import org.apache.asterix.compiler.provider.AqlCompilationProvider;
 import org.apache.asterix.compiler.provider.ILangCompilationProvider;
-import org.apache.asterix.compiler.provider.SqlppCompilationProvider;
 import org.apache.asterix.lang.aql.parser.TokenMgrError;
 import org.apache.asterix.lang.common.base.IParser;
 import org.apache.asterix.lang.common.base.IParserFactory;
 import org.apache.asterix.lang.common.base.Statement;
 import org.apache.asterix.metadata.MetadataManager;
-import org.apache.asterix.result.ResultReader;
-import org.apache.asterix.result.ResultUtils;
+import org.apache.asterix.translator.IStatementExecutor;
+import org.apache.asterix.translator.IStatementExecutorFactory;
 import org.apache.hyracks.api.client.IHyracksClientConnection;
 import org.apache.hyracks.api.dataset.IHyracksDataset;
 import org.apache.hyracks.client.dataset.HyracksDataset;
 
-import static org.apache.asterix.api.http.servlet.ServletConstants.HYRACKS_CONNECTION_ATTR;
-import static org.apache.asterix.api.http.servlet.ServletConstants.HYRACKS_DATASET_ATTR;
-
 public class APIServlet extends HttpServlet {
     private static final long serialVersionUID = 1L;
-
     private static final Logger LOGGER = Logger.getLogger(APIServlet.class.getName());
+    public static final String HTML_STATEMENT_SEPARATOR = "<!-- BEGIN -->";
 
     private final ILangCompilationProvider aqlCompilationProvider;
     private final ILangCompilationProvider sqlppCompilationProvider;
+    private final transient IStatementExecutorFactory statementExectorFactory;
 
-    public APIServlet() {
-        this.aqlCompilationProvider = new AqlCompilationProvider();
-        this.sqlppCompilationProvider = new SqlppCompilationProvider();
+    public APIServlet(ILangCompilationProvider aqlCompilationProvider,
+            ILangCompilationProvider sqlppCompilationProvider, IStatementExecutorFactory statementExecutorFactory) {
+        this.aqlCompilationProvider = aqlCompilationProvider;
+        this.sqlppCompilationProvider = sqlppCompilationProvider;
+        this.statementExectorFactory = statementExecutorFactory;
     }
 
     @Override
@@ -124,20 +125,21 @@ public class APIServlet extends HttpServlet {
             sessionConfig.setOOBData(isSet(printExprParam), isSet(printRewrittenExprParam),
                     isSet(printLogicalPlanParam), isSet(printOptimizedLogicalPlanParam), isSet(printJob));
             MetadataManager.INSTANCE.init();
-            QueryTranslator translator = new QueryTranslator(aqlStatements, sessionConfig, compilationProvider);
+            IStatementExecutor translator =
+                    statementExectorFactory.create(aqlStatements, sessionConfig, compilationProvider);
             double duration = 0;
             long startTime = System.currentTimeMillis();
-            translator.compileAndExecute(hcc, hds, QueryTranslator.ResultDelivery.SYNC);
+            translator.compileAndExecute(hcc, hds, IStatementExecutor.ResultDelivery.SYNC);
             long endTime = System.currentTimeMillis();
             duration = (endTime - startTime) / 1000.00;
-            out.println(APIFramework.HTML_STATEMENT_SEPARATOR);
+            out.println(HTML_STATEMENT_SEPARATOR);
             out.println("<PRE>Duration of all jobs: " + duration + " sec</PRE>");
         } catch (AsterixException | TokenMgrError | org.apache.asterix.aqlplus.parser.TokenMgrError pe) {
             GlobalConfig.ASTERIX_LOGGER.log(Level.INFO, pe.toString(), pe);
-            ResultUtils.webUIParseExceptionHandler(out, pe, query);
+            ResultUtil.webUIParseExceptionHandler(out, pe, query);
         } catch (Exception e) {
             GlobalConfig.ASTERIX_LOGGER.log(Level.SEVERE, e.getMessage(), e);
-            ResultUtils.webUIErrorHandler(out, e);
+            ResultUtil.webUIErrorHandler(out, e);
         }
     }
 

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/AQLAPIServlet.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/AQLAPIServlet.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/AQLAPIServlet.java
index de19715..bbb9dde 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/AQLAPIServlet.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/AQLAPIServlet.java
@@ -18,21 +18,22 @@
  */
 package org.apache.asterix.api.http.servlet;
 
-import java.util.List;
-
 import javax.servlet.http.HttpServletRequest;
 
 import org.apache.asterix.compiler.provider.ILangCompilationProvider;
 import org.apache.asterix.lang.common.base.Statement;
+import org.apache.asterix.translator.IStatementExecutorFactory;
 
 public class AQLAPIServlet extends RESTAPIServlet {
 
     private static final long serialVersionUID = 1L;
     private static final String AQL_STMT_PARAM_NAME = "aql";
-    private static final List<Byte> ALLOWED_STATEMENTS = Statement.KINDS;
+    private static final byte ALLOWED_CATEGORIES = Statement.Category.QUERY | Statement.Category.UPDATE
+            | Statement.Category.DDL | Statement.Category.PROCEDURE;
 
-    public AQLAPIServlet(ILangCompilationProvider compilationProvider) {
-        super(compilationProvider);
+    public AQLAPIServlet(ILangCompilationProvider compilationProvider,
+            IStatementExecutorFactory statementExecutorFactory) {
+        super(compilationProvider, statementExecutorFactory);
     }
 
     @Override
@@ -41,13 +42,12 @@ public class AQLAPIServlet extends RESTAPIServlet {
     }
 
     @Override
-    protected List<Byte> getAllowedStatements() {
-        return ALLOWED_STATEMENTS;
+    protected byte getAllowedCategories() {
+        return ALLOWED_CATEGORIES;
     }
 
     @Override
     protected String getErrorMessage() {
         throw new IllegalStateException();
     }
-
 }

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/ClusterAPIServlet.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/ClusterAPIServlet.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/ClusterAPIServlet.java
index 8f3b14b..eb23902 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/ClusterAPIServlet.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/ClusterAPIServlet.java
@@ -25,8 +25,8 @@ import javax.servlet.http.HttpServlet;
 import javax.servlet.http.HttpServletRequest;
 import javax.servlet.http.HttpServletResponse;
 
+import org.apache.asterix.app.result.ResultUtil;
 import org.apache.asterix.om.util.AsterixClusterProperties;
-import org.apache.asterix.result.ResultUtils;
 import org.json.JSONException;
 import org.json.JSONObject;
 
@@ -43,7 +43,7 @@ public class ClusterAPIServlet extends HttpServlet {
             responseWriter.write(responseObject.toString());
             response.setStatus(HttpServletResponse.SC_OK);
         } catch (JSONException e) {
-            ResultUtils.apiErrorHandler(responseWriter, e);
+            ResultUtil.apiErrorHandler(responseWriter, e);
             response.setStatus(HttpServletResponse.SC_INTERNAL_SERVER_ERROR);
         }
         responseWriter.flush();

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/ConnectorAPIServlet.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/ConnectorAPIServlet.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/ConnectorAPIServlet.java
index bca6d35..c43377a 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/ConnectorAPIServlet.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/ConnectorAPIServlet.java
@@ -19,6 +19,8 @@
 
 package org.apache.asterix.api.http.servlet;
 
+import static org.apache.asterix.api.http.servlet.ServletConstants.HYRACKS_CONNECTION_ATTR;
+
 import java.io.IOException;
 import java.io.PrintWriter;
 import java.util.List;
@@ -43,8 +45,6 @@ import org.json.JSONArray;
 import org.json.JSONException;
 import org.json.JSONObject;
 
-import static org.apache.asterix.api.http.servlet.ServletConstants.HYRACKS_CONNECTION_ATTR;
-
 /***
  * The REST API that takes a dataverse name and a dataset name as the input
  * and returns an array of file splits (IP, file-path) of the dataset in LOSSLESS_JSON.
@@ -94,8 +94,8 @@ public class ConnectorAPIServlet extends HttpServlet {
                 return;
             }
             boolean temp = dataset.getDatasetDetails().isTemp();
-            FileSplit[] fileSplits = metadataProvider.splitsForDataset(mdTxnCtx, dataverseName, datasetName,
-                    datasetName, temp);
+            FileSplit[] fileSplits =
+                    metadataProvider.splitsForDataset(mdTxnCtx, dataverseName, datasetName, datasetName, temp);
             ARecordType recordType = (ARecordType) metadataProvider.findType(dataset.getItemTypeDataverseName(),
                     dataset.getItemTypeName());
             List<List<String>> primaryKeys = DatasetUtils.getPartitioningKeys(dataset);

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/DDLAPIServlet.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/DDLAPIServlet.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/DDLAPIServlet.java
index 943691d..19a7609 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/DDLAPIServlet.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/DDLAPIServlet.java
@@ -18,29 +18,20 @@
  */
 package org.apache.asterix.api.http.servlet;
 
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.List;
-
 import javax.servlet.http.HttpServletRequest;
 
 import org.apache.asterix.compiler.provider.ILangCompilationProvider;
 import org.apache.asterix.lang.common.base.Statement;
+import org.apache.asterix.translator.IStatementExecutorFactory;
 
 public class DDLAPIServlet extends RESTAPIServlet {
     private static final long serialVersionUID = 1L;
+    private static final byte ALLOWED_CATEGORIES = Statement.Category.QUERY | Statement.Category.UPDATE
+            | Statement.Category.DDL;
 
-    private static final List<Byte> allowedStatements = Collections.unmodifiableList(Arrays.asList(
-            Statement.Kind.DATAVERSE_DECL, Statement.Kind.DATAVERSE_DROP, Statement.Kind.DATASET_DECL,
-            Statement.Kind.NODEGROUP_DECL, Statement.Kind.NODEGROUP_DROP, Statement.Kind.TYPE_DECL,
-            Statement.Kind.TYPE_DROP, Statement.Kind.CREATE_INDEX, Statement.Kind.INDEX_DECL,
-            Statement.Kind.CREATE_DATAVERSE, Statement.Kind.DATASET_DROP, Statement.Kind.INDEX_DROP,
-            Statement.Kind.CREATE_FUNCTION, Statement.Kind.FUNCTION_DROP, Statement.Kind.CREATE_PRIMARY_FEED,
-            Statement.Kind.CREATE_SECONDARY_FEED, Statement.Kind.DROP_FEED, Statement.Kind.CREATE_FEED_POLICY,
-            Statement.Kind.DROP_FEED_POLICY));
-
-    public DDLAPIServlet(ILangCompilationProvider compilationProvider) {
-        super(compilationProvider);
+    public DDLAPIServlet(ILangCompilationProvider compilationProvider,
+            IStatementExecutorFactory statementExecutorFactory) {
+        super(compilationProvider, statementExecutorFactory);
     }
 
     @Override
@@ -49,8 +40,8 @@ public class DDLAPIServlet extends RESTAPIServlet {
     }
 
     @Override
-    protected List<Byte> getAllowedStatements() {
-        return allowedStatements;
+    protected byte getAllowedCategories() {
+        return ALLOWED_CATEGORIES;
     }
 
     @Override

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/QueryAPIServlet.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/QueryAPIServlet.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/QueryAPIServlet.java
index 63aa2df..cc70324 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/QueryAPIServlet.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/QueryAPIServlet.java
@@ -18,24 +18,19 @@
  */
 package org.apache.asterix.api.http.servlet;
 
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.List;
-
 import javax.servlet.http.HttpServletRequest;
 
 import org.apache.asterix.compiler.provider.ILangCompilationProvider;
 import org.apache.asterix.lang.common.base.Statement;
+import org.apache.asterix.translator.IStatementExecutorFactory;
 
 public class QueryAPIServlet extends RESTAPIServlet {
     private static final long serialVersionUID = 1L;
+    private static final byte ALLOWED_CATEGORIES = Statement.Category.QUERY;
 
-    private static final List<Byte> allowedStatements = Collections.unmodifiableList(Arrays.asList(
-            Statement.Kind.DATAVERSE_DECL, Statement.Kind.FUNCTION_DECL, Statement.Kind.QUERY, Statement.Kind.SET,
-            Statement.Kind.WRITE, Statement.Kind.RUN));
-
-    public QueryAPIServlet(ILangCompilationProvider compilationProvider) {
-        super(compilationProvider);
+    public QueryAPIServlet(ILangCompilationProvider compilationProvider,
+            IStatementExecutorFactory queryTranslatorFactory) {
+        super(compilationProvider, queryTranslatorFactory);
     }
 
     @Override
@@ -44,8 +39,8 @@ public class QueryAPIServlet extends RESTAPIServlet {
     }
 
     @Override
-    protected List<Byte> getAllowedStatements() {
-        return allowedStatements;
+    protected byte getAllowedCategories() {
+        return ALLOWED_CATEGORIES;
     }
 
     @Override

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/QueryResultAPIServlet.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/QueryResultAPIServlet.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/QueryResultAPIServlet.java
index 9d29adf..fdbf253 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/QueryResultAPIServlet.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/QueryResultAPIServlet.java
@@ -18,6 +18,9 @@
  */
 package org.apache.asterix.api.http.servlet;
 
+import static org.apache.asterix.api.http.servlet.ServletConstants.HYRACKS_CONNECTION_ATTR;
+import static org.apache.asterix.api.http.servlet.ServletConstants.HYRACKS_DATASET_ATTR;
+
 import java.io.IOException;
 import java.io.PrintWriter;
 
@@ -26,9 +29,10 @@ import javax.servlet.http.HttpServlet;
 import javax.servlet.http.HttpServletRequest;
 import javax.servlet.http.HttpServletResponse;
 
-import org.apache.asterix.api.common.SessionConfig;
-import org.apache.asterix.result.ResultReader;
-import org.apache.asterix.result.ResultUtils;
+import org.apache.asterix.app.result.ResultReader;
+import org.apache.asterix.app.result.ResultUtil;
+import org.apache.asterix.common.app.SessionConfig;
+import org.apache.asterix.translator.IStatementExecutor.Stats;
 import org.apache.hyracks.api.client.HyracksConnection;
 import org.apache.hyracks.api.client.IHyracksClientConnection;
 import org.apache.hyracks.api.dataset.IHyracksDataset;
@@ -38,9 +42,6 @@ import org.apache.hyracks.client.dataset.HyracksDataset;
 import org.json.JSONArray;
 import org.json.JSONObject;
 
-import static org.apache.asterix.api.http.servlet.ServletConstants.HYRACKS_CONNECTION_ATTR;
-import static org.apache.asterix.api.http.servlet.ServletConstants.HYRACKS_DATASET_ATTR;
-
 public class QueryResultAPIServlet extends HttpServlet {
     private static final long serialVersionUID = 1L;
 
@@ -77,7 +78,7 @@ public class QueryResultAPIServlet extends HttpServlet {
             JobId jobId = new JobId(handle.getLong(0));
             ResultSetId rsId = new ResultSetId(handle.getLong(1));
 
-            ResultReader resultReader = new ResultReader(hcc, hds);
+            ResultReader resultReader = new ResultReader(hds);
             resultReader.open(jobId, rsId);
 
             // QQQ The output format is determined by the initial
@@ -87,7 +88,7 @@ public class QueryResultAPIServlet extends HttpServlet {
             // originally determined there. Need to save this value on
             // some object that we can obtain here.
             SessionConfig sessionConfig = RESTAPIServlet.initResponse(request, response);
-            ResultUtils.displayResults(resultReader, sessionConfig, new ResultUtils.Stats(), null);
+            ResultUtil.displayResults(resultReader, sessionConfig, new Stats(), null);
 
         } catch (Exception e) {
             out.println(e.getMessage());

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/QueryServiceServlet.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/QueryServiceServlet.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/QueryServiceServlet.java
index 90e8adf..5b91f2f 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/QueryServiceServlet.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/QueryServiceServlet.java
@@ -18,6 +18,9 @@
  */
 package org.apache.asterix.api.http.servlet;
 
+import static org.apache.asterix.api.http.servlet.ServletConstants.HYRACKS_CONNECTION_ATTR;
+import static org.apache.asterix.api.http.servlet.ServletConstants.HYRACKS_DATASET_ATTR;
+
 import java.io.IOException;
 import java.io.PrintWriter;
 import java.io.StringWriter;
@@ -33,19 +36,20 @@ import javax.servlet.http.HttpServlet;
 import javax.servlet.http.HttpServletRequest;
 import javax.servlet.http.HttpServletResponse;
 
-import org.apache.asterix.api.common.SessionConfig;
-import org.apache.asterix.aql.translator.QueryTranslator;
+import org.apache.asterix.app.result.ResultReader;
+import org.apache.asterix.app.translator.QueryTranslator;
+import org.apache.asterix.common.app.SessionConfig;
 import org.apache.asterix.common.config.GlobalConfig;
 import org.apache.asterix.common.exceptions.AsterixException;
 import org.apache.asterix.common.utils.JSONUtil;
 import org.apache.asterix.compiler.provider.ILangCompilationProvider;
-import org.apache.asterix.compiler.provider.SqlppCompilationProvider;
 import org.apache.asterix.lang.aql.parser.TokenMgrError;
 import org.apache.asterix.lang.common.base.IParser;
 import org.apache.asterix.lang.common.base.Statement;
 import org.apache.asterix.metadata.MetadataManager;
-import org.apache.asterix.result.ResultReader;
-import org.apache.asterix.result.ResultUtils;
+import org.apache.asterix.translator.IStatementExecutor;
+import org.apache.asterix.translator.IStatementExecutor.Stats;
+import org.apache.asterix.translator.IStatementExecutorFactory;
 import org.apache.commons.io.IOUtils;
 import org.apache.commons.lang3.exception.ExceptionUtils;
 import org.apache.hyracks.algebricks.core.algebra.prettyprint.AlgebricksAppendable;
@@ -53,15 +57,18 @@ import org.apache.hyracks.api.client.IHyracksClientConnection;
 import org.apache.hyracks.api.dataset.IHyracksDataset;
 import org.apache.hyracks.client.dataset.HyracksDataset;
 
-import static org.apache.asterix.api.http.servlet.ServletConstants.HYRACKS_CONNECTION_ATTR;
-import static org.apache.asterix.api.http.servlet.ServletConstants.HYRACKS_DATASET_ATTR;
-
 public class QueryServiceServlet extends HttpServlet {
     private static final long serialVersionUID = 1L;
 
     private static final Logger LOGGER = Logger.getLogger(QueryServiceServlet.class.getName());
+    private final transient ILangCompilationProvider compilationProvider;
+    private final transient IStatementExecutorFactory statementExecutorFactory;
 
-    private transient final ILangCompilationProvider compilationProvider;
+    public QueryServiceServlet(ILangCompilationProvider compilationProvider,
+            IStatementExecutorFactory statementExecutorFactory) {
+        this.compilationProvider = compilationProvider;
+        this.statementExecutorFactory = statementExecutorFactory;
+    }
 
     public enum Parameter {
         // Standard
@@ -210,10 +217,6 @@ public class QueryServiceServlet extends HttpServlet {
         }
     }
 
-    public QueryServiceServlet(final ILangCompilationProvider compilationProvider) {
-        this.compilationProvider = compilationProvider;
-    }
-
     private static String getParameterValue(String content, String attribute) {
         if (content == null || attribute == null) {
             return null;
@@ -396,12 +399,12 @@ public class QueryServiceServlet extends HttpServlet {
         response.setContentType(MediaType.JSON.str());
 
         int respCode = HttpServletResponse.SC_OK;
-        ResultUtils.Stats stats = new ResultUtils.Stats();
+        Stats stats = new Stats();
         long execStart = 0;
         long execEnd = -1;
 
         resultWriter.print("{\n");
-        UUID requestId = printRequestId(resultWriter);
+        printRequestId(resultWriter);
         printSignature(resultWriter);
         printType(resultWriter, sessionConfig);
         try {
@@ -419,7 +422,8 @@ public class QueryServiceServlet extends HttpServlet {
             IParser parser = compilationProvider.getParserFactory().createParser(query);
             List<Statement> aqlStatements = parser.parse();
             MetadataManager.INSTANCE.init();
-            QueryTranslator translator = new QueryTranslator(aqlStatements, sessionConfig, compilationProvider);
+            IStatementExecutor translator =
+                    statementExecutorFactory.create(aqlStatements, sessionConfig, compilationProvider);
             execStart = System.nanoTime();
             translator.compileAndExecute(hcc, hds, QueryTranslator.ResultDelivery.SYNC, stats);
             execEnd = System.nanoTime();
@@ -439,7 +443,8 @@ public class QueryServiceServlet extends HttpServlet {
                 execEnd = System.nanoTime();
             }
         }
-        printMetrics(resultWriter, System.nanoTime() - elapsedStart, execEnd - execStart, stats.count, stats.size);
+        printMetrics(resultWriter, System.nanoTime() - elapsedStart, execEnd - execStart, stats.getCount(),
+                stats.getSize());
         resultWriter.print("}\n");
         resultWriter.flush();
         String result = stringWriter.toString();

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/QueryStatusAPIServlet.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/QueryStatusAPIServlet.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/QueryStatusAPIServlet.java
index 2eb5e81..9555e11 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/QueryStatusAPIServlet.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/QueryStatusAPIServlet.java
@@ -18,6 +18,9 @@
  */
 package org.apache.asterix.api.http.servlet;
 
+import static org.apache.asterix.api.http.servlet.ServletConstants.HYRACKS_CONNECTION_ATTR;
+import static org.apache.asterix.api.http.servlet.ServletConstants.HYRACKS_DATASET_ATTR;
+
 import java.io.IOException;
 import java.io.PrintWriter;
 
@@ -26,18 +29,14 @@ import javax.servlet.http.HttpServlet;
 import javax.servlet.http.HttpServletRequest;
 import javax.servlet.http.HttpServletResponse;
 
-import org.json.JSONArray;
-import org.json.JSONObject;
-
-import org.apache.asterix.result.ResultReader;
+import org.apache.asterix.app.result.ResultReader;
 import org.apache.hyracks.api.client.IHyracksClientConnection;
 import org.apache.hyracks.api.dataset.IHyracksDataset;
 import org.apache.hyracks.api.dataset.ResultSetId;
 import org.apache.hyracks.api.job.JobId;
 import org.apache.hyracks.client.dataset.HyracksDataset;
-
-import static org.apache.asterix.api.http.servlet.ServletConstants.HYRACKS_CONNECTION_ATTR;
-import static org.apache.asterix.api.http.servlet.ServletConstants.HYRACKS_DATASET_ATTR;
+import org.json.JSONArray;
+import org.json.JSONObject;
 
 public class QueryStatusAPIServlet extends HttpServlet {
     private static final long serialVersionUID = 1L;
@@ -70,7 +69,7 @@ public class QueryStatusAPIServlet extends HttpServlet {
             /* TODO(madhusudancs): We need to find a way to LOSSLESS_JSON serialize default format obtained from
              * metadataProvider in the AQLTranslator and store it as part of the result handle.
              */
-            ResultReader resultReader = new ResultReader(hcc, hds);
+            ResultReader resultReader = new ResultReader(hds);
             resultReader.open(jobId, rsId);
 
             JSONObject jsonResponse = new JSONObject();

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/RESTAPIServlet.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/RESTAPIServlet.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/RESTAPIServlet.java
index 81bf21d..100efed 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/RESTAPIServlet.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/RESTAPIServlet.java
@@ -18,6 +18,9 @@
  */
 package org.apache.asterix.api.http.servlet;
 
+import static org.apache.asterix.api.http.servlet.ServletConstants.HYRACKS_CONNECTION_ATTR;
+import static org.apache.asterix.api.http.servlet.ServletConstants.HYRACKS_DATASET_ATTR;
+
 import java.io.IOException;
 import java.io.StringWriter;
 import java.nio.charset.StandardCharsets;
@@ -30,9 +33,11 @@ import javax.servlet.http.HttpServlet;
 import javax.servlet.http.HttpServletRequest;
 import javax.servlet.http.HttpServletResponse;
 
-import org.apache.asterix.api.common.SessionConfig;
-import org.apache.asterix.api.common.SessionConfig.OutputFormat;
-import org.apache.asterix.aql.translator.QueryTranslator;
+import org.apache.asterix.app.result.ResultReader;
+import org.apache.asterix.app.result.ResultUtil;
+import org.apache.asterix.app.translator.QueryTranslator;
+import org.apache.asterix.common.app.SessionConfig;
+import org.apache.asterix.common.app.SessionConfig.OutputFormat;
 import org.apache.asterix.common.config.GlobalConfig;
 import org.apache.asterix.common.exceptions.AsterixException;
 import org.apache.asterix.compiler.provider.ILangCompilationProvider;
@@ -41,26 +46,26 @@ import org.apache.asterix.lang.common.base.IParser;
 import org.apache.asterix.lang.common.base.IParserFactory;
 import org.apache.asterix.lang.common.base.Statement;
 import org.apache.asterix.metadata.MetadataManager;
-import org.apache.asterix.result.ResultReader;
-import org.apache.asterix.result.ResultUtils;
+import org.apache.asterix.translator.IStatementExecutor;
+import org.apache.asterix.translator.IStatementExecutorFactory;
 import org.apache.commons.io.IOUtils;
 import org.apache.hyracks.api.client.IHyracksClientConnection;
 import org.apache.hyracks.api.dataset.IHyracksDataset;
 import org.apache.hyracks.client.dataset.HyracksDataset;
 import org.json.JSONObject;
 
-import static org.apache.asterix.api.http.servlet.ServletConstants.HYRACKS_CONNECTION_ATTR;
-import static org.apache.asterix.api.http.servlet.ServletConstants.HYRACKS_DATASET_ATTR;
-
 abstract class RESTAPIServlet extends HttpServlet {
     private static final long serialVersionUID = 1L;
 
     private final ILangCompilationProvider compilationProvider;
     private final IParserFactory parserFactory;
+    private final transient IStatementExecutorFactory statementExecutorFactory;
 
-    public RESTAPIServlet(ILangCompilationProvider compilationProvider) {
+    public RESTAPIServlet(ILangCompilationProvider compilationProvider,
+            IStatementExecutorFactory statementExecutorFactory) {
         this.compilationProvider = compilationProvider;
         this.parserFactory = compilationProvider.getParserFactory();
+        this.statementExecutorFactory = statementExecutorFactory;
     }
 
     /**
@@ -193,26 +198,27 @@ abstract class RESTAPIServlet extends HttpServlet {
             List<Statement> aqlStatements = parser.parse();
             if (!containsForbiddenStatements(aqlStatements)) {
                 MetadataManager.INSTANCE.init();
-                QueryTranslator translator = new QueryTranslator(aqlStatements, sessionConfig, compilationProvider);
+                IStatementExecutor translator =
+                        statementExecutorFactory.create(aqlStatements, sessionConfig, compilationProvider);
                 translator.compileAndExecute(hcc, hds, resultDelivery);
             }
         } catch (AsterixException | TokenMgrError | org.apache.asterix.aqlplus.parser.TokenMgrError pe) {
             GlobalConfig.ASTERIX_LOGGER.log(Level.SEVERE, pe.getMessage(), pe);
-            String errorMessage = ResultUtils.buildParseExceptionMessage(pe, query);
+            String errorMessage = ResultUtil.buildParseExceptionMessage(pe, query);
             JSONObject errorResp =
-                    ResultUtils.getErrorResponse(2, errorMessage, "", ResultUtils.extractFullStackTrace(pe));
+                    ResultUtil.getErrorResponse(2, errorMessage, "", ResultUtil.extractFullStackTrace(pe));
             sessionConfig.out().write(errorResp.toString());
             response.setStatus(HttpServletResponse.SC_INTERNAL_SERVER_ERROR);
         } catch (Exception e) {
             GlobalConfig.ASTERIX_LOGGER.log(Level.SEVERE, e.getMessage(), e);
-            ResultUtils.apiErrorHandler(sessionConfig.out(), e);
+            ResultUtil.apiErrorHandler(sessionConfig.out(), e);
             response.setStatus(HttpServletResponse.SC_INTERNAL_SERVER_ERROR);
         }
     }
 
     private boolean containsForbiddenStatements(List<Statement> aqlStatements) throws AsterixException {
         for (Statement st : aqlStatements) {
-            if (!getAllowedStatements().contains(st.getKind())) {
+            if ((st.getCategory() & getAllowedCategories()) == 0) {
                 throw new AsterixException(String.format(getErrorMessage(), st.getKind()));
             }
         }
@@ -233,7 +239,7 @@ abstract class RESTAPIServlet extends HttpServlet {
 
     protected abstract String getQueryParameter(HttpServletRequest request);
 
-    protected abstract List<Byte> getAllowedStatements();
+    protected abstract byte getAllowedCategories();
 
     protected abstract String getErrorMessage();
 }

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/UpdateAPIServlet.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/UpdateAPIServlet.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/UpdateAPIServlet.java
index afa202b..1c62cdb 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/UpdateAPIServlet.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/UpdateAPIServlet.java
@@ -18,25 +18,19 @@
  */
 package org.apache.asterix.api.http.servlet;
 
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.List;
-
 import javax.servlet.http.HttpServletRequest;
 
 import org.apache.asterix.compiler.provider.ILangCompilationProvider;
 import org.apache.asterix.lang.common.base.Statement;
+import org.apache.asterix.translator.IStatementExecutorFactory;
 
 public class UpdateAPIServlet extends RESTAPIServlet {
     private static final long serialVersionUID = 1L;
-    private static final List<Byte> allowedStatements = Collections.unmodifiableList(Arrays.asList(
-            Statement.Kind.DATAVERSE_DECL, Statement.Kind.DELETE, Statement.Kind.INSERT,
-            Statement.Kind.UPSERT, Statement.Kind.UPDATE, Statement.Kind.DML_CMD_LIST, Statement.Kind.LOAD,
-            Statement.Kind.CONNECT_FEED, Statement.Kind.DISCONNECT_FEED, Statement.Kind.SET,
-            Statement.Kind.COMPACT, Statement.Kind.EXTERNAL_DATASET_REFRESH, Statement.Kind.RUN));
+    private static final byte ALLOWED_CATEGORIES = Statement.Category.QUERY | Statement.Category.UPDATE;
 
-    public UpdateAPIServlet(ILangCompilationProvider compilationProvider) {
-        super(compilationProvider);
+    public UpdateAPIServlet(ILangCompilationProvider compilationProvider,
+            IStatementExecutorFactory statementExecutorFactory) {
+        super(compilationProvider, statementExecutorFactory);
     }
 
     @Override
@@ -45,8 +39,8 @@ public class UpdateAPIServlet extends RESTAPIServlet {
     }
 
     @Override
-    protected List<Byte> getAllowedStatements() {
-        return allowedStatements;
+    protected byte getAllowedCategories() {
+        return ALLOWED_CATEGORIES;
     }
 
     @Override

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/java/AsterixJavaClient.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/java/AsterixJavaClient.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/java/AsterixJavaClient.java
index 964893b..e358963 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/java/AsterixJavaClient.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/java/AsterixJavaClient.java
@@ -24,14 +24,16 @@ import java.util.List;
 
 import org.apache.asterix.api.common.APIFramework;
 import org.apache.asterix.api.common.Job;
-import org.apache.asterix.api.common.SessionConfig;
-import org.apache.asterix.api.common.SessionConfig.OutputFormat;
-import org.apache.asterix.aql.translator.QueryTranslator;
+import org.apache.asterix.app.translator.QueryTranslator;
+import org.apache.asterix.common.app.SessionConfig;
+import org.apache.asterix.common.app.SessionConfig.OutputFormat;
 import org.apache.asterix.compiler.provider.ILangCompilationProvider;
 import org.apache.asterix.lang.common.base.IParser;
 import org.apache.asterix.lang.common.base.IParserFactory;
 import org.apache.asterix.lang.common.base.Statement;
 import org.apache.asterix.metadata.MetadataManager;
+import org.apache.asterix.translator.IStatementExecutor;
+import org.apache.asterix.translator.IStatementExecutorFactory;
 import org.apache.hyracks.api.client.IHyracksClientConnection;
 import org.apache.hyracks.api.job.JobSpecification;
 
@@ -46,20 +48,22 @@ public class AsterixJavaClient {
     private final ILangCompilationProvider compilationProvider;
     private final IParserFactory parserFactory;
     private final APIFramework apiFramework;
+    private final IStatementExecutorFactory statementExecutorFactory;
 
     public AsterixJavaClient(IHyracksClientConnection hcc, Reader queryText, PrintWriter writer,
-            ILangCompilationProvider compilationProvider) {
+            ILangCompilationProvider compilationProvider, IStatementExecutorFactory statementExecutorFactory) {
         this.hcc = hcc;
         this.queryText = queryText;
         this.writer = writer;
         this.compilationProvider = compilationProvider;
+        this.apiFramework = new APIFramework(compilationProvider, null);
+        this.statementExecutorFactory = statementExecutorFactory;
         parserFactory = compilationProvider.getParserFactory();
-        this.apiFramework = new APIFramework(compilationProvider);
     }
 
     public AsterixJavaClient(IHyracksClientConnection hcc, Reader queryText,
-            ILangCompilationProvider compilationProvider) {
-        this(hcc, queryText, new PrintWriter(System.out, true), compilationProvider);
+            ILangCompilationProvider compilationProvider, IStatementExecutorFactory statementExecutorFactory) {
+        this(hcc, queryText, new PrintWriter(System.out, true), compilationProvider, statementExecutorFactory);
     }
 
     public void compile() throws Exception {
@@ -68,7 +72,7 @@ public class AsterixJavaClient {
 
     public void compile(boolean optimize, boolean printRewrittenExpressions, boolean printLogicalPlan,
             boolean printOptimizedPlan, boolean printPhysicalOpsOnly, boolean generateBinaryRuntime, boolean printJob)
-                    throws Exception {
+            throws Exception {
         queryJobSpec = null;
         dmlJobs = null;
 
@@ -90,7 +94,7 @@ public class AsterixJavaClient {
             conf.set(SessionConfig.FORMAT_ONLY_PHYSICAL_OPS, true);
         }
 
-        QueryTranslator translator = new QueryTranslator(statements, conf, compilationProvider);
+        IStatementExecutor translator = statementExecutorFactory.create(statements, conf, compilationProvider);
         translator.compileAndExecute(hcc, null, QueryTranslator.ResultDelivery.SYNC);
         writer.flush();
     }

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/ab81748a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/cc/CompilerExtensionManager.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/cc/CompilerExtensionManager.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/cc/CompilerExtensionManager.java
new file mode 100644
index 0000000..af10394
--- /dev/null
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/cc/CompilerExtensionManager.java
@@ -0,0 +1,156 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.app.cc;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.asterix.algebra.base.ILangExtension;
+import org.apache.asterix.algebra.base.ILangExtension.Language;
+import org.apache.asterix.algebra.extension.ExtensionFunctionIdentifier;
+import org.apache.asterix.algebra.extension.IAlgebraExtensionManager;
+import org.apache.asterix.app.translator.DefaultStatementExecutorFactory;
+import org.apache.asterix.common.api.ExtensionId;
+import org.apache.asterix.common.api.IExtension;
+import org.apache.asterix.common.config.AsterixExtension;
+import org.apache.asterix.common.exceptions.ErrorCode;
+import org.apache.asterix.compiler.provider.AqlCompilationProvider;
+import org.apache.asterix.compiler.provider.ILangCompilationProvider;
+import org.apache.asterix.compiler.provider.SqlppCompilationProvider;
+import org.apache.asterix.translator.IStatementExecutorFactory;
+import org.apache.commons.lang3.mutable.Mutable;
+import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
+import org.apache.hyracks.algebricks.common.utils.Pair;
+import org.apache.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import org.apache.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import org.apache.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import org.apache.hyracks.algebricks.core.algebra.expressions.AbstractFunctionCallExpression;
+import org.apache.hyracks.algebricks.core.algebra.functions.FunctionIdentifier;
+import org.apache.hyracks.algebricks.core.algebra.operators.logical.UnnestOperator;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+
+/**
+ * AsterixDB's implementation of {@code IAlgebraExtensionManager} which takes care of
+ * initializing extensions for App and Compilation purposes
+ */
+public class CompilerExtensionManager implements IAlgebraExtensionManager {
+
+    private static final String ERROR_MESSAGE_ID_CONFLICT = "Two Extensions share the same Id: %1$s";
+    public static final String ERROR_MESSAGE_COMPONENT_CONFLICT =
+            "Extension Conflict between %1$s and %2$s both extensions extend %3$s";
+
+    private final Map<ExtensionId, IExtension> extensions = new HashMap<>();
+
+    private final IStatementExecutorExtension statementExecutorExtension;
+    private final ILangCompilationProvider aqlCompilationProvider;
+    private final ILangCompilationProvider sqlppCompilationProvider;
+    private final DefaultStatementExecutorFactory defaultQueryTranslatorFactory;
+
+    /**
+     * Initialize {@code CompilerExtensionManager} from configuration
+     *
+     * @param list
+     * @throws InstantiationException
+     * @throws IllegalAccessException
+     * @throws ClassNotFoundException
+     * @throws HyracksDataException
+     */
+    public CompilerExtensionManager(List<AsterixExtension> list)
+            throws InstantiationException, IllegalAccessException, ClassNotFoundException, HyracksDataException {
+        Pair<ExtensionId, ILangCompilationProvider> aqlcp = null;
+        Pair<ExtensionId, ILangCompilationProvider> sqlppcp = null;
+        IStatementExecutorExtension see = null;
+        defaultQueryTranslatorFactory = new DefaultStatementExecutorFactory(this);
+
+        if (list != null) {
+            for (AsterixExtension extensionConf : list) {
+                IExtension extension = (IExtension) Class.forName(extensionConf.getClassName()).newInstance();
+                extension.configure(extensionConf.getArgs());
+                if (extensions.containsKey(extension.getId())) {
+                    throw new HyracksDataException(ErrorCode.ASTERIX, ErrorCode.ERROR_EXTENSION_CONFLICT,
+                            ERROR_MESSAGE_ID_CONFLICT, extension.getId());
+                }
+                extensions.put(extension.getId(), extension);
+                switch (extension.getExtensionKind()) {
+                    case STATEMENT_EXECUTOR:
+                        see = extendStatementExecutor(see, (IStatementExecutorExtension) extension);
+                        break;
+                    case LANG:
+                        ILangExtension le = (ILangExtension) extension;
+                        aqlcp = extendLangCompilationProvider(Language.AQL, aqlcp, le);
+                        sqlppcp = extendLangCompilationProvider(Language.SQLPP, sqlppcp, le);
+                        break;
+                    default:
+                        break;
+                }
+            }
+        }
+        this.statementExecutorExtension = see;
+        this.aqlCompilationProvider = aqlcp == null ? new AqlCompilationProvider() : aqlcp.second;
+        this.sqlppCompilationProvider = sqlppcp == null ? new SqlppCompilationProvider() : sqlppcp.second;
+    }
+
+    private Pair<ExtensionId, ILangCompilationProvider> extendLangCompilationProvider(Language lang,
+            Pair<ExtensionId, ILangCompilationProvider> cp, ILangExtension le) throws HyracksDataException {
+        if (cp != null && le.getLangCompilationProvider(lang) != null) {
+            throw new HyracksDataException(ErrorCode.ASTERIX, ErrorCode.ERROR_EXTENSION_CONFLICT,
+                    ERROR_MESSAGE_COMPONENT_CONFLICT, le.getId(), cp.first, lang.toString());
+        }
+        return (le.getLangCompilationProvider(lang) != null)
+                ? new Pair<>(le.getId(), le.getLangCompilationProvider(lang)) : cp;
+    }
+
+    private IStatementExecutorExtension extendStatementExecutor(IStatementExecutorExtension qte,
+            IStatementExecutorExtension extension) throws HyracksDataException {
+        if (qte != null) {
+            throw new HyracksDataException(ErrorCode.ASTERIX, ErrorCode.ERROR_EXTENSION_CONFLICT,
+                    ERROR_MESSAGE_COMPONENT_CONFLICT, qte.getId(), extension.getId(),
+                    IStatementExecutorFactory.class.getSimpleName());
+        }
+        return extension;
+    }
+
+    public IStatementExecutorFactory getQueryTranslatorFactory() {
+        return statementExecutorExtension == null ? defaultQueryTranslatorFactory
+                : statementExecutorExtension.getQueryTranslatorFactory();
+    }
+
+    public ILangCompilationProvider getAqlCompilationProvider() {
+        return aqlCompilationProvider;
+    }
+
+    public ILangCompilationProvider getSqlppCompilationProvider() {
+        return sqlppCompilationProvider;
+    }
+
+    // TODO(amoudi/yingyi): this is not a good way to extend rewrite rules. introduce re-write rule provider
+    @Override
+    public boolean unnestToDataScan(Mutable<ILogicalOperator> opRef, IOptimizationContext context,
+            UnnestOperator unnestOp, ILogicalExpression unnestExpr, AbstractFunctionCallExpression functionCallExpr)
+            throws AlgebricksException {
+        FunctionIdentifier functionId = functionCallExpr.getFunctionIdentifier();
+        if (functionId instanceof ExtensionFunctionIdentifier) {
+            ILangExtension extension =
+                    (ILangExtension) extensions.get(((ExtensionFunctionIdentifier) functionId).getExtensionId());
+            return extension.unnestToDataScan(opRef, context, unnestOp, unnestExpr, functionCallExpr);
+        }
+        return false;
+    }
+}


Mime
View raw message