asterixdb-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From buyin...@apache.org
Subject [8/8] asterixdb git commit: Supports flexible degree of parallelism.
Date Thu, 22 Dec 2016 01:04:32 GMT
Supports flexible degree of parallelism.

Detailed list of changes include:
- Add the IClusterInfoCollector interface and feed that to the compiler;
- Add the number of cores into the heartbeat;
- Add partition calculation in APIFramework;
- Fix partitioning property requirements in binary operators such as Join and UnionAll;
- Fix PushSubplanIntoGroupByRule for type propagation;
- Fix InlineLeftNtsInSubplanJoinFlatteningVisitor for the returned top join operator;
- Add unit test for partition calculation in APIFrameworkTest
- Add integration test SqlppExecutionFullParallelismIT, SqlppExecutionLessParallelismIT,
  AqlExecutionFullParallelismIT, and AqlExecutionLessParallelismIT for different
  parallelism settings;
- Refactor AQLExecutionTest and SqlppExecutionTest;
- Separate one cluster state test out from AqlExecutionTest.

Change-Id: If70faf52bed995c7098cc343f2fabd1b6c8d96e7
Reviewed-on: https://asterix-gerrit.ics.uci.edu/1398
Tested-by: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
BAD: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Integration-Tests: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Reviewed-by: Till Westmann <tillw@apache.org>


Project: http://git-wip-us.apache.org/repos/asf/asterixdb/repo
Commit: http://git-wip-us.apache.org/repos/asf/asterixdb/commit/03e940a7
Tree: http://git-wip-us.apache.org/repos/asf/asterixdb/tree/03e940a7
Diff: http://git-wip-us.apache.org/repos/asf/asterixdb/diff/03e940a7

Branch: refs/heads/master
Commit: 03e940a7781dbdbb654fe13ec7077aae74e1c416
Parents: 35ea53b
Author: Yingyi Bu <yingyi@couchbase.com>
Authored: Wed Dec 21 14:18:52 2016 -0800
Committer: Yingyi Bu <buyingyi@gmail.com>
Committed: Wed Dec 21 17:03:51 2016 -0800

----------------------------------------------------------------------
 ...neLeftNtsInSubplanJoinFlatteningVisitor.java |  12 +-
 .../asterix/translator/IStatementExecutor.java  |   8 +-
 .../apache/asterix/api/common/APIFramework.java |  66 +-
 .../asterix/app/translator/QueryTranslator.java |  22 +-
 .../resources/asterix-build-configuration.xml   |   4 -
 .../resources/asterix-build-configuration2.xml  | 116 +++
 .../resources/asterix-build-configuration3.xml  | 116 +++
 .../asterix/api/common/APIFrameworkTest.java    | 134 +++
 .../http/servlet/ConnectorAPIServletTest.java   |   6 +-
 .../api/http/servlet/VersionAPIServletTest.java |  18 +-
 .../runtime/AqlExecutionFullParallelismIT.java  |  65 ++
 .../runtime/AqlExecutionLessParallelismIT.java  |  65 ++
 .../asterix/test/runtime/AqlExecutionTest.java  |  65 ++
 ...lusterStateExecutionFullParallelismTest.java |  65 ++
 ...lusterStateExecutionLessParallelismTest.java |  65 ++
 .../test/runtime/ClusterStateExecutionTest.java |  65 ++
 .../asterix/test/runtime/ExecutionTest.java     | 139 ---
 .../asterix/test/runtime/ExecutionTestUtil.java |   2 +-
 .../asterix/test/runtime/LangExecutionUtil.java | 115 +++
 .../asterix/test/runtime/RepeatedTest.java      |   9 +-
 .../SqlppExecutionFullParallelismIT.java        |  63 ++
 .../SqlppExecutionLessParallelismIT.java        |  65 ++
 .../test/runtime/SqlppExecutionTest.java        |  56 +-
 .../results/nonpure/query-ASTERIXDB-1608.plan   |  16 +-
 .../optimizerts/results/query-issue827-2.plan   |  44 +-
 .../test/resources/runtimets/cluster_state.xml  |  28 +
 .../resources/runtimets/cluster_state_full.xml  |  28 +
 .../resources/runtimets/cluster_state_less.xml  |  28 +
 .../resources/runtimets/only_cluster_state.xml  |  25 +
 .../runtimets/queries/api/APIQueries.xml        |   5 -
 .../queries/flwor/at00/at00.3.query.aql         |   6 +-
 .../queries/flwor/at00/at00.4.asyncdefer.aql    |   4 +-
 .../queries/flwor/at00/at00.5.async.aql         |   4 +-
 .../queries/flwor/at06/at06.3.query.aql         |   4 +-
 .../query-issue456/query-issue456.3.query.aql   |   3 +-
 .../query-issue601/query-issue601.3.query.aql   |   1 +
 .../query-issue785-2.3.query.aql                |   1 +
 .../query-issue785/query-issue785.3.query.aql   |   1 +
 .../query-issue601/query-issue601.3.query.aql   |   1 +
 .../query-issue785-2.3.query.aql                |   3 +-
 .../query-issue785/query-issue785.3.query.aql   |   1 +
 .../queries_sqlpp/flwor/at00/at00.3.query.sqlpp |   9 +-
 .../flwor/at00/at00.4.asyncdefer.sqlpp          |   9 +-
 .../queries_sqlpp/flwor/at00/at00.5.async.sqlpp |  10 +-
 .../queries_sqlpp/flwor/at06/at06.3.query.sqlpp |  15 +-
 .../query-issue456/query-issue456.3.query.sqlpp |   5 +-
 .../group-by.26.query.sqlpp                     |   5 +-
 .../query-issue601/query-issue601.3.query.sqlpp |   1 +
 .../query-issue785-2.3.query.sqlpp              |   5 +-
 .../query-issue785/query-issue785.3.query.sqlpp |   5 +-
 .../query-issue601/query-issue601.3.query.sqlpp |   1 +
 .../query-issue785-2.3.query.sqlpp              |   1 +
 .../query-issue785/query-issue785.3.query.sqlpp |   1 +
 .../query-issue601/query-issue601.3.query.sqlpp |   1 +
 .../query-issue785-2.3.query.sqlpp              |   5 +-
 .../query-issue785/query-issue785.3.query.sqlpp |   5 +-
 .../query-issue601/query-issue601.3.query.sqlpp |   5 +-
 .../query-issue785-2.3.query.sqlpp              |   5 +-
 .../query-issue785/query-issue785.3.query.sqlpp |   5 +-
 .../query-issue601/query-issue601.3.query.sqlpp |   5 +-
 .../query-issue785-2.3.query.sqlpp              |   1 +
 .../query-issue785/query-issue785.3.query.sqlpp |   1 +
 .../types/any-object/any-object.2.query.sqlpp   |   4 +-
 .../api/cluster_state_1/cluster_state_1.1.adm   |   3 +-
 .../cluster_state_1_full.1.adm                  | 129 +++
 .../cluster_state_1_less.1.adm                  | 129 +++
 .../runtimets/results/flwor/at00/at00.1.adm     | 920 +++++++++---------
 .../runtimets/results/flwor/at00/at00.2.adm     | 920 +++++++++---------
 .../runtimets/results/flwor/at00/at00.3.adm     | 920 +++++++++---------
 .../runtimets/results/flwor/at06/at06.1.adm     | 922 +++++++++----------
 .../tinysocial-suite-open/group-by.26.adm       |  12 +-
 .../query-issue601/query-issue601.1.adm         |   4 +-
 .../query-issue785-2/query-issue785-2.1.adm     |   8 +-
 .../query-issue785/query-issue785.1.adm         |  30 +-
 .../query-issue601/query-issue601.1.adm         |   4 +-
 .../query-issue785-2/query-issue785-2.1.adm     |  20 +-
 .../query-issue785/query-issue785.1.adm         |  48 +-
 .../query-issue601/query-issue601.1.adm         |   4 +-
 .../query-issue785-2/query-issue785-2.1.adm     |   8 +-
 .../query-issue785/query-issue785.1.adm         |  30 +-
 .../query-issue601/query-issue601.1.adm         |   4 +-
 .../query-issue785-2/query-issue785-2.1.adm     |   8 +-
 .../query-issue785/query-issue785.1.adm         |  30 +-
 .../tpch/query-issue601/query-issue601.1.adm    |   4 +-
 .../query-issue785-2/query-issue785-2.1.adm     |   8 +-
 .../tpch/query-issue785/query-issue785.1.adm    |  30 +-
 .../results_parser_sqlpp/flwor/at00/at00.3.ast  |  64 +-
 .../results_parser_sqlpp/flwor/at00/at00.4.ast  |  64 +-
 .../results_parser_sqlpp/flwor/at00/at00.5.ast  |  64 +-
 .../results_parser_sqlpp/flwor/at06/at06.3.ast  |  63 +-
 .../query-issue456/query-issue456.3.ast         |  14 +-
 .../tinysocial-suite.26.ast                     |  14 +-
 .../query-issue601/query-issue601.3.ast         |   4 +
 .../query-issue785-2/query-issue785-2.3.ast     |  12 +-
 .../query-issue785/query-issue785.3.ast         |  12 +-
 .../tpch/query-issue601/query-issue601.3.ast    |  12 +-
 .../query-issue785-2/query-issue785-2.3.ast     |   4 +
 .../tpch/query-issue785/query-issue785.3.ast    |   4 +
 .../common/config/CompilerProperties.java       |  14 +-
 .../physical/AbstractHashJoinPOperator.java     |  49 +-
 .../physical/NestedLoopJoinPOperator.java       |   9 +-
 .../operators/physical/UnionAllPOperator.java   |  34 +-
 .../properties/StructuralPropertiesVector.java  |   3 +-
 .../algebra/util/OperatorPropertiesUtil.java    |  20 +
 .../subplan/PushSubplanIntoGroupByRule.java     |   3 +-
 .../hyracks/api/client/HyracksConnection.java   |  16 +-
 .../api/client/IClusterInfoCollector.java       |  46 +
 .../api/client/IHyracksClientConnection.java    |  30 +-
 .../hyracks/api/client/NodeControllerInfo.java  |   9 +-
 .../hyracks/control/cc/NodeControllerState.java |   8 +
 .../cc/work/GetNodeControllersInfoWork.java     |   5 +-
 .../control/common/heartbeat/HeartbeatData.java |   3 +
 .../control/nc/NodeControllerService.java       |   1 +
 .../std/union/UnionAllOperatorDescriptor.java   |  12 +-
 .../hyracks/hdfs/scheduler/SchedulerTest.java   |   4 +-
 .../apache/hyracks/hdfs/utils/TestUtils.java    |   2 +-
 116 files changed, 3951 insertions(+), 2491 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/asterixdb/blob/03e940a7/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/subplan/InlineLeftNtsInSubplanJoinFlatteningVisitor.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/subplan/InlineLeftNtsInSubplanJoinFlatteningVisitor.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/subplan/InlineLeftNtsInSubplanJoinFlatteningVisitor.java
index d3a0c0f..6a318d6 100644
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/subplan/InlineLeftNtsInSubplanJoinFlatteningVisitor.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/subplan/InlineLeftNtsInSubplanJoinFlatteningVisitor.java
@@ -38,10 +38,10 @@ import org.apache.hyracks.algebricks.core.algebra.operators.logical.AbstractBina
 import org.apache.hyracks.algebricks.core.algebra.operators.logical.AggregateOperator;
 import org.apache.hyracks.algebricks.core.algebra.operators.logical.AssignOperator;
 import org.apache.hyracks.algebricks.core.algebra.operators.logical.DataSourceScanOperator;
+import org.apache.hyracks.algebricks.core.algebra.operators.logical.DelegateOperator;
 import org.apache.hyracks.algebricks.core.algebra.operators.logical.DistinctOperator;
 import org.apache.hyracks.algebricks.core.algebra.operators.logical.EmptyTupleSourceOperator;
 import org.apache.hyracks.algebricks.core.algebra.operators.logical.ExchangeOperator;
-import org.apache.hyracks.algebricks.core.algebra.operators.logical.DelegateOperator;
 import org.apache.hyracks.algebricks.core.algebra.operators.logical.GroupByOperator;
 import org.apache.hyracks.algebricks.core.algebra.operators.logical.InnerJoinOperator;
 import org.apache.hyracks.algebricks.core.algebra.operators.logical.IntersectOperator;
@@ -52,7 +52,6 @@ import org.apache.hyracks.algebricks.core.algebra.operators.logical.LimitOperato
 import org.apache.hyracks.algebricks.core.algebra.operators.logical.MaterializeOperator;
 import org.apache.hyracks.algebricks.core.algebra.operators.logical.NestedTupleSourceOperator;
 import org.apache.hyracks.algebricks.core.algebra.operators.logical.OrderOperator;
-import org.apache.hyracks.algebricks.core.algebra.operators.logical.OrderOperator.IOrder;
 import org.apache.hyracks.algebricks.core.algebra.operators.logical.ProjectOperator;
 import org.apache.hyracks.algebricks.core.algebra.operators.logical.ReplicateOperator;
 import org.apache.hyracks.algebricks.core.algebra.operators.logical.RunningAggregateOperator;
@@ -64,6 +63,7 @@ import org.apache.hyracks.algebricks.core.algebra.operators.logical.TokenizeOper
 import org.apache.hyracks.algebricks.core.algebra.operators.logical.UnionAllOperator;
 import org.apache.hyracks.algebricks.core.algebra.operators.logical.UnnestMapOperator;
 import org.apache.hyracks.algebricks.core.algebra.operators.logical.UnnestOperator;
+import org.apache.hyracks.algebricks.core.algebra.operators.logical.OrderOperator.IOrder;
 import org.apache.hyracks.algebricks.core.algebra.operators.logical.visitors.VariableUtilities;
 import org.apache.hyracks.algebricks.core.algebra.visitors.IQueryOperatorVisitor;
 
@@ -381,11 +381,9 @@ class InlineLeftNtsInSubplanJoinFlatteningVisitor implements IQueryOperatorVisit
             // Deals with single input operators.
             Mutable<ILogicalOperator> childRef = op.getInputs().get(0);
             ILogicalOperator newChild = childRef.getValue().accept(this, null);
-            if (topJoinRef == null) {
-                LogicalOperatorTag childOpTag = newChild.getOperatorTag();
-                if (childOpTag == LogicalOperatorTag.INNERJOIN || childOpTag == LogicalOperatorTag.LEFTOUTERJOIN) {
-                    topJoinRef = childRef;
-                }
+            LogicalOperatorTag childOpTag = newChild.getOperatorTag();
+            if (childOpTag == LogicalOperatorTag.INNERJOIN || childOpTag == LogicalOperatorTag.LEFTOUTERJOIN) {
+                topJoinRef = childRef;
             }
             op.getInputs().get(0).setValue(newChild);
         }

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/03e940a7/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/IStatementExecutor.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/IStatementExecutor.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/IStatementExecutor.java
index a1f3055..2066f73 100644
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/IStatementExecutor.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/IStatementExecutor.java
@@ -26,6 +26,7 @@ import org.apache.asterix.lang.common.statement.Query;
 import org.apache.asterix.metadata.declared.MetadataProvider;
 import org.apache.asterix.translator.CompiledStatements.ICompiledDmlStatement;
 import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
+import org.apache.hyracks.api.client.IClusterInfoCollector;
 import org.apache.hyracks.api.client.IHyracksClientConnection;
 import org.apache.hyracks.api.dataset.IHyracksDataset;
 import org.apache.hyracks.api.job.JobSpecification;
@@ -108,6 +109,8 @@ public interface IStatementExecutor {
     /**
      * rewrites and compiles query into a hyracks job specifications
      *
+     * @param clusterInfoCollector
+     *            The cluster info collector
      * @param metadataProvider
      *            The metadataProvider used to access metadata and build runtimes
      * @param query
@@ -120,9 +123,8 @@ public interface IStatementExecutor {
      * @throws AlgebricksException
      * @throws ACIDException
      */
-    JobSpecification rewriteCompileQuery(MetadataProvider metadataProvider, Query query,
-            ICompiledDmlStatement dmlStatement)
-                    throws AsterixException, RemoteException, AlgebricksException, ACIDException;
+    JobSpecification rewriteCompileQuery(IClusterInfoCollector clusterInfoCollector, MetadataProvider metadataProvider,
+            Query query, ICompiledDmlStatement dmlStatement) throws RemoteException, AlgebricksException, ACIDException;
 
     /**
      * returns the active dataverse for an entity or a statement

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/03e940a7/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/APIFramework.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/APIFramework.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/APIFramework.java
index e27edb1..16646d2 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/APIFramework.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/APIFramework.java
@@ -18,10 +18,18 @@
  */
 package org.apache.asterix.api.common;
 
+
 import java.io.IOException;
 import java.io.PrintWriter;
+import java.rmi.Remote;
 import java.rmi.RemoteException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashSet;
 import java.util.List;
+import java.util.Map;
+import java.util.Random;
+import java.util.Set;
 
 import org.apache.asterix.algebra.base.ILangExpressionToPlanTranslator;
 import org.apache.asterix.algebra.base.ILangExpressionToPlanTranslatorFactory;
@@ -55,6 +63,7 @@ import org.apache.asterix.transaction.management.service.transaction.JobIdFactor
 import org.apache.asterix.translator.CompiledStatements.ICompiledDmlStatement;
 import org.apache.asterix.translator.IStatementExecutor.Stats;
 import org.apache.asterix.translator.SessionConfig;
+import org.apache.hyracks.algebricks.common.constraints.AlgebricksAbsolutePartitionConstraint;
 import org.apache.hyracks.algebricks.common.constraints.AlgebricksPartitionConstraint;
 import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
 import org.apache.hyracks.algebricks.common.utils.Pair;
@@ -76,7 +85,9 @@ import org.apache.hyracks.algebricks.core.algebra.prettyprint.PlanPrettyPrinter;
 import org.apache.hyracks.algebricks.core.rewriter.base.AlgebricksOptimizationContext;
 import org.apache.hyracks.algebricks.core.rewriter.base.IOptimizationContextFactory;
 import org.apache.hyracks.algebricks.core.rewriter.base.PhysicalOptimizationConfig;
+import org.apache.hyracks.api.client.IClusterInfoCollector;
 import org.apache.hyracks.api.client.IHyracksClientConnection;
+import org.apache.hyracks.api.client.NodeControllerInfo;
 import org.apache.hyracks.api.job.JobId;
 import org.apache.hyracks.api.job.JobSpecification;
 import org.json.JSONException;
@@ -150,7 +161,7 @@ public class APIFramework {
         return new Pair<>(q, q.getVarCounter());
     }
 
-    public JobSpecification compileQuery(List<FunctionDecl> declaredFunctions,
+    public JobSpecification compileQuery(IClusterInfoCollector clusterInfoCollector,
             MetadataProvider metadataProvider, Query rwQ, int varCounter, String outputDatasetName,
             SessionConfig conf, ICompiledDmlStatement statement)
             throws AlgebricksException, RemoteException, ACIDException {
@@ -219,7 +230,10 @@ public class APIFramework {
         builder.setExpressionTypeComputer(ExpressionTypeComputer.INSTANCE);
         builder.setMissableTypeComputer(AqlMissableTypeComputer.INSTANCE);
         builder.setConflictingTypeResolver(ConflictingTypeResolver.INSTANCE);
-        builder.setClusterLocations(metadataProvider.getClusterLocations());
+
+        int parallelism = compilerProperties.getParallelism();
+        builder.setClusterLocations(parallelism == CompilerProperties.COMPILER_PARALLELISM_AS_STORAGE
+                ? metadataProvider.getClusterLocations() : getComputationLocations(clusterInfoCollector, parallelism));
 
         ICompiler compiler = compilerFactory.createCompiler(plan, metadataProvider, t.getVarCounter());
         if (conf.isOptimize()) {
@@ -342,4 +356,52 @@ public class APIFramework {
             out.println("<pre>Duration: " + duration + " sec</pre>");
         }
     }
+
+    // Computes the location constraints based on user-configured parallelism parameter.
+    // Note that the parallelism parameter is only a hint -- it will not be respected if it is too small or too large.
+    private AlgebricksAbsolutePartitionConstraint getComputationLocations(IClusterInfoCollector clusterInfoCollector,
+            int parallelismHint) throws AlgebricksException {
+        try {
+            Map<String, NodeControllerInfo> ncMap = clusterInfoCollector.getNodeControllerInfos();
+
+            // Unifies the handling of non-positive parallelism.
+            int parallelism = parallelismHint <= 0 ? -2 * ncMap.size() : parallelismHint;
+
+            // Calculates per node parallelism, with load balance, i.e., randomly selecting nodes with larger
+            // parallelism.
+            int numNodes = ncMap.size();
+            int numNodesWithOneMorePartition = parallelism % numNodes;
+            int perNodeParallelismMin = parallelism / numNodes;
+            int perNodeParallelismMax = parallelism / numNodes + 1;
+            List<String> allNodes = new ArrayList<>();
+            Set<String> selectedNodesWithOneMorePartition = new HashSet<>();
+            for (Map.Entry<String, NodeControllerInfo> entry : ncMap.entrySet()) {
+                allNodes.add(entry.getKey());
+            }
+            Random random = new Random();
+            for (int index = numNodesWithOneMorePartition; index >= 1; --index) {
+                int pick = random.nextInt(index);
+                selectedNodesWithOneMorePartition.add(allNodes.get(pick));
+                Collections.swap(allNodes, pick, index - 1);
+            }
+
+            // Generates cluster locations, which has duplicates for a node if it contains more than one partitions.
+            List<String> locations = new ArrayList<>();
+            for (Map.Entry<String, NodeControllerInfo> entry : ncMap.entrySet()) {
+                String nodeId = entry.getKey();
+                int numCores = entry.getValue().getNumCores();
+                int availableCores = numCores > 1 ? numCores - 1 : numCores; // Reserves one core for heartbeat.
+                int nodeParallelism = selectedNodesWithOneMorePartition.contains(nodeId) ? perNodeParallelismMax
+                        : perNodeParallelismMin;
+                int coresToUse = nodeParallelism >= 0 && nodeParallelism < availableCores ? nodeParallelism
+                        : availableCores;
+                for (int count = 0; count < coresToUse; ++count) {
+                    locations.add(nodeId);
+                }
+            }
+            return new AlgebricksAbsolutePartitionConstraint(locations.toArray(new String[0]));
+        } catch (Exception e) {
+            throw new AlgebricksException(e);
+        }
+    }
 }

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/03e940a7/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
index 643a352..4fab8d7 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
@@ -53,13 +53,13 @@ import org.apache.asterix.app.external.FeedOperations;
 import org.apache.asterix.app.result.ResultHandle;
 import org.apache.asterix.app.result.ResultReader;
 import org.apache.asterix.app.result.ResultUtil;
-import org.apache.asterix.common.config.ExternalProperties;
 import org.apache.asterix.common.config.ClusterProperties;
+import org.apache.asterix.common.config.ExternalProperties;
+import org.apache.asterix.common.config.GlobalConfig;
 import org.apache.asterix.common.config.DatasetConfig.DatasetType;
 import org.apache.asterix.common.config.DatasetConfig.ExternalDatasetTransactionState;
 import org.apache.asterix.common.config.DatasetConfig.ExternalFilePendingOp;
 import org.apache.asterix.common.config.DatasetConfig.IndexType;
-import org.apache.asterix.common.config.GlobalConfig;
 import org.apache.asterix.common.exceptions.ACIDException;
 import org.apache.asterix.common.exceptions.AsterixException;
 import org.apache.asterix.common.functions.FunctionSignature;
@@ -68,8 +68,8 @@ import org.apache.asterix.external.api.IAdapterFactory;
 import org.apache.asterix.external.feed.api.IActiveLifecycleEventSubscriber;
 import org.apache.asterix.external.feed.api.IActiveLifecycleEventSubscriber.ActiveLifecycleEvent;
 import org.apache.asterix.external.feed.api.IFeed;
-import org.apache.asterix.external.feed.api.IFeed.FeedType;
 import org.apache.asterix.external.feed.api.IFeedJoint;
+import org.apache.asterix.external.feed.api.IFeed.FeedType;
 import org.apache.asterix.external.feed.api.IFeedJoint.FeedJointType;
 import org.apache.asterix.external.feed.management.ActiveLifecycleEventSubscriber;
 import org.apache.asterix.external.feed.management.FeedConnectionId;
@@ -193,6 +193,7 @@ import org.apache.hyracks.algebricks.data.IAWriterFactory;
 import org.apache.hyracks.algebricks.data.IResultSerializerFactoryProvider;
 import org.apache.hyracks.algebricks.runtime.serializer.ResultSerializerFactoryProvider;
 import org.apache.hyracks.algebricks.runtime.writers.PrinterBasedWriterFactory;
+import org.apache.hyracks.api.client.IClusterInfoCollector;
 import org.apache.hyracks.api.client.IHyracksClientConnection;
 import org.apache.hyracks.api.dataflow.value.ITypeTraits;
 import org.apache.hyracks.api.dataset.IHyracksDataset;
@@ -1836,7 +1837,7 @@ public class QueryTranslator extends AbstractLangTranslator implements IStatemen
             CompiledLoadFromFileStatement cls = new CompiledLoadFromFileStatement(dataverseName,
                     loadStmt.getDatasetName().getValue(), loadStmt.getAdapter(), loadStmt.getProperties(),
                     loadStmt.dataIsAlreadySorted());
-            JobSpecification spec = apiFramework.compileQuery(null, metadataProvider, null, 0, null, sessionConfig,
+            JobSpecification spec = apiFramework.compileQuery(hcc, metadataProvider, null, 0, null, sessionConfig,
                     cls);
             MetadataManager.INSTANCE.commitTransaction(mdTxnCtx);
             bActiveTxn = false;
@@ -1896,7 +1897,7 @@ public class QueryTranslator extends AbstractLangTranslator implements IStatemen
                 default:
                     throw new AlgebricksException("Unsupported statement type " + stmtInsertUpsert.getKind());
             }
-            JobSpecification jobSpec = rewriteCompileQuery(metadataProvider, query, clfrqs);
+            JobSpecification jobSpec = rewriteCompileQuery(hcc, metadataProvider, query, clfrqs);
 
             MetadataManager.INSTANCE.commitTransaction(mdTxnCtx);
             bActiveTxn = false;
@@ -1938,7 +1939,7 @@ public class QueryTranslator extends AbstractLangTranslator implements IStatemen
             CompiledDeleteStatement clfrqs = new CompiledDeleteStatement(stmtDelete.getVariableExpr(), dataverseName,
                     stmtDelete.getDatasetName().getValue(), stmtDelete.getCondition(), stmtDelete.getVarCounter(),
                     stmtDelete.getQuery());
-            JobSpecification jobSpec = rewriteCompileQuery(metadataProvider, clfrqs.getQuery(), clfrqs);
+            JobSpecification jobSpec = rewriteCompileQuery(hcc, metadataProvider, clfrqs.getQuery(), clfrqs);
 
             MetadataManager.INSTANCE.commitTransaction(mdTxnCtx);
             bActiveTxn = false;
@@ -1960,7 +1961,8 @@ public class QueryTranslator extends AbstractLangTranslator implements IStatemen
     }
 
     @Override
-    public JobSpecification rewriteCompileQuery(MetadataProvider metadataProvider, Query query,
+    public JobSpecification rewriteCompileQuery(IClusterInfoCollector clusterInfoCollector,
+            MetadataProvider metadataProvider, Query query,
             ICompiledDmlStatement stmt)
             throws AsterixException, RemoteException, AlgebricksException, ACIDException {
 
@@ -1969,7 +1971,7 @@ public class QueryTranslator extends AbstractLangTranslator implements IStatemen
                 sessionConfig);
 
         // Query Compilation (happens under the same ongoing metadata transaction)
-        JobSpecification spec = apiFramework.compileQuery(declaredFunctions, metadataProvider, reWrittenQuery.first,
+        JobSpecification spec = apiFramework.compileQuery(clusterInfoCollector, metadataProvider, reWrittenQuery.first,
                 reWrittenQuery.second, stmt == null ? null : stmt.getDatasetName(), sessionConfig, stmt);
 
         return spec;
@@ -2417,7 +2419,7 @@ public class QueryTranslator extends AbstractLangTranslator implements IStatemen
         metadataProvider.getConfig().put(FeedActivityDetails.COLLECT_LOCATIONS,
                 StringUtils.join(bfs.getLocations(), ','));
 
-        JobSpecification jobSpec = rewriteCompileQuery(metadataProvider, bfs.getQuery(), csfs);
+        JobSpecification jobSpec = rewriteCompileQuery(hcc, metadataProvider, bfs.getQuery(), csfs);
         FeedConnectionId feedConnectionId = new FeedConnectionId(bfs.getSubscriptionRequest().getReceivingFeedId(),
                 bfs.getSubscriptionRequest().getTargetDataset());
         String dataverse = feedConnectionId.getFeedId().getDataverse();
@@ -2560,7 +2562,7 @@ public class QueryTranslator extends AbstractLangTranslator implements IStatemen
         metadataProvider.setMetadataTxnContext(mdTxnCtx);
         MetadataLockManager.INSTANCE.queryBegin(activeDefaultDataverse, query.getDataverses(), query.getDatasets());
         try {
-            JobSpecification jobSpec = rewriteCompileQuery(metadataProvider, query, null);
+            JobSpecification jobSpec = rewriteCompileQuery(hcc, metadataProvider, query, null);
 
             MetadataManager.INSTANCE.commitTransaction(mdTxnCtx);
             bActiveTxn = false;

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/03e940a7/asterixdb/asterix-app/src/main/resources/asterix-build-configuration.xml
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/main/resources/asterix-build-configuration.xml b/asterixdb/asterix-app/src/main/resources/asterix-build-configuration.xml
index 90dbe5f..bad6a35 100644
--- a/asterixdb/asterix-app/src/main/resources/asterix-build-configuration.xml
+++ b/asterixdb/asterix-app/src/main/resources/asterix-build-configuration.xml
@@ -67,10 +67,6 @@
     <value>160KB</value>
   </property>
   <property>
-    <name>compiler.pregelix.home</name>
-    <value>~/pregelix</value>
-  </property>
-  <property>
     <name>storage.buffercache.pagesize</name>
     <value>32KB</value>
     <description>The page size in bytes for pages in the buffer cache.

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/03e940a7/asterixdb/asterix-app/src/main/resources/asterix-build-configuration2.xml
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/main/resources/asterix-build-configuration2.xml b/asterixdb/asterix-app/src/main/resources/asterix-build-configuration2.xml
new file mode 100644
index 0000000..7b8d274
--- /dev/null
+++ b/asterixdb/asterix-app/src/main/resources/asterix-build-configuration2.xml
@@ -0,0 +1,116 @@
+<!--
+ ! Licensed to the Apache Software Foundation (ASF) under one
+ ! or more contributor license agreements.  See the NOTICE file
+ ! distributed with this work for additional information
+ ! regarding copyright ownership.  The ASF licenses this file
+ ! to you under the Apache License, Version 2.0 (the
+ ! "License"); you may not use this file except in compliance
+ ! with the License.  You may obtain a copy of the License at
+ !
+ !   http://www.apache.org/licenses/LICENSE-2.0
+ !
+ ! Unless required by applicable law or agreed to in writing,
+ ! software distributed under the License is distributed on an
+ ! "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ ! KIND, either express or implied.  See the License for the
+ ! specific language governing permissions and limitations
+ ! under the License.
+ !-->
+<asterixConfiguration xmlns="asterixconf">
+    <metadataNode>asterix_nc1</metadataNode>
+    <store>
+        <ncId>asterix_nc1</ncId>
+        <storeDirs>iodevice0,iodevice1</storeDirs>
+    </store>
+    <store>
+        <ncId>asterix_nc2</ncId>
+        <storeDirs>iodevice0,iodevice1</storeDirs>
+    </store>
+    <transactionLogDir>
+        <ncId>asterix_nc1</ncId>
+        <txnLogDirPath>target/txnLogDir/asterix_nc1</txnLogDirPath>
+    </transactionLogDir>
+    <transactionLogDir>
+        <ncId>asterix_nc2</ncId>
+        <txnLogDirPath>target/txnLogDir/asterix_nc2</txnLogDirPath>
+    </transactionLogDir>
+
+    <property>
+        <name>max.wait.active.cluster</name>
+        <value>60</value>
+        <description>Maximum wait (in seconds) for a cluster to be ACTIVE (all
+            nodes are available)
+            before a submitted query/statement can be
+            executed. (Default = 60 seconds)
+        </description>
+    </property>
+
+    <property>
+        <name>log.level</name>
+        <value>INFO</value>
+        <description>Log level for running tests/build</description>
+    </property>
+    <property>
+        <name>compiler.framesize</name>
+        <value>32KB</value>
+    </property>
+    <property>
+        <name>compiler.sortmemory</name>
+        <value>320KB</value>
+    </property>
+    <property>
+        <name>compiler.groupmemory</name>
+        <value>160KB</value>
+    </property>
+    <property>
+        <name>compiler.joinmemory</name>
+        <value>160KB</value>
+    </property>
+    <property>
+        <name>compiler.parallelism</name>
+        <value>-1</value>
+    </property>
+    <property>
+        <name>storage.buffercache.pagesize</name>
+        <value>32KB</value>
+        <description>The page size in bytes for pages in the buffer cache.
+            (Default = "128KB")
+        </description>
+    </property>
+    <property>
+        <name>storage.buffercache.size</name>
+        <value>48MB</value>
+        <description>The size of memory allocated to the disk buffer cache.
+            The value should be a multiple of the buffer cache page size
+            (Default = "512MB")
+        </description>
+    </property>
+    <property>
+        <name>storage.memorycomponent.numpages</name>
+        <value>8</value>
+        <description>The number of pages to allocate for a memory component.
+            This budget is shared by all the memory components of the primary
+            index and all its secondary indexes across all I/O devices on a node.
+            Note: in-memory components usually has fill factor of 75% since
+            the pages are 75% full and the remaining 25% is un-utilized. (Default = 256)
+        </description>
+    </property>
+    <property>
+        <name>plot.activate</name>
+        <value>false</value>
+        <description>Enabling plot of Algebricks plan to tmp folder. (Default = false)
+        </description>
+    </property>
+    <property>
+        <name>messaging.frame.size</name>
+        <value>4096</value>
+        <description>The frame size to be used for NC to NC messaging. (Default = 4kb)
+        </description>
+    </property>
+    <property>
+        <name>messaging.frame.count</name>
+        <value>512</value>
+        <description>Number of reusable frames for NC to NC messaging. (Default = 512)
+        </description>
+    </property>
+</asterixConfiguration>

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/03e940a7/asterixdb/asterix-app/src/main/resources/asterix-build-configuration3.xml
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/main/resources/asterix-build-configuration3.xml b/asterixdb/asterix-app/src/main/resources/asterix-build-configuration3.xml
new file mode 100644
index 0000000..85b0cdb
--- /dev/null
+++ b/asterixdb/asterix-app/src/main/resources/asterix-build-configuration3.xml
@@ -0,0 +1,116 @@
+<!--
+ ! Licensed to the Apache Software Foundation (ASF) under one
+ ! or more contributor license agreements.  See the NOTICE file
+ ! distributed with this work for additional information
+ ! regarding copyright ownership.  The ASF licenses this file
+ ! to you under the Apache License, Version 2.0 (the
+ ! "License"); you may not use this file except in compliance
+ ! with the License.  You may obtain a copy of the License at
+ !
+ !   http://www.apache.org/licenses/LICENSE-2.0
+ !
+ ! Unless required by applicable law or agreed to in writing,
+ ! software distributed under the License is distributed on an
+ ! "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ ! KIND, either express or implied.  See the License for the
+ ! specific language governing permissions and limitations
+ ! under the License.
+ !-->
+<asterixConfiguration xmlns="asterixconf">
+    <metadataNode>asterix_nc1</metadataNode>
+    <store>
+        <ncId>asterix_nc1</ncId>
+        <storeDirs>iodevice0,iodevice1</storeDirs>
+    </store>
+    <store>
+        <ncId>asterix_nc2</ncId>
+        <storeDirs>iodevice0,iodevice1</storeDirs>
+    </store>
+    <transactionLogDir>
+        <ncId>asterix_nc1</ncId>
+        <txnLogDirPath>target/txnLogDir/asterix_nc1</txnLogDirPath>
+    </transactionLogDir>
+    <transactionLogDir>
+        <ncId>asterix_nc2</ncId>
+        <txnLogDirPath>target/txnLogDir/asterix_nc2</txnLogDirPath>
+    </transactionLogDir>
+
+    <property>
+        <name>max.wait.active.cluster</name>
+        <value>60</value>
+        <description>Maximum wait (in seconds) for a cluster to be ACTIVE (all
+            nodes are available)
+            before a submitted query/statement can be
+            executed. (Default = 60 seconds)
+        </description>
+    </property>
+
+    <property>
+        <name>log.level</name>
+        <value>INFO</value>
+        <description>Log level for running tests/build</description>
+    </property>
+    <property>
+        <name>compiler.framesize</name>
+        <value>32KB</value>
+    </property>
+    <property>
+        <name>compiler.sortmemory</name>
+        <value>320KB</value>
+    </property>
+    <property>
+        <name>compiler.groupmemory</name>
+        <value>160KB</value>
+    </property>
+    <property>
+        <name>compiler.joinmemory</name>
+        <value>160KB</value>
+    </property>
+    <property>
+        <name>compiler.parallelism</name>
+        <value>3</value>
+    </property>
+    <property>
+        <name>storage.buffercache.pagesize</name>
+        <value>32KB</value>
+        <description>The page size in bytes for pages in the buffer cache.
+            (Default = "128KB")
+        </description>
+    </property>
+    <property>
+        <name>storage.buffercache.size</name>
+        <value>48MB</value>
+        <description>The size of memory allocated to the disk buffer cache.
+            The value should be a multiple of the buffer cache page size
+            (Default = "512MB")
+        </description>
+    </property>
+    <property>
+        <name>storage.memorycomponent.numpages</name>
+        <value>8</value>
+        <description>The number of pages to allocate for a memory component.
+            This budget is shared by all the memory components of the primary
+            index and all its secondary indexes across all I/O devices on a node.
+            Note: in-memory components usually has fill factor of 75% since
+            the pages are 75% full and the remaining 25% is un-utilized. (Default = 256)
+        </description>
+    </property>
+    <property>
+        <name>plot.activate</name>
+        <value>false</value>
+        <description>Enabling plot of Algebricks plan to tmp folder. (Default = false)
+        </description>
+    </property>
+    <property>
+        <name>messaging.frame.size</name>
+        <value>4096</value>
+        <description>The frame size to be used for NC to NC messaging. (Default = 4kb)
+        </description>
+    </property>
+    <property>
+        <name>messaging.frame.count</name>
+        <value>512</value>
+        <description>Number of reusable frames for NC to NC messaging. (Default = 512)
+        </description>
+    </property>
+</asterixConfiguration>

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/03e940a7/asterixdb/asterix-app/src/test/java/org/apache/asterix/api/common/APIFrameworkTest.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/api/common/APIFrameworkTest.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/api/common/APIFrameworkTest.java
new file mode 100644
index 0000000..6256857
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/api/common/APIFrameworkTest.java
@@ -0,0 +1,134 @@
+/*
+ *
+ *  * Licensed to the Apache Software Foundation (ASF) under one
+ *  * or more contributor license agreements.  See the NOTICE file
+ *  * distributed with this work for additional information
+ *  * regarding copyright ownership.  The ASF licenses this file
+ *  * to you under the Apache License, Version 2.0 (the
+ *  * "License"); you may not use this file except in compliance
+ *  * with the License.  You may obtain a copy of the License at
+ *  *
+ *  *   http://www.apache.org/licenses/LICENSE-2.0
+ *  *
+ *  * Unless required by applicable law or agreed to in writing,
+ *  * software distributed under the License is distributed on an
+ *  * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ *  * KIND, either express or implied.  See the License for the
+ *  * specific language governing permissions and limitations
+ *  * under the License.
+ *
+ */
+
+package org.apache.asterix.api.common;
+
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.asterix.compiler.provider.ILangCompilationProvider;
+import org.apache.hyracks.algebricks.common.constraints.AlgebricksAbsolutePartitionConstraint;
+import org.apache.hyracks.api.client.IClusterInfoCollector;
+import org.apache.hyracks.api.client.NodeControllerInfo;
+import org.junit.Assert;
+import org.junit.Test;
+
+import junit.extensions.PA;
+
+public class APIFrameworkTest {
+
+    @Test
+    public void testGetComputationLocations() throws Exception {
+        IClusterInfoCollector clusterInfoCollector = mock(IClusterInfoCollector.class);
+
+        // Constructs mocked cluster nodes.
+        Map<String, NodeControllerInfo> map = new HashMap<>();
+        NodeControllerInfo nc1Info = mock(NodeControllerInfo.class);
+        when(nc1Info.getNumCores()).thenReturn(4);
+        NodeControllerInfo nc2Info = mock(NodeControllerInfo.class);
+        when(nc2Info.getNumCores()).thenReturn(4);
+        String nc1 = "nc1";
+        String nc2 = "nc2";
+        map.put(nc1, nc1Info);
+        map.put(nc2, nc2Info);
+        when(clusterInfoCollector.getNodeControllerInfos()).thenReturn(map);
+
+        // Creates an APIFramework.
+        APIFramework apiFramework = new APIFramework(mock(ILangCompilationProvider.class));
+
+        // Tests odd number parallelism.
+        AlgebricksAbsolutePartitionConstraint loc = (AlgebricksAbsolutePartitionConstraint) PA.invokeMethod(
+                apiFramework, "getComputationLocations(" + IClusterInfoCollector.class.getName() + ",int)",
+                clusterInfoCollector, 5);
+        int nc1Count = 0, nc2Count = 0;
+        String[] partitions = loc.getLocations();
+        for (String partition : partitions) {
+            if (partition.equals(nc1)) {
+                ++nc1Count;
+            }
+            if (partition.equals(nc2)) {
+                ++nc2Count;
+            }
+        }
+        Assert.assertTrue(nc1Count > 0);
+        Assert.assertTrue(nc2Count > 0);
+        Assert.assertTrue(Math.abs(nc1Count - nc2Count) == 1); // Tests load balance.
+        Assert.assertTrue(partitions.length == 5);
+
+        // Tests even number parallelism.
+        loc = (AlgebricksAbsolutePartitionConstraint) PA.invokeMethod(apiFramework,
+                "getComputationLocations(" + IClusterInfoCollector.class.getName() + ",int)", clusterInfoCollector, 8);
+        nc1Count = 0;
+        nc2Count = 0;
+        partitions = loc.getLocations();
+        for (String partition : partitions) {
+            if (partition.equals(nc1)) {
+                ++nc1Count;
+            }
+            if (partition.equals(nc2)) {
+                ++nc2Count;
+            }
+        }
+        Assert.assertTrue(nc1Count > 0);
+        Assert.assertTrue(nc2Count > 0);
+        Assert.assertTrue(Math.abs(nc1Count - nc2Count) == 0); // Tests load balance.
+        // The maximum parallelism cannot be beyond n *(#core-1), where n is the number of NCs and #core is the number
+        // of cores per NC.
+        Assert.assertTrue(partitions.length == 6);
+
+        // Tests the case when parallelism is one.
+        loc = (AlgebricksAbsolutePartitionConstraint) PA.invokeMethod(apiFramework,
+                "getComputationLocations(" + IClusterInfoCollector.class.getName() + ",int)", clusterInfoCollector, 1);
+        Assert.assertTrue(loc.getLocations().length == 1);
+
+        // Tests the case when parallelism is a negative.
+        // In this case, the compiler has no idea and falls back to the default setting where all possible cores
+        // are used.
+        loc = (AlgebricksAbsolutePartitionConstraint) PA.invokeMethod(apiFramework,
+                "getComputationLocations(" + IClusterInfoCollector.class.getName() + ",int)", clusterInfoCollector,
+                -100);
+        Assert.assertTrue(loc.getLocations().length == 6);
+
+        // Tests the case when parallelism is -1.
+        // In this case, the compiler has no idea and falls back to the default setting where all possible cores
+        // are used.
+        loc = (AlgebricksAbsolutePartitionConstraint) PA.invokeMethod(apiFramework,
+                "getComputationLocations(" + IClusterInfoCollector.class.getName() + ",int)", clusterInfoCollector, -1);
+        Assert.assertTrue(loc.getLocations().length == 6);
+
+        // Tests the case when parallelism is zero.
+        // In this case, the compiler has no idea and falls back to the default setting where all possible cores
+        // are used.
+        loc = (AlgebricksAbsolutePartitionConstraint) PA.invokeMethod(apiFramework,
+                "getComputationLocations(" + IClusterInfoCollector.class.getName() + ",int)", clusterInfoCollector, 0);
+        Assert.assertTrue(loc.getLocations().length == 6);
+
+        // Verifies the number of calls on clusterInfoCollector.getNodeControllerInfos() in
+        // APIFramework.getComputationLocations(...).
+        verify(clusterInfoCollector, times(6)).getNodeControllerInfos();
+    }
+
+}

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/03e940a7/asterixdb/asterix-app/src/test/java/org/apache/asterix/api/http/servlet/ConnectorAPIServletTest.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/api/http/servlet/ConnectorAPIServletTest.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/api/http/servlet/ConnectorAPIServletTest.java
index 71b2df5..fdcbad5 100644
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/api/http/servlet/ConnectorAPIServletTest.java
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/api/http/servlet/ConnectorAPIServletTest.java
@@ -44,7 +44,7 @@ import org.apache.asterix.om.types.ARecordType;
 import org.apache.asterix.om.types.BuiltinType;
 import org.apache.asterix.om.types.IAType;
 import org.apache.asterix.om.util.JSONDeserializerForTypes;
-import org.apache.asterix.test.runtime.ExecutionTest;
+import org.apache.asterix.test.runtime.SqlppExecutionTest;
 import org.apache.hyracks.api.client.IHyracksClientConnection;
 import org.apache.hyracks.api.client.NodeControllerInfo;
 import org.apache.hyracks.api.comm.NetworkAddress;
@@ -63,7 +63,7 @@ public class ConnectorAPIServletTest {
     @Test
     public void testGet() throws Exception {
         // Starts test asterixdb cluster.
-        ExecutionTest.setUp();
+        SqlppExecutionTest.setUp();
 
         // Configures a test connector api servlet.
         ConnectorAPIServlet servlet = spy(new ConnectorAPIServlet());
@@ -116,7 +116,7 @@ public class ConnectorAPIServletTest {
         Assert.assertTrue(path.endsWith("Metadata/Dataset_idx_Dataset"));
 
         // Tears down the asterixdb cluster.
-        ExecutionTest.tearDown();
+        SqlppExecutionTest.tearDown();
     }
 
     @Test

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/03e940a7/asterixdb/asterix-app/src/test/java/org/apache/asterix/api/http/servlet/VersionAPIServletTest.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/api/http/servlet/VersionAPIServletTest.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/api/http/servlet/VersionAPIServletTest.java
index df7c151..2209fc5 100644
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/api/http/servlet/VersionAPIServletTest.java
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/api/http/servlet/VersionAPIServletTest.java
@@ -19,6 +19,12 @@
 
 package org.apache.asterix.api.http.servlet;
 
+import static org.apache.asterix.api.http.servlet.ServletConstants.ASTERIX_BUILD_PROP_ATTR;
+import static org.apache.asterix.api.http.servlet.ServletConstants.HYRACKS_CONNECTION_ATTR;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.when;
+
 import java.io.ByteArrayInputStream;
 import java.io.ByteArrayOutputStream;
 import java.io.InputStreamReader;
@@ -33,25 +39,19 @@ import javax.servlet.http.HttpServletResponse;
 
 import org.apache.asterix.common.config.BuildProperties;
 import org.apache.asterix.runtime.util.AppContextInfo;
-import org.apache.asterix.test.runtime.ExecutionTest;
+import org.apache.asterix.test.runtime.SqlppExecutionTest;
 import org.apache.hyracks.api.client.IHyracksClientConnection;
 import org.json.JSONObject;
 import org.json.JSONTokener;
 import org.junit.Assert;
 import org.junit.Test;
 
-import static org.apache.asterix.api.http.servlet.ServletConstants.ASTERIX_BUILD_PROP_ATTR;
-import static org.apache.asterix.api.http.servlet.ServletConstants.HYRACKS_CONNECTION_ATTR;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.spy;
-import static org.mockito.Mockito.when;
-
 public class VersionAPIServletTest {
 
     @Test
     public void testGet() throws Exception {
         // Starts test asterixdb cluster.
-        ExecutionTest.setUp();
+        SqlppExecutionTest.setUp();
 
         // Configures a test version api servlet.
         VersionAPIServlet servlet = spy(new VersionAPIServlet());
@@ -112,6 +112,6 @@ public class VersionAPIServletTest {
         Assert.assertEquals(actualResponse.toString(), expectedResponse.toString());
 
         // Tears down the asterixdb cluster.
-        ExecutionTest.tearDown();
+        SqlppExecutionTest.tearDown();
     }
 }

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/03e940a7/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/AqlExecutionFullParallelismIT.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/AqlExecutionFullParallelismIT.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/AqlExecutionFullParallelismIT.java
new file mode 100644
index 0000000..5afcb55
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/AqlExecutionFullParallelismIT.java
@@ -0,0 +1,65 @@
+/*
+ *
+ *  * Licensed to the Apache Software Foundation (ASF) under one
+ *  * or more contributor license agreements.  See the NOTICE file
+ *  * distributed with this work for additional information
+ *  * regarding copyright ownership.  The ASF licenses this file
+ *  * to you under the Apache License, Version 2.0 (the
+ *  * "License"); you may not use this file except in compliance
+ *  * with the License.  You may obtain a copy of the License at
+ *  *
+ *  *   http://www.apache.org/licenses/LICENSE-2.0
+ *  *
+ *  * Unless required by applicable law or agreed to in writing,
+ *  * software distributed under the License is distributed on an
+ *  * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ *  * KIND, either express or implied.  See the License for the
+ *  * specific language governing permissions and limitations
+ *  * under the License.
+ *
+ */
+package org.apache.asterix.test.runtime;
+
+import java.util.Collection;
+
+import org.apache.asterix.testframework.context.TestCaseContext;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import org.junit.runners.Parameterized.Parameters;
+
+/**
+ * Runs the AQL runtime tests with full parallelism on node controllers.
+ */
+@RunWith(Parameterized.class)
+public class AqlExecutionFullParallelismIT {
+    protected static final String TEST_CONFIG_FILE_NAME = "asterix-build-configuration2.xml";
+
+    @BeforeClass
+    public static void setUp() throws Exception {
+        LangExecutionUtil.setUp(TEST_CONFIG_FILE_NAME);
+    }
+
+    @AfterClass
+    public static void tearDown() throws Exception {
+        LangExecutionUtil.tearDown();
+    }
+
+    @Parameters(name = "AqlExecutionFullParallelismIT {index}: {0}")
+    public static Collection<Object[]> tests() throws Exception {
+        return LangExecutionUtil.tests("only.xml", "testsuite.xml");
+    }
+
+    protected TestCaseContext tcCtx;
+
+    public AqlExecutionFullParallelismIT(TestCaseContext tcCtx) {
+        this.tcCtx = tcCtx;
+    }
+
+    @Test
+    public void test() throws Exception {
+        LangExecutionUtil.test(tcCtx);
+    }
+}

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/03e940a7/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/AqlExecutionLessParallelismIT.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/AqlExecutionLessParallelismIT.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/AqlExecutionLessParallelismIT.java
new file mode 100644
index 0000000..da826a3
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/AqlExecutionLessParallelismIT.java
@@ -0,0 +1,65 @@
+/*
+ *
+ *  * Licensed to the Apache Software Foundation (ASF) under one
+ *  * or more contributor license agreements.  See the NOTICE file
+ *  * distributed with this work for additional information
+ *  * regarding copyright ownership.  The ASF licenses this file
+ *  * to you under the Apache License, Version 2.0 (the
+ *  * "License"); you may not use this file except in compliance
+ *  * with the License.  You may obtain a copy of the License at
+ *  *
+ *  *   http://www.apache.org/licenses/LICENSE-2.0
+ *  *
+ *  * Unless required by applicable law or agreed to in writing,
+ *  * software distributed under the License is distributed on an
+ *  * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ *  * KIND, either express or implied.  See the License for the
+ *  * specific language governing permissions and limitations
+ *  * under the License.
+ *
+ */
+package org.apache.asterix.test.runtime;
+
+import java.util.Collection;
+
+import org.apache.asterix.testframework.context.TestCaseContext;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import org.junit.runners.Parameterized.Parameters;
+
+/**
+ * Runs the AQL runtime tests with less parallelism on node controllers than using all the cores.
+ */
+@RunWith(Parameterized.class)
+public class AqlExecutionLessParallelismIT {
+    protected static final String TEST_CONFIG_FILE_NAME = "asterix-build-configuration3.xml";
+
+    @BeforeClass
+    public static void setUp() throws Exception {
+        LangExecutionUtil.setUp(TEST_CONFIG_FILE_NAME);
+    }
+
+    @AfterClass
+    public static void tearDown() throws Exception {
+        LangExecutionUtil.tearDown();
+    }
+
+    @Parameters(name = "AqlExecutionLessParallelismIT {index}: {0}")
+    public static Collection<Object[]> tests() throws Exception {
+        return LangExecutionUtil.tests("only.xml", "testsuite.xml");
+    }
+
+    protected TestCaseContext tcCtx;
+
+    public AqlExecutionLessParallelismIT(TestCaseContext tcCtx) {
+        this.tcCtx = tcCtx;
+    }
+
+    @Test
+    public void test() throws Exception {
+        LangExecutionUtil.test(tcCtx);
+    }
+}

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/03e940a7/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/AqlExecutionTest.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/AqlExecutionTest.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/AqlExecutionTest.java
new file mode 100644
index 0000000..4588d87
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/AqlExecutionTest.java
@@ -0,0 +1,65 @@
+/*
+ *
+ *  * Licensed to the Apache Software Foundation (ASF) under one
+ *  * or more contributor license agreements.  See the NOTICE file
+ *  * distributed with this work for additional information
+ *  * regarding copyright ownership.  The ASF licenses this file
+ *  * to you under the Apache License, Version 2.0 (the
+ *  * "License"); you may not use this file except in compliance
+ *  * with the License.  You may obtain a copy of the License at
+ *  *
+ *  *   http://www.apache.org/licenses/LICENSE-2.0
+ *  *
+ *  * Unless required by applicable law or agreed to in writing,
+ *  * software distributed under the License is distributed on an
+ *  * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ *  * KIND, either express or implied.  See the License for the
+ *  * specific language governing permissions and limitations
+ *  * under the License.
+ *
+ */
+package org.apache.asterix.test.runtime;
+
+import java.util.Collection;
+
+import org.apache.asterix.testframework.context.TestCaseContext;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import org.junit.runners.Parameterized.Parameters;
+
+/**
+ * Runs the AQL runtime tests with the storage parallelism.
+ */
+@RunWith(Parameterized.class)
+public class AqlExecutionTest {
+    protected static final String TEST_CONFIG_FILE_NAME = "asterix-build-configuration.xml";
+
+    @BeforeClass
+    public static void setUp() throws Exception {
+        LangExecutionUtil.setUp(TEST_CONFIG_FILE_NAME);
+    }
+
+    @AfterClass
+    public static void tearDown() throws Exception {
+        LangExecutionUtil.tearDown();
+    }
+
+    @Parameters(name = "AqlExecutionTest {index}: {0}")
+    public static Collection<Object[]> tests() throws Exception {
+        return LangExecutionUtil.tests("only.xml", "testsuite.xml");
+    }
+
+    protected TestCaseContext tcCtx;
+
+    public AqlExecutionTest(TestCaseContext tcCtx) {
+        this.tcCtx = tcCtx;
+    }
+
+    @Test
+    public void test() throws Exception {
+        LangExecutionUtil.test(tcCtx);
+    }
+}

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/03e940a7/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/ClusterStateExecutionFullParallelismTest.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/ClusterStateExecutionFullParallelismTest.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/ClusterStateExecutionFullParallelismTest.java
new file mode 100644
index 0000000..2f6aa1f
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/ClusterStateExecutionFullParallelismTest.java
@@ -0,0 +1,65 @@
+/*
+ *
+ *  * Licensed to the Apache Software Foundation (ASF) under one
+ *  * or more contributor license agreements.  See the NOTICE file
+ *  * distributed with this work for additional information
+ *  * regarding copyright ownership.  The ASF licenses this file
+ *  * to you under the Apache License, Version 2.0 (the
+ *  * "License"); you may not use this file except in compliance
+ *  * with the License.  You may obtain a copy of the License at
+ *  *
+ *  *   http://www.apache.org/licenses/LICENSE-2.0
+ *  *
+ *  * Unless required by applicable law or agreed to in writing,
+ *  * software distributed under the License is distributed on an
+ *  * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ *  * KIND, either express or implied.  See the License for the
+ *  * specific language governing permissions and limitations
+ *  * under the License.
+ *
+ */
+package org.apache.asterix.test.runtime;
+
+import java.util.Collection;
+
+import org.apache.asterix.testframework.context.TestCaseContext;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import org.junit.runners.Parameterized.Parameters;
+
+/**
+ * Runs the cluster state runtime tests with with full parallelism on node controllers.
+ */
+@RunWith(Parameterized.class)
+public class ClusterStateExecutionFullParallelismTest {
+    protected static final String TEST_CONFIG_FILE_NAME = "asterix-build-configuration2.xml";
+
+    @BeforeClass
+    public static void setUp() throws Exception {
+        LangExecutionUtil.setUp(TEST_CONFIG_FILE_NAME);
+    }
+
+    @AfterClass
+    public static void tearDown() throws Exception {
+        LangExecutionUtil.tearDown();
+    }
+
+    @Parameters(name = "ClusterStateExecutionFullParallelismTest {index}: {0}")
+    public static Collection<Object[]> tests() throws Exception {
+        return LangExecutionUtil.tests("only_cluster_state.xml", "cluster_state_full.xml");
+    }
+
+    protected TestCaseContext tcCtx;
+
+    public ClusterStateExecutionFullParallelismTest(TestCaseContext tcCtx) {
+        this.tcCtx = tcCtx;
+    }
+
+    @Test
+    public void test() throws Exception {
+        LangExecutionUtil.test(tcCtx);
+    }
+}

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/03e940a7/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/ClusterStateExecutionLessParallelismTest.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/ClusterStateExecutionLessParallelismTest.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/ClusterStateExecutionLessParallelismTest.java
new file mode 100644
index 0000000..73b8cdb
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/ClusterStateExecutionLessParallelismTest.java
@@ -0,0 +1,65 @@
+/*
+ *
+ *  * Licensed to the Apache Software Foundation (ASF) under one
+ *  * or more contributor license agreements.  See the NOTICE file
+ *  * distributed with this work for additional information
+ *  * regarding copyright ownership.  The ASF licenses this file
+ *  * to you under the Apache License, Version 2.0 (the
+ *  * "License"); you may not use this file except in compliance
+ *  * with the License.  You may obtain a copy of the License at
+ *  *
+ *  *   http://www.apache.org/licenses/LICENSE-2.0
+ *  *
+ *  * Unless required by applicable law or agreed to in writing,
+ *  * software distributed under the License is distributed on an
+ *  * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ *  * KIND, either express or implied.  See the License for the
+ *  * specific language governing permissions and limitations
+ *  * under the License.
+ *
+ */
+package org.apache.asterix.test.runtime;
+
+import java.util.Collection;
+
+import org.apache.asterix.testframework.context.TestCaseContext;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import org.junit.runners.Parameterized.Parameters;
+
+/**
+ * Runs the cluster state tests with less parallelism on node controllers than using all the cores.
+ */
+@RunWith(Parameterized.class)
+public class ClusterStateExecutionLessParallelismTest {
+    protected static final String TEST_CONFIG_FILE_NAME = "asterix-build-configuration3.xml";
+
+    @BeforeClass
+    public static void setUp() throws Exception {
+        LangExecutionUtil.setUp(TEST_CONFIG_FILE_NAME);
+    }
+
+    @AfterClass
+    public static void tearDown() throws Exception {
+        LangExecutionUtil.tearDown();
+    }
+
+    @Parameters(name = "ClusterStateExecutionLessParallelismTest {index}: {0}")
+    public static Collection<Object[]> tests() throws Exception {
+        return LangExecutionUtil.tests("only_cluster_state.xml", "cluster_state_less.xml");
+    }
+
+    protected TestCaseContext tcCtx;
+
+    public ClusterStateExecutionLessParallelismTest(TestCaseContext tcCtx) {
+        this.tcCtx = tcCtx;
+    }
+
+    @Test
+    public void test() throws Exception {
+        LangExecutionUtil.test(tcCtx);
+    }
+}

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/03e940a7/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/ClusterStateExecutionTest.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/ClusterStateExecutionTest.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/ClusterStateExecutionTest.java
new file mode 100644
index 0000000..14551db
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/ClusterStateExecutionTest.java
@@ -0,0 +1,65 @@
+/*
+ *
+ *  * Licensed to the Apache Software Foundation (ASF) under one
+ *  * or more contributor license agreements.  See the NOTICE file
+ *  * distributed with this work for additional information
+ *  * regarding copyright ownership.  The ASF licenses this file
+ *  * to you under the Apache License, Version 2.0 (the
+ *  * "License"); you may not use this file except in compliance
+ *  * with the License.  You may obtain a copy of the License at
+ *  *
+ *  *   http://www.apache.org/licenses/LICENSE-2.0
+ *  *
+ *  * Unless required by applicable law or agreed to in writing,
+ *  * software distributed under the License is distributed on an
+ *  * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ *  * KIND, either express or implied.  See the License for the
+ *  * specific language governing permissions and limitations
+ *  * under the License.
+ *
+ */
+package org.apache.asterix.test.runtime;
+
+import java.util.Collection;
+
+import org.apache.asterix.testframework.context.TestCaseContext;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import org.junit.runners.Parameterized.Parameters;
+
+/**
+ * Runs the cluster state runtime tests with the storage parallelism.
+ */
+@RunWith(Parameterized.class)
+public class ClusterStateExecutionTest {
+    protected static final String TEST_CONFIG_FILE_NAME = "asterix-build-configuration.xml";
+
+    @BeforeClass
+    public static void setUp() throws Exception {
+        LangExecutionUtil.setUp(TEST_CONFIG_FILE_NAME);
+    }
+
+    @AfterClass
+    public static void tearDown() throws Exception {
+        LangExecutionUtil.tearDown();
+    }
+
+    @Parameters(name = "ClusterStateExecutionTest {index}: {0}")
+    public static Collection<Object[]> tests() throws Exception {
+        return LangExecutionUtil.tests("only_cluster_state.xml", "cluster_state.xml");
+    }
+
+    protected TestCaseContext tcCtx;
+
+    public ClusterStateExecutionTest(TestCaseContext tcCtx) {
+        this.tcCtx = tcCtx;
+    }
+
+    @Test
+    public void test() throws Exception {
+        LangExecutionUtil.test(tcCtx);
+    }
+}

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/03e940a7/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/ExecutionTest.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/ExecutionTest.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/ExecutionTest.java
deleted file mode 100644
index a733645..0000000
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/ExecutionTest.java
+++ /dev/null
@@ -1,139 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.asterix.test.runtime;
-
-import java.io.File;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.List;
-import java.util.logging.Logger;
-
-import org.apache.asterix.app.external.TestLibrarian;
-import org.apache.asterix.common.config.TransactionProperties;
-import org.apache.asterix.common.library.ILibraryManager;
-import org.apache.asterix.test.aql.TestExecutor;
-import org.apache.asterix.testframework.context.TestCaseContext;
-import org.apache.commons.lang3.StringUtils;
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.junit.runners.Parameterized;
-import org.junit.runners.Parameterized.Parameters;
-
-/**
- * Runs the runtime test cases under 'asterix-app/src/test/resources/runtimets'.
- */
-@RunWith(Parameterized.class)
-public class ExecutionTest {
-
-    protected static final Logger LOGGER = Logger.getLogger(ExecutionTest.class.getName());
-
-    protected static final String PATH_ACTUAL = "target" + File.separator + "rttest" + File.separator;
-    protected static final String PATH_BASE = StringUtils.join(new String[] { "src", "test", "resources", "runtimets" },
-            File.separator);
-
-    protected static final String TEST_CONFIG_FILE_NAME = "asterix-build-configuration.xml";
-    protected static final List<String> badTestCases = new ArrayList<>();
-    protected static TransactionProperties txnProperties;
-    protected static final TestExecutor testExecutor = new TestExecutor();
-    private static final boolean cleanupOnStart = true;
-    private static final boolean cleanupOnStop = true;
-    private static TestLibrarian librarian;
-    private static final int repeat = Integer.getInteger("test.repeat", 1);
-
-    @BeforeClass
-    public static void setUp() throws Exception {
-        try {
-            File outdir = new File(PATH_ACTUAL);
-            outdir.mkdirs();
-            // remove library directory
-            TestLibrarian.removeLibraryDir();
-            List<ILibraryManager> libraryManagers = ExecutionTestUtil.setUp(cleanupOnStart);
-            librarian = new TestLibrarian(libraryManagers);
-            testExecutor.setLibrarian(librarian);
-            if (repeat != 1) {
-                System.out.println("FYI: each test will be run " + repeat + " times.");
-            }
-        } catch (Throwable th) {
-            th.printStackTrace();
-            throw th;
-        }
-    }
-
-    @AfterClass
-    public static void tearDown() throws Exception {
-        // remove library directory
-        TestLibrarian.removeLibraryDir();
-        ExecutionTestUtil.tearDown(cleanupOnStop);
-        if (!badTestCases.isEmpty()) {
-            System.out.println("The following test cases left some data");
-            for (String testCase : badTestCases) {
-                System.out.println(testCase);
-            }
-        }
-    }
-
-    @Parameters(name = "ExecutionTest {index}: {0}")
-    public static Collection<Object[]> tests() throws Exception {
-        Collection<Object[]> testArgs = buildTestsInXml(TestCaseContext.ONLY_TESTSUITE_XML_NAME);
-        if (testArgs.size() == 0) {
-            testArgs = buildTestsInXml(TestCaseContext.DEFAULT_TESTSUITE_XML_NAME);
-        }
-        return testArgs;
-    }
-
-    protected static Collection<Object[]> buildTestsInXml(String xmlfile) throws Exception {
-        Collection<Object[]> testArgs = new ArrayList<Object[]>();
-        TestCaseContext.Builder b = new TestCaseContext.Builder();
-        for (TestCaseContext ctx : b.build(new File(PATH_BASE), xmlfile)) {
-            testArgs.add(new Object[] { ctx });
-        }
-        return testArgs;
-
-    }
-
-    protected TestCaseContext tcCtx;
-
-    public ExecutionTest(TestCaseContext tcCtx) {
-        this.tcCtx = tcCtx;
-    }
-
-    @Test
-    public void test() throws Exception {
-        int repeat = ExecutionTest.repeat * tcCtx.getRepeat();
-        try {
-            for (int i = 1; i <= repeat; i++) {
-                if (repeat > 1) {
-                    System.err.print("[" + i + "/" + repeat + "] ");
-                }
-                librarian.cleanup();
-                testExecutor.executeTest(PATH_ACTUAL, tcCtx, null, false, ExecutionTestUtil.FailedGroup);
-                try {
-                    testExecutor.cleanup(tcCtx.toString(), badTestCases);
-                } catch (Throwable th) {
-                    th.printStackTrace();
-                    throw th;
-                }
-            }
-        } finally {
-            System.err.flush();
-        }
-    }
-}

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/03e940a7/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/ExecutionTestUtil.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/ExecutionTestUtil.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/ExecutionTestUtil.java
index b4bbc36..f488244 100644
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/ExecutionTestUtil.java
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/ExecutionTestUtil.java
@@ -38,7 +38,7 @@ import org.apache.hyracks.control.nc.NodeControllerService;
 
 public class ExecutionTestUtil {
 
-    protected static final Logger LOGGER = Logger.getLogger(ExecutionTest.class.getName());
+    protected static final Logger LOGGER = Logger.getLogger(ExecutionTestUtil.class.getName());
 
     protected static final String PATH_ACTUAL = "rttest" + File.separator;
 

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/03e940a7/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/LangExecutionUtil.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/LangExecutionUtil.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/LangExecutionUtil.java
new file mode 100644
index 0000000..29534f1
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/LangExecutionUtil.java
@@ -0,0 +1,115 @@
+/*
+ *
+ *  * Licensed to the Apache Software Foundation (ASF) under one
+ *  * or more contributor license agreements.  See the NOTICE file
+ *  * distributed with this work for additional information
+ *  * regarding copyright ownership.  The ASF licenses this file
+ *  * to you under the Apache License, Version 2.0 (the
+ *  * "License"); you may not use this file except in compliance
+ *  * with the License.  You may obtain a copy of the License at
+ *  *
+ *  *   http://www.apache.org/licenses/LICENSE-2.0
+ *  *
+ *  * Unless required by applicable law or agreed to in writing,
+ *  * software distributed under the License is distributed on an
+ *  * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ *  * KIND, either express or implied.  See the License for the
+ *  * specific language governing permissions and limitations
+ *  * under the License.
+ *
+ */
+package org.apache.asterix.test.runtime;
+
+import java.io.File;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+
+import org.apache.asterix.app.external.TestLibrarian;
+import org.apache.asterix.common.library.ILibraryManager;
+import org.apache.asterix.test.aql.TestExecutor;
+import org.apache.asterix.testframework.context.TestCaseContext;
+import org.apache.commons.lang3.StringUtils;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+
+/**
+ * Utils for running SQL++ or AQL runtime tests.
+ */
+@RunWith(Parameterized.class)
+public class LangExecutionUtil {
+
+    private static final String PATH_ACTUAL = "target" + File.separator + "rttest" + File.separator;
+    private static final String PATH_BASE = StringUtils.join(new String[] { "src", "test", "resources", "runtimets" },
+            File.separator);
+
+    private static final boolean cleanupOnStart = true;
+    private static final boolean cleanupOnStop = true;
+    private static final List<String> badTestCases = new ArrayList<>();
+    private static final TestExecutor testExecutor = new TestExecutor();
+
+    private static TestLibrarian librarian;
+    private static final int repeat = Integer.getInteger("test.repeat", 1);
+
+    public static void setUp(String configFile) throws Exception {
+        File outdir = new File(PATH_ACTUAL);
+        outdir.mkdirs();
+        List<ILibraryManager> libraryManagers = ExecutionTestUtil.setUp(cleanupOnStart, configFile);
+        TestLibrarian.removeLibraryDir();
+        librarian = new TestLibrarian(libraryManagers);
+        testExecutor.setLibrarian(librarian);
+        if (repeat != 1) {
+            System.out.println("FYI: each test will be run " + repeat + " times.");
+        }
+    }
+
+    public static void tearDown() throws Exception {
+        TestLibrarian.removeLibraryDir();
+        ExecutionTestUtil.tearDown(cleanupOnStop);
+        ExecutionTestUtil.integrationUtil.removeTestStorageFiles();
+        if (!badTestCases.isEmpty()) {
+            System.out.println("The following test cases left some data");
+            for (String testCase : badTestCases) {
+                System.out.println(testCase);
+            }
+        }
+    }
+
+    public static Collection<Object[]> tests(String onlyFilePath, String suiteFilePath) throws Exception {
+        Collection<Object[]> testArgs = buildTestsInXml(onlyFilePath);
+        if (testArgs.size() == 0) {
+            testArgs = buildTestsInXml(suiteFilePath);
+        }
+        return testArgs;
+    }
+
+    protected static Collection<Object[]> buildTestsInXml(String xmlfile) throws Exception {
+        Collection<Object[]> testArgs = new ArrayList<>();
+        TestCaseContext.Builder b = new TestCaseContext.Builder();
+        for (TestCaseContext ctx : b.build(new File(PATH_BASE), xmlfile)) {
+            testArgs.add(new Object[] { ctx });
+        }
+        return testArgs;
+    }
+
+    public static void test(TestCaseContext tcCtx) throws Exception {
+        int repeat = LangExecutionUtil.repeat * tcCtx.getRepeat();
+        try {
+            for (int i = 1; i <= repeat; i++) {
+                if (repeat > 1) {
+                    System.err.print("[" + i + "/" + repeat + "] ");
+                }
+                librarian.cleanup();
+                testExecutor.executeTest(PATH_ACTUAL, tcCtx, null, false, ExecutionTestUtil.FailedGroup);
+                try {
+                    testExecutor.cleanup(tcCtx.toString(), badTestCases);
+                } catch (Throwable th) {
+                    th.printStackTrace();
+                    throw th;
+                }
+            }
+        } finally {
+            System.err.flush();
+        }
+    }
+}

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/03e940a7/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/RepeatedTest.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/RepeatedTest.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/RepeatedTest.java
index 7d39b31..b7e32ab 100644
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/RepeatedTest.java
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/RepeatedTest.java
@@ -18,6 +18,8 @@
  */
 package org.apache.asterix.test.runtime;
 
+import static org.apache.asterix.test.runtime.LangExecutionUtil.buildTestsInXml;
+
 import java.lang.annotation.Retention;
 import java.lang.annotation.RetentionPolicy;
 import java.lang.annotation.Target;
@@ -79,14 +81,13 @@ class RepeatRule implements MethodRule {
 }
 
 @RunWith(Parameterized.class)
-public class RepeatedTest extends ExecutionTest {
+public class RepeatedTest extends SqlppExecutionTest {
 
     private int count;
 
     @Parameters(name = "RepeatedTest {index}: {0}")
     public static Collection<Object[]> tests() throws Exception {
-        Collection<Object[]> testArgs = buildTestsInXml(TestCaseContext.DEFAULT_REPEATED_TESTSUITE_XML_NAME);
-        return testArgs;
+        return LangExecutionUtil.buildTestsInXml(TestCaseContext.DEFAULT_REPEATED_TESTSUITE_XML_NAME);
     }
 
     public RepeatedTest(TestCaseContext tcCtx) {
@@ -102,6 +103,6 @@ public class RepeatedTest extends ExecutionTest {
     @Repeat(times = 100)
     public void test() throws Exception {
         System.err.println("***** Test Count: " + (++count) + " ******");
-        testExecutor.executeTest(PATH_ACTUAL, tcCtx, null, false);
+        super.test();
     }
 }

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/03e940a7/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/SqlppExecutionFullParallelismIT.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/SqlppExecutionFullParallelismIT.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/SqlppExecutionFullParallelismIT.java
new file mode 100644
index 0000000..62654ca
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/SqlppExecutionFullParallelismIT.java
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.test.runtime;
+
+import java.util.Collection;
+
+import org.apache.asterix.testframework.context.TestCaseContext;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import org.junit.runners.Parameterized.Parameters;
+
+/**
+ * Runs the SQL++ runtime tests with full parallelism on node controllers.
+ */
+@RunWith(Parameterized.class)
+public class SqlppExecutionFullParallelismIT {
+    protected static final String TEST_CONFIG_FILE_NAME = "asterix-build-configuration2.xml";
+
+    @BeforeClass
+    public static void setUp() throws Exception {
+        LangExecutionUtil.setUp(TEST_CONFIG_FILE_NAME);
+    }
+
+    @AfterClass
+    public static void tearDown() throws Exception {
+        LangExecutionUtil.tearDown();
+    }
+
+    @Parameters(name = "SqlppExecutionFullParallelismIT {index}: {0}")
+    public static Collection<Object[]> tests() throws Exception {
+        return LangExecutionUtil.tests("only_sqlpp.xml", "testsuite_sqlpp.xml");
+    }
+
+    protected TestCaseContext tcCtx;
+
+    public SqlppExecutionFullParallelismIT(TestCaseContext tcCtx) {
+        this.tcCtx = tcCtx;
+    }
+
+    @Test
+    public void test() throws Exception {
+        LangExecutionUtil.test(tcCtx);
+    }
+}

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/03e940a7/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/SqlppExecutionLessParallelismIT.java
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/SqlppExecutionLessParallelismIT.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/SqlppExecutionLessParallelismIT.java
new file mode 100644
index 0000000..1b22d71
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/SqlppExecutionLessParallelismIT.java
@@ -0,0 +1,65 @@
+/*
+ *
+ *  * Licensed to the Apache Software Foundation (ASF) under one
+ *  * or more contributor license agreements.  See the NOTICE file
+ *  * distributed with this work for additional information
+ *  * regarding copyright ownership.  The ASF licenses this file
+ *  * to you under the Apache License, Version 2.0 (the
+ *  * "License"); you may not use this file except in compliance
+ *  * with the License.  You may obtain a copy of the License at
+ *  *
+ *  *   http://www.apache.org/licenses/LICENSE-2.0
+ *  *
+ *  * Unless required by applicable law or agreed to in writing,
+ *  * software distributed under the License is distributed on an
+ *  * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ *  * KIND, either express or implied.  See the License for the
+ *  * specific language governing permissions and limitations
+ *  * under the License.
+ *
+ */
+package org.apache.asterix.test.runtime;
+
+import java.util.Collection;
+
+import org.apache.asterix.testframework.context.TestCaseContext;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import org.junit.runners.Parameterized.Parameters;
+
+/**
+ * Runs the SQL++ runtime tests with less parallelism on node controllers than using all the cores.
+ */
+@RunWith(Parameterized.class)
+public class SqlppExecutionLessParallelismIT {
+    protected static final String TEST_CONFIG_FILE_NAME = "asterix-build-configuration3.xml";
+
+    @BeforeClass
+    public static void setUp() throws Exception {
+        LangExecutionUtil.setUp(TEST_CONFIG_FILE_NAME);
+    }
+
+    @AfterClass
+    public static void tearDown() throws Exception {
+        LangExecutionUtil.tearDown();
+    }
+
+    @Parameters(name = "SqlppExecutionLessParallelismIT {index}: {0}")
+    public static Collection<Object[]> tests() throws Exception {
+        return LangExecutionUtil.tests("only_sqlpp.xml", "testsuite_sqlpp.xml");
+    }
+
+    protected TestCaseContext tcCtx;
+
+    public SqlppExecutionLessParallelismIT(TestCaseContext tcCtx) {
+        this.tcCtx = tcCtx;
+    }
+
+    @Test
+    public void test() throws Exception {
+        LangExecutionUtil.test(tcCtx);
+    }
+}


Mime
View raw message