asterixdb-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ima...@apache.org
Subject [14/15] incubator-asterixdb git commit: Merge asterix-experiments to master
Date Wed, 16 Mar 2016 00:35:19 GMT
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/a70fba5c/asterix-experiments/src/main/java/org/apache/asterix/experiment/builder/AbstractExperiment3Builder.java
----------------------------------------------------------------------
diff --git a/asterix-experiments/src/main/java/org/apache/asterix/experiment/builder/AbstractExperiment3Builder.java b/asterix-experiments/src/main/java/org/apache/asterix/experiment/builder/AbstractExperiment3Builder.java
new file mode 100644
index 0000000..f5fb4a4
--- /dev/null
+++ b/asterix-experiments/src/main/java/org/apache/asterix/experiment/builder/AbstractExperiment3Builder.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.asterix.experiment.builder;
+
+import org.apache.asterix.experiment.action.base.SequentialActionList;
+import org.apache.asterix.experiment.action.derived.RunAQLFileAction;
+import org.apache.asterix.experiment.client.LSMExperimentConstants;
+import org.apache.asterix.experiment.client.LSMExperimentSetRunner.LSMExperimentSetRunnerConfig;
+
+public abstract class AbstractExperiment3Builder extends AbstractLSMBaseExperimentBuilder {
+
+    public AbstractExperiment3Builder(String name, LSMExperimentSetRunnerConfig config, String clusterConfigFileName,
+            String ingestFileName, String dgenFileName) {
+        super(name, config, clusterConfigFileName, ingestFileName, dgenFileName, "count.aql");
+    }
+
+    @Override
+    protected void doBuildDDL(SequentialActionList seq) {
+        seq.add(new RunAQLFileAction(httpClient, restHost, restPort, localExperimentRoot.resolve(
+                LSMExperimentConstants.AQL_DIR).resolve("3.aql")));
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/a70fba5c/asterix-experiments/src/main/java/org/apache/asterix/experiment/builder/AbstractExperiment4Builder.java
----------------------------------------------------------------------
diff --git a/asterix-experiments/src/main/java/org/apache/asterix/experiment/builder/AbstractExperiment4Builder.java b/asterix-experiments/src/main/java/org/apache/asterix/experiment/builder/AbstractExperiment4Builder.java
new file mode 100644
index 0000000..e18975c
--- /dev/null
+++ b/asterix-experiments/src/main/java/org/apache/asterix/experiment/builder/AbstractExperiment4Builder.java
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.asterix.experiment.builder;
+
+import org.apache.asterix.experiment.client.LSMExperimentSetRunner.LSMExperimentSetRunnerConfig;
+
+public abstract class AbstractExperiment4Builder extends AbstractLSMBaseExperimentBuilder {
+
+    public AbstractExperiment4Builder(String name, LSMExperimentSetRunnerConfig config) {
+        super(name, config, "8node.xml", "base_8_ingest.aql", "8.dgen", "count.aql");
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/a70fba5c/asterix-experiments/src/main/java/org/apache/asterix/experiment/builder/AbstractExperiment6Builder.java
----------------------------------------------------------------------
diff --git a/asterix-experiments/src/main/java/org/apache/asterix/experiment/builder/AbstractExperiment6Builder.java b/asterix-experiments/src/main/java/org/apache/asterix/experiment/builder/AbstractExperiment6Builder.java
new file mode 100644
index 0000000..a34e089
--- /dev/null
+++ b/asterix-experiments/src/main/java/org/apache/asterix/experiment/builder/AbstractExperiment6Builder.java
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.asterix.experiment.builder;
+
+import org.apache.asterix.experiment.client.LSMExperimentSetRunner.LSMExperimentSetRunnerConfig;
+
+public abstract class AbstractExperiment6Builder extends AbstractLSMBaseExperimentBuilder {
+
+    public AbstractExperiment6Builder(String name, LSMExperimentSetRunnerConfig config) {
+        super(name, config, "8node.xml", "base_8_ingest.aql", "8.dgen", "count.aql");
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/a70fba5c/asterix-experiments/src/main/java/org/apache/asterix/experiment/builder/AbstractExperiment7Builder.java
----------------------------------------------------------------------
diff --git a/asterix-experiments/src/main/java/org/apache/asterix/experiment/builder/AbstractExperiment7Builder.java b/asterix-experiments/src/main/java/org/apache/asterix/experiment/builder/AbstractExperiment7Builder.java
new file mode 100644
index 0000000..d955167
--- /dev/null
+++ b/asterix-experiments/src/main/java/org/apache/asterix/experiment/builder/AbstractExperiment7Builder.java
@@ -0,0 +1,189 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.asterix.experiment.builder;
+
+import java.nio.ByteBuffer;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.List;
+import java.util.Map;
+import java.util.Random;
+
+import org.apache.commons.lang3.StringUtils;
+
+import org.apache.asterix.experiment.action.base.AbstractAction;
+import org.apache.asterix.experiment.action.base.IAction;
+import org.apache.asterix.experiment.action.base.ParallelActionSet;
+import org.apache.asterix.experiment.action.base.SequentialActionList;
+import org.apache.asterix.experiment.action.derived.AbstractRemoteExecutableAction;
+import org.apache.asterix.experiment.action.derived.RunAQLFileAction;
+import org.apache.asterix.experiment.action.derived.RunAQLStringAction;
+import org.apache.asterix.experiment.action.derived.RunRESTIOWaitAction;
+import org.apache.asterix.experiment.action.derived.SleepAction;
+import org.apache.asterix.experiment.action.derived.TimedAction;
+import org.apache.asterix.experiment.client.LSMExperimentConstants;
+import org.apache.asterix.experiment.client.LSMExperimentSetRunner.LSMExperimentSetRunnerConfig;
+import org.apache.asterix.experiment.client.OrchestratorServer7;
+import org.apache.asterix.experiment.client.OrchestratorServer7.IProtocolActionBuilder;
+
+public abstract class AbstractExperiment7Builder extends AbstractLSMBaseExperimentBuilder {
+
+    private static final long DOMAIN_SIZE = (1L << 32);
+
+    public static final long QUERY_BEGIN_ROUND = 6;
+
+    private static int N_PARTITIONS = 16;
+
+    private final int nIntervals;
+
+    private final String orchHost;
+
+    private final int orchPort;
+
+    protected final long dataInterval;
+
+    protected final int nQueryRuns;
+
+    protected final Random randGen;
+
+    public AbstractExperiment7Builder(String name, LSMExperimentSetRunnerConfig config, String clusterConfigFileName,
+            String ingestFileName, String dgenFileName) {
+        super(name, config, clusterConfigFileName, ingestFileName, dgenFileName, null);
+        nIntervals = config.getNIntervals();
+        orchHost = config.getOrchestratorHost();
+        orchPort = config.getOrchestratorPort();
+        dataInterval = config.getDataInterval();
+        this.nQueryRuns = config.getNQueryRuns();
+        this.randGen = new Random();
+    }
+
+    @Override
+    protected void doBuildDataGen(SequentialActionList seq, Map<String, List<String>> dgenPairs) throws Exception {
+        int nDgens = 0;
+        for (List<String> v : dgenPairs.values()) {
+            nDgens += v.size();
+        }
+        final OrchestratorServer7 oServer = new OrchestratorServer7(orchPort, nDgens, nIntervals,
+                new ProtocolActionBuilder(), this.lsAction);
+
+        seq.add(new AbstractAction() {
+
+            @Override
+            protected void doPerform() throws Exception {
+                oServer.start();
+            }
+        });
+
+        ParallelActionSet dgenActions = new ParallelActionSet();
+        int partition = 0;
+
+        // run dgen
+        for (String dgenHost : dgenPairs.keySet()) {
+            final List<String> rcvrs = dgenPairs.get(dgenHost);
+            final int p = partition;
+            dgenActions.add(new AbstractRemoteExecutableAction(dgenHost, username, sshKeyLocation) {
+
+                @Override
+                protected String getCommand() {
+                    String ipPortPairs = StringUtils.join(rcvrs.iterator(), " ");
+                    String binary = "JAVA_HOME=" + javaHomePath + " "
+                            + localExperimentRoot.resolve("bin").resolve("datagenrunner").toString();
+                    return StringUtils.join(new String[] { binary, "-si", "" + locationSampleInterval, "-of",
+                            openStreetMapFilePath, "-p", "" + p, "-di", "" + dataInterval, "-ni", "" + nIntervals,
+                            "-oh", orchHost, "-op", "" + orchPort, ipPortPairs }, " ");
+                }
+            });
+            partition += rcvrs.size();
+        }
+        seq.add(dgenActions);
+
+        // wait until all dgen / queries are done
+        seq.add(new AbstractAction() {
+
+            @Override
+            protected void doPerform() throws Exception {
+                oServer.awaitFinished();
+            }
+        });
+    }
+
+    public class ProtocolActionBuilder implements IProtocolActionBuilder {
+
+        private final String rangeQueryTemplate;
+
+        public ProtocolActionBuilder() {
+            this.rangeQueryTemplate = getRangeQueryTemplate();
+        }
+
+        private String getRangeQueryTemplate() {
+            try {
+                Path aqlTemplateFilePath = localExperimentRoot.resolve(LSMExperimentConstants.AQL_DIR).resolve(
+                        "8_q2.aql");
+                return StandardCharsets.UTF_8.decode(ByteBuffer.wrap(Files.readAllBytes(aqlTemplateFilePath)))
+                        .toString();
+            } catch (Exception e) {
+                throw new RuntimeException(e);
+            }
+        }
+
+        @Override
+        public IAction buildQueryAction(long cardinality, boolean finalRound) throws Exception {
+            SequentialActionList protoAction = new SequentialActionList();
+            IAction rangeQueryAction = new TimedAction(new RunAQLStringAction(httpClient, restHost, restPort,
+                    getRangeAQL(cardinality, finalRound)));
+            protoAction.add(rangeQueryAction);
+            return protoAction;
+        }
+
+        private String getRangeAQL(long cardinaliry, boolean finalRound) throws Exception {
+            long round = QUERY_BEGIN_ROUND;
+            if (finalRound) {
+                ++round;
+            }
+            long numKeys = ((round * dataInterval) / 1000) * N_PARTITIONS;
+            long rangeSize = (long) ((cardinaliry / (double) numKeys) * DOMAIN_SIZE);
+            int lowKey = randGen.nextInt();
+            long maxLowKey = Integer.MAX_VALUE - rangeSize;
+            if (lowKey > maxLowKey) {
+                lowKey = (int) maxLowKey;
+            }
+            int highKey = (int) (lowKey + rangeSize);
+            return rangeQueryTemplate.replaceAll("\\$LKEY\\$", Long.toString(lowKey)).replaceAll("\\$HKEY\\$",
+                    Long.toString(highKey));
+        }
+
+        @Override
+        public IAction buildIOWaitAction() throws Exception {
+            SequentialActionList ioAction = new SequentialActionList();
+            ioAction.add(new SleepAction(10000));
+            ioAction.add(new RunRESTIOWaitAction(httpClient, restHost, restPort));
+            ioAction.add(new SleepAction(10000));
+            return ioAction;
+        }
+
+        @Override
+        public IAction buildCompactAction() throws Exception {
+            return (new RunAQLFileAction(httpClient, restHost, restPort, localExperimentRoot.resolve(
+                    LSMExperimentConstants.AQL_DIR).resolve("8_compact.aql")));
+        }
+    }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/a70fba5c/asterix-experiments/src/main/java/org/apache/asterix/experiment/builder/AbstractExperiment8Builder.java
----------------------------------------------------------------------
diff --git a/asterix-experiments/src/main/java/org/apache/asterix/experiment/builder/AbstractExperiment8Builder.java b/asterix-experiments/src/main/java/org/apache/asterix/experiment/builder/AbstractExperiment8Builder.java
new file mode 100644
index 0000000..eaafae2
--- /dev/null
+++ b/asterix-experiments/src/main/java/org/apache/asterix/experiment/builder/AbstractExperiment8Builder.java
@@ -0,0 +1,189 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.asterix.experiment.builder;
+
+import java.nio.ByteBuffer;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.List;
+import java.util.Map;
+import java.util.Random;
+
+import org.apache.commons.lang3.StringUtils;
+
+import org.apache.asterix.experiment.action.base.AbstractAction;
+import org.apache.asterix.experiment.action.base.ParallelActionSet;
+import org.apache.asterix.experiment.action.base.SequentialActionList;
+import org.apache.asterix.experiment.action.derived.AbstractRemoteExecutableAction;
+import org.apache.asterix.experiment.action.derived.RunAQLStringAction;
+import org.apache.asterix.experiment.action.derived.RunRESTIOWaitAction;
+import org.apache.asterix.experiment.action.derived.SleepAction;
+import org.apache.asterix.experiment.action.derived.TimedAction;
+import org.apache.asterix.experiment.client.LSMExperimentConstants;
+import org.apache.asterix.experiment.client.LSMExperimentSetRunner.LSMExperimentSetRunnerConfig;
+import org.apache.asterix.experiment.client.OrchestratorServer;
+
+public abstract class AbstractExperiment8Builder extends AbstractLSMBaseExperimentBuilder {
+
+    private static final long DOMAIN_SIZE = (1L << 32);
+
+    private static final long EXPECTED_RANGE_CARDINALITY = 50000;
+
+    private static int N_PARTITIONS = 16;
+
+    private final int nIntervals;
+
+    private final String orchHost;
+
+    private final int orchPort;
+
+    protected final long dataInterval;
+
+    protected final int nQueryRuns;
+
+    protected final Random randGen;
+
+    private final String pointQueryTemplate;
+    private final String rangeQueryTemplate;
+
+    public AbstractExperiment8Builder(String name, LSMExperimentSetRunnerConfig config, String clusterConfigFileName,
+            String ingestFileName, String dgenFileName) {
+        super(name, config, clusterConfigFileName, ingestFileName, dgenFileName, null);
+        nIntervals = config.getNIntervals();
+        orchHost = config.getOrchestratorHost();
+        orchPort = config.getOrchestratorPort();
+        dataInterval = config.getDataInterval();
+        this.nQueryRuns = config.getNQueryRuns();
+        this.randGen = new Random();
+        this.pointQueryTemplate = getPointQueryTemplate();
+        this.rangeQueryTemplate = getRangeQueryTemplate();
+    }
+
+    protected void doBuildProtocolAction(SequentialActionList seq, int round) throws Exception {
+        for (int i = 0; i < nQueryRuns; ++i) {
+            //            String aql = getPointLookUpAQL(round);
+            //            seq.add(new TimedAction(new RunAQLStringAction(httpClient, restHost, restPort, aql)));
+            String aql = getRangeAQL(round);
+            seq.add(new TimedAction(new RunAQLStringAction(httpClient, restHost, restPort, aql)));
+        }
+    }
+
+    @Override
+    protected void doBuildDataGen(SequentialActionList seq, Map<String, List<String>> dgenPairs) throws Exception {
+        //start datagen
+        SequentialActionList[] protocolActions = new SequentialActionList[nIntervals];
+        for (int i = 0; i < nIntervals; i++) {
+            protocolActions[i] = new SequentialActionList();
+            protocolActions[i].add(new SleepAction(10000));
+            protocolActions[i].add(new RunRESTIOWaitAction(httpClient, restHost, restPort));
+            protocolActions[i].add(new SleepAction(10000));
+            doBuildProtocolAction(protocolActions[i], i);
+        }
+
+        int nDgens = 0;
+        for (List<String> v : dgenPairs.values()) {
+            nDgens += v.size();
+        }
+        final OrchestratorServer oServer = new OrchestratorServer(orchPort, nDgens, nIntervals, protocolActions);
+
+        seq.add(new AbstractAction() {
+
+            @Override
+            protected void doPerform() throws Exception {
+                oServer.start();
+            }
+        });
+
+        ParallelActionSet dgenActions = new ParallelActionSet();
+        int partition = 0;
+
+        // run dgen
+        for (String dgenHost : dgenPairs.keySet()) {
+            final List<String> rcvrs = dgenPairs.get(dgenHost);
+            final int p = partition;
+            dgenActions.add(new AbstractRemoteExecutableAction(dgenHost, username, sshKeyLocation) {
+
+                @Override
+                protected String getCommand() {
+                    String ipPortPairs = StringUtils.join(rcvrs.iterator(), " ");
+                    String binary = "JAVA_HOME=" + javaHomePath + " "
+                            + localExperimentRoot.resolve("bin").resolve("datagenrunner").toString();
+                    return StringUtils.join(new String[] { binary, "-si", "" + locationSampleInterval, "-of",
+                            openStreetMapFilePath, "-p", "" + p, "-di", "" + dataInterval, "-ni", "" + nIntervals,
+                            "-oh", orchHost, "-op", "" + orchPort, ipPortPairs }, " ");
+                }
+            });
+            partition += rcvrs.size();
+        }
+        seq.add(dgenActions);
+
+        // wait until all dgen / queries are done
+        seq.add(new AbstractAction() {
+
+            @Override
+            protected void doPerform() throws Exception {
+                oServer.awaitFinished();
+            }
+        });
+    }
+
+    private String getRangeQueryTemplate() {
+        try {
+            Path aqlTemplateFilePath = localExperimentRoot.resolve(LSMExperimentConstants.AQL_DIR).resolve("8_q2.aql");
+            return StandardCharsets.UTF_8.decode(ByteBuffer.wrap(Files.readAllBytes(aqlTemplateFilePath))).toString();
+        } catch (Exception e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    private String getPointQueryTemplate() {
+        try {
+            Path aqlTemplateFilePath = localExperimentRoot.resolve(LSMExperimentConstants.AQL_DIR).resolve("8_q1.aql");
+            return StandardCharsets.UTF_8.decode(ByteBuffer.wrap(Files.readAllBytes(aqlTemplateFilePath))).toString();
+        } catch (Exception e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    protected String getPointLookUpAQL(int round) throws Exception {
+        ByteBuffer bb = ByteBuffer.allocate(8);
+        bb.put((byte) 0);
+        bb.put((byte) randGen.nextInt(N_PARTITIONS));
+        bb.putShort((short) 0);
+        bb.putInt(randGen.nextInt((int) (((1 + round) * dataInterval) / 1000)));
+        bb.flip();
+        long key = bb.getLong();
+        return pointQueryTemplate.replaceAll("\\$KEY\\$", Long.toString(key));
+    }
+
+    protected String getRangeAQL(int round) throws Exception {
+        long numKeys = (((1 + round) * dataInterval) / 1000) * N_PARTITIONS;
+        long rangeSize = (long) ((EXPECTED_RANGE_CARDINALITY / (double) numKeys) * DOMAIN_SIZE);
+        int lowKey = randGen.nextInt();
+        long maxLowKey = Integer.MAX_VALUE - rangeSize;
+        if (lowKey > maxLowKey) {
+            lowKey = (int) maxLowKey;
+        }
+        int highKey = (int) (lowKey + rangeSize);
+        return rangeQueryTemplate.replaceAll("\\$LKEY\\$", Long.toString(lowKey)).replaceAll("\\$HKEY\\$",
+                Long.toString(highKey));
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/a70fba5c/asterix-experiments/src/main/java/org/apache/asterix/experiment/builder/AbstractExperiment9Builder.java
----------------------------------------------------------------------
diff --git a/asterix-experiments/src/main/java/org/apache/asterix/experiment/builder/AbstractExperiment9Builder.java b/asterix-experiments/src/main/java/org/apache/asterix/experiment/builder/AbstractExperiment9Builder.java
new file mode 100644
index 0000000..6d36f1d
--- /dev/null
+++ b/asterix-experiments/src/main/java/org/apache/asterix/experiment/builder/AbstractExperiment9Builder.java
@@ -0,0 +1,196 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.asterix.experiment.builder;
+
+import java.nio.ByteBuffer;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.List;
+import java.util.Map;
+import java.util.Random;
+
+import org.apache.commons.lang3.StringUtils;
+
+import org.apache.asterix.experiment.action.base.AbstractAction;
+import org.apache.asterix.experiment.action.base.IAction;
+import org.apache.asterix.experiment.action.base.ParallelActionSet;
+import org.apache.asterix.experiment.action.base.SequentialActionList;
+import org.apache.asterix.experiment.action.derived.AbstractRemoteExecutableAction;
+import org.apache.asterix.experiment.action.derived.RunAQLStringAction;
+import org.apache.asterix.experiment.action.derived.TimedAction;
+import org.apache.asterix.experiment.client.LSMExperimentConstants;
+import org.apache.asterix.experiment.client.LSMExperimentSetRunner.LSMExperimentSetRunnerConfig;
+import org.apache.asterix.experiment.client.OrchestratorServer9;
+import org.apache.asterix.experiment.client.OrchestratorServer9.IProtocolActionBuilder;
+
+public abstract class AbstractExperiment9Builder extends AbstractLSMBaseExperimentBuilder {
+
+    private static final long DOMAIN_SIZE = (1L << 32);
+
+    private static final long EXPECTED_RANGE_CARDINALITY = 1000;
+
+    private static int N_PARTITIONS = 16;
+
+    private final int nIntervals;
+
+    private final String orchHost;
+
+    private final int orchPort;
+
+    protected final long dataInterval;
+
+    protected final int nQueryRuns;
+
+    protected final Random randGen;
+
+    public AbstractExperiment9Builder(String name, LSMExperimentSetRunnerConfig config, String clusterConfigFileName,
+            String ingestFileName, String dgenFileName) {
+        super(name, config, clusterConfigFileName, ingestFileName, dgenFileName, null);
+        nIntervals = config.getNIntervals();
+        orchHost = config.getOrchestratorHost();
+        orchPort = config.getOrchestratorPort();
+        dataInterval = config.getDataInterval();
+        this.nQueryRuns = config.getNQueryRuns();
+        this.randGen = new Random();
+    }
+
+    @Override
+    protected void doBuildDataGen(SequentialActionList seq, Map<String, List<String>> dgenPairs) throws Exception {
+        int nDgens = 0;
+        for (List<String> v : dgenPairs.values()) {
+            nDgens += v.size();
+        }
+        final OrchestratorServer9 oServer = new OrchestratorServer9(orchPort, nDgens, nIntervals,
+                new ProtocolActionBuilder());
+
+        seq.add(new AbstractAction() {
+
+            @Override
+            protected void doPerform() throws Exception {
+                oServer.start();
+            }
+        });
+
+        ParallelActionSet dgenActions = new ParallelActionSet();
+        int partition = 0;
+
+        // run dgen
+        for (String dgenHost : dgenPairs.keySet()) {
+            final List<String> rcvrs = dgenPairs.get(dgenHost);
+            final int p = partition;
+            dgenActions.add(new AbstractRemoteExecutableAction(dgenHost, username, sshKeyLocation) {
+
+                @Override
+                protected String getCommand() {
+                    String ipPortPairs = StringUtils.join(rcvrs.iterator(), " ");
+                    String binary = "JAVA_HOME=" + javaHomePath + " "
+                            + localExperimentRoot.resolve("bin").resolve("datagenrunner").toString();
+                    return StringUtils.join(new String[] { binary, "-si", "" + locationSampleInterval, "-of",
+                            openStreetMapFilePath, "-p", "" + p, "-di", "" + dataInterval, "-ni", "" + nIntervals,
+                            "-oh", orchHost, "-op", "" + orchPort, ipPortPairs }, " ");
+                }
+            });
+            partition += rcvrs.size();
+        }
+        seq.add(dgenActions);
+
+        // wait until all dgen / queries are done
+        seq.add(new AbstractAction() {
+
+            @Override
+            protected void doPerform() throws Exception {
+                oServer.awaitFinished();
+            }
+        });
+    }
+
+    public class ProtocolActionBuilder implements IProtocolActionBuilder {
+
+        private final String pointQueryTemplate;
+
+        private final String rangeQueryTemplate;
+
+        public ProtocolActionBuilder() {
+            this.pointQueryTemplate = getPointQueryTemplate();
+            this.rangeQueryTemplate = getRangeQueryTemplate();
+        }
+
+        private String getRangeQueryTemplate() {
+            try {
+                Path aqlTemplateFilePath = localExperimentRoot.resolve(LSMExperimentConstants.AQL_DIR).resolve(
+                        "8_q2.aql");
+                return StandardCharsets.UTF_8.decode(ByteBuffer.wrap(Files.readAllBytes(aqlTemplateFilePath)))
+                        .toString();
+            } catch (Exception e) {
+                throw new RuntimeException(e);
+            }
+        }
+
+        private String getPointQueryTemplate() {
+            try {
+                Path aqlTemplateFilePath = localExperimentRoot.resolve(LSMExperimentConstants.AQL_DIR).resolve(
+                        "8_q1.aql");
+                return StandardCharsets.UTF_8.decode(ByteBuffer.wrap(Files.readAllBytes(aqlTemplateFilePath)))
+                        .toString();
+            } catch (Exception e) {
+                throw new RuntimeException(e);
+            }
+        }
+
+        @Override
+        public IAction buildAction(int round) throws Exception {
+            SequentialActionList protoAction = new SequentialActionList();
+            IAction pointQueryAction = new TimedAction(new RunAQLStringAction(httpClient, restHost, restPort,
+                    getPointLookUpAQL(round)));
+            IAction rangeQueryAction = new TimedAction(new RunAQLStringAction(httpClient, restHost, restPort,
+                    getRangeAQL(round)));
+            protoAction.add(pointQueryAction);
+            protoAction.add(rangeQueryAction);
+            return protoAction;
+        }
+
+        private String getPointLookUpAQL(int round) throws Exception {
+            ByteBuffer bb = ByteBuffer.allocate(8);
+            bb.put((byte) 0);
+            bb.put((byte) randGen.nextInt(N_PARTITIONS));
+            bb.putShort((short) 0);
+            bb.putInt(randGen.nextInt((int) (((1 + round) * dataInterval) / 1000)));
+            bb.flip();
+            long key = bb.getLong();
+            return pointQueryTemplate.replaceAll("\\$KEY\\$", Long.toString(key));
+        }
+
+        private String getRangeAQL(int round) throws Exception {
+            long numKeys = (((1 + round) * dataInterval) / 1000) * N_PARTITIONS;
+            long rangeSize = (long) ((EXPECTED_RANGE_CARDINALITY / (double) numKeys) * DOMAIN_SIZE);
+            int lowKey = randGen.nextInt();
+            long maxLowKey = Integer.MAX_VALUE - rangeSize;
+            if (lowKey > maxLowKey) {
+                lowKey = (int) maxLowKey;
+            }
+            int highKey = (int) (lowKey + rangeSize);
+            return rangeQueryTemplate.replaceAll("\\$LKEY\\$", Long.toString(lowKey)).replaceAll("\\$HKEY\\$",
+                    Long.toString(highKey));
+        }
+
+    }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/a70fba5c/asterix-experiments/src/main/java/org/apache/asterix/experiment/builder/AbstractExperimentBuilder.java
----------------------------------------------------------------------
diff --git a/asterix-experiments/src/main/java/org/apache/asterix/experiment/builder/AbstractExperimentBuilder.java b/asterix-experiments/src/main/java/org/apache/asterix/experiment/builder/AbstractExperimentBuilder.java
new file mode 100644
index 0000000..1717652
--- /dev/null
+++ b/asterix-experiments/src/main/java/org/apache/asterix/experiment/builder/AbstractExperimentBuilder.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.asterix.experiment.builder;
+
+public abstract class AbstractExperimentBuilder {
+    private final String name;
+
+    protected AbstractExperimentBuilder(String name) {
+        this.name = name;
+    }
+
+    public String getName() {
+        return name;
+    }
+
+    public final Experiment build() throws Exception {
+        Experiment e = new Experiment(name);
+        doBuild(e);
+        return e;
+    }
+
+    protected abstract void doBuild(Experiment e) throws Exception;
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/a70fba5c/asterix-experiments/src/main/java/org/apache/asterix/experiment/builder/AbstractLSMBaseExperimentBuilder.java
----------------------------------------------------------------------
diff --git a/asterix-experiments/src/main/java/org/apache/asterix/experiment/builder/AbstractLSMBaseExperimentBuilder.java b/asterix-experiments/src/main/java/org/apache/asterix/experiment/builder/AbstractLSMBaseExperimentBuilder.java
new file mode 100644
index 0000000..1a57480
--- /dev/null
+++ b/asterix-experiments/src/main/java/org/apache/asterix/experiment/builder/AbstractLSMBaseExperimentBuilder.java
@@ -0,0 +1,357 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.asterix.experiment.builder;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Scanner;
+import java.util.Set;
+
+import javax.xml.bind.JAXBContext;
+import javax.xml.bind.Unmarshaller;
+
+import org.apache.asterix.event.schema.cluster.Cluster;
+import org.apache.asterix.experiment.action.base.ParallelActionSet;
+import org.apache.asterix.experiment.action.base.SequentialActionList;
+import org.apache.asterix.experiment.action.derived.AbstractRemoteExecutableAction;
+import org.apache.asterix.experiment.action.derived.ManagixActions.CreateAsterixManagixAction;
+import org.apache.asterix.experiment.action.derived.ManagixActions.DeleteAsterixManagixAction;
+import org.apache.asterix.experiment.action.derived.ManagixActions.LogAsterixManagixAction;
+import org.apache.asterix.experiment.action.derived.ManagixActions.StopAsterixManagixAction;
+import org.apache.asterix.experiment.action.derived.RemoteAsterixDriverKill;
+import org.apache.asterix.experiment.action.derived.RunAQLFileAction;
+import org.apache.asterix.experiment.action.derived.SleepAction;
+import org.apache.asterix.experiment.client.LSMExperimentConstants;
+import org.apache.asterix.experiment.client.LSMExperimentSetRunner.LSMExperimentSetRunnerConfig;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.http.client.HttpClient;
+import org.apache.http.impl.client.DefaultHttpClient;
+
+public abstract class AbstractLSMBaseExperimentBuilder extends AbstractExperimentBuilder {
+
+    private static final String ASTERIX_INSTANCE_NAME = "a1";
+
+    private final String logDirSuffix;
+
+    protected final HttpClient httpClient;
+
+    protected final String restHost;
+
+    protected final int restPort;
+
+    private final String managixHomePath;
+
+    protected final String javaHomePath;
+
+    protected final Path localExperimentRoot;
+
+    protected final String username;
+
+    protected final String sshKeyLocation;
+
+    private final int duration;
+
+    private final String clusterConfigFileName;
+
+    private final String ingestFileName;
+
+    protected final String dgenFileName;
+
+    private final String countFileName;
+
+    private final String statFile;
+
+    protected final SequentialActionList lsAction;
+
+    protected final String openStreetMapFilePath;
+    protected final int locationSampleInterval;
+
+    protected final int recordCountPerBatchDuringIngestionOnly;
+    protected final int recordCountPerBatchDuringQuery;
+    protected final long dataGenSleepTimeDuringIngestionOnly;
+    protected final long dataGenSleepTimeDuringQuery;
+
+    public AbstractLSMBaseExperimentBuilder(String name, LSMExperimentSetRunnerConfig config,
+            String clusterConfigFileName, String ingestFileName, String dgenFileName, String countFileName) {
+        super(name);
+        this.logDirSuffix = config.getLogDirSuffix();
+        this.httpClient = new DefaultHttpClient();
+        this.restHost = config.getRESTHost();
+        this.restPort = config.getRESTPort();
+        this.managixHomePath = config.getManagixHome();
+        this.javaHomePath = config.getJavaHome();
+        this.localExperimentRoot = Paths.get(config.getLocalExperimentRoot());
+        this.username = config.getUsername();
+        this.sshKeyLocation = config.getSSHKeyLocation();
+        this.duration = config.getDuration();
+        this.clusterConfigFileName = clusterConfigFileName;
+        this.ingestFileName = ingestFileName;
+        this.dgenFileName = dgenFileName;
+        this.countFileName = countFileName;
+        this.statFile = config.getStatFile();
+        this.lsAction = new SequentialActionList();
+        this.openStreetMapFilePath = config.getOpenStreetMapFilePath();
+        this.locationSampleInterval = config.getLocationSampleInterval();
+        recordCountPerBatchDuringIngestionOnly = config.getRecordCountPerBatchDuringIngestionOnly();
+        recordCountPerBatchDuringQuery = config.getRecordCountPerBatchDuringQuery();
+        dataGenSleepTimeDuringIngestionOnly = config.getDataGenSleepTimeDuringIngestionOnly();
+        dataGenSleepTimeDuringQuery = config.getDataGenSleepTimeDuringQuery();
+    }
+
+    protected abstract void doBuildDDL(SequentialActionList seq);
+
+    protected void doPost(SequentialActionList seq) {
+    }
+
+    protected void doBuildDataGen(SequentialActionList seq, final Map<String, List<String>> dgenPairs) throws Exception {
+
+        //start datagen
+        ParallelActionSet dgenActions = new ParallelActionSet();
+        int partition = 0;
+        for (String dgenHost : dgenPairs.keySet()) {
+            final List<String> rcvrs = dgenPairs.get(dgenHost);
+            final int p = partition;
+            dgenActions.add(new AbstractRemoteExecutableAction(dgenHost, username, sshKeyLocation) {
+
+                @Override
+                protected String getCommand() {
+                    String ipPortPairs = StringUtils.join(rcvrs.iterator(), " ");
+                    String binary = "JAVA_HOME=" + javaHomePath + " "
+                            + localExperimentRoot.resolve("bin").resolve("datagenrunner").toString();
+                    if (openStreetMapFilePath == null) {
+                        return StringUtils.join(new String[] { binary, "-rcbi",
+                                "" + recordCountPerBatchDuringIngestionOnly, "-rcbq",
+                                "" + recordCountPerBatchDuringQuery, "-dsti", "" + dataGenSleepTimeDuringIngestionOnly,
+                                "-dstq", "" + dataGenSleepTimeDuringQuery, "-si", "" + locationSampleInterval, "-p",
+                                "" + p, "-d", "" + duration, ipPortPairs }, " ");
+                    } else {
+                        return StringUtils.join(new String[] { binary, "-rcbi",
+                                "" + recordCountPerBatchDuringIngestionOnly, "-rcbq",
+                                "" + recordCountPerBatchDuringQuery, "-dsti", "" + dataGenSleepTimeDuringIngestionOnly,
+                                "-dstq", "" + dataGenSleepTimeDuringQuery, "-si", "" + locationSampleInterval, "-of",
+                                openStreetMapFilePath, "-p", "" + p, "-d", "" + duration, ipPortPairs }, " ");
+                    }
+                }
+            });
+            partition += rcvrs.size();
+        }
+        seq.add(dgenActions);
+    }
+
+    @Override
+    protected void doBuild(Experiment e) throws Exception {
+        SequentialActionList execs = new SequentialActionList();
+
+        String clusterConfigPath = localExperimentRoot.resolve(LSMExperimentConstants.CONFIG_DIR)
+                .resolve(clusterConfigFileName).toString();
+        String asterixConfigPath = localExperimentRoot.resolve(LSMExperimentConstants.CONFIG_DIR)
+                .resolve(LSMExperimentConstants.ASTERIX_CONFIGURATION).toString();
+
+        //create instance
+        execs.add(new StopAsterixManagixAction(managixHomePath, ASTERIX_INSTANCE_NAME));
+        execs.add(new DeleteAsterixManagixAction(managixHomePath, ASTERIX_INSTANCE_NAME));
+        execs.add(new SleepAction(30000));
+        execs.add(new CreateAsterixManagixAction(managixHomePath, ASTERIX_INSTANCE_NAME, clusterConfigPath,
+                asterixConfigPath));
+
+        //run ddl statements
+        execs.add(new SleepAction(15000));
+        // TODO: implement retry handler
+        execs.add(new RunAQLFileAction(httpClient, restHost, restPort, localExperimentRoot.resolve(
+                LSMExperimentConstants.AQL_DIR).resolve(LSMExperimentConstants.BASE_TYPES)));
+        doBuildDDL(execs);
+        execs.add(new RunAQLFileAction(httpClient, restHost, restPort, localExperimentRoot
+                .resolve(LSMExperimentConstants.AQL_DIR).resolve(LSMExperimentConstants.BASE_DIR)
+                .resolve(ingestFileName)));
+
+        Map<String, List<String>> dgenPairs = readDatagenPairs(localExperimentRoot.resolve(
+                LSMExperimentConstants.DGEN_DIR).resolve(dgenFileName));
+        final Set<String> ncHosts = new HashSet<>();
+        for (List<String> ncHostList : dgenPairs.values()) {
+            for (String ncHost : ncHostList) {
+                ncHosts.add(ncHost.split(":")[0]);
+            }
+        }
+
+        if (statFile != null) {
+            ParallelActionSet ioCountActions = new ParallelActionSet();
+            for (String ncHost : ncHosts) {
+                ioCountActions.add(new AbstractRemoteExecutableAction(ncHost, username, sshKeyLocation) {
+
+                    @Override
+                    protected String getCommand() {
+                        String cmd = "screen -d -m sh -c \"sar -b -u 1 > " + statFile + "\"";
+                        return cmd;
+                    }
+                });
+            }
+            execs.add(ioCountActions);
+        }
+
+        SequentialActionList postLSAction = new SequentialActionList();
+        File file = new File(clusterConfigPath);
+        JAXBContext ctx = JAXBContext.newInstance(Cluster.class);
+        Unmarshaller unmarshaller = ctx.createUnmarshaller();
+        final Cluster cluster = (Cluster) unmarshaller.unmarshal(file);
+        String[] storageRoots = cluster.getIodevices().split(",");
+        for (String ncHost : ncHosts) {
+            for (final String sRoot : storageRoots) {
+                lsAction.add(new AbstractRemoteExecutableAction(ncHost, username, sshKeyLocation) {
+                    @Override
+                    protected String getCommand() {
+                        return "ls -Rl " + sRoot;
+                    }
+                });
+                postLSAction.add(new AbstractRemoteExecutableAction(ncHost, username, sshKeyLocation) {
+                    @Override
+                    protected String getCommand() {
+                        return "ls -Rl " + sRoot;
+                    }
+                });
+
+            }
+        }
+
+        // main exp
+        doBuildDataGen(execs, dgenPairs);
+
+        //        if (statFile != null) {
+        //            ParallelActionSet ioCountKillActions = new ParallelActionSet();
+        //            for (String ncHost : ncHosts) {
+        //                ioCountKillActions.add(new AbstractRemoteExecutableAction(ncHost, username, sshKeyLocation) {
+        //
+        //                    @Override
+        //                    protected String getCommand() {
+        //                        String cmd = "screen -X -S `screen -list | grep Detached | awk '{print $1}'` quit";
+        //                        return cmd;
+        //                    }
+        //                });
+        //            }
+        //            execs.add(ioCountKillActions);
+        //        }
+
+        execs.add(new SleepAction(10000));
+        if (countFileName != null) {
+            execs.add(new RunAQLFileAction(httpClient, restHost, restPort, localExperimentRoot.resolve(
+                    LSMExperimentConstants.AQL_DIR).resolve(countFileName)));
+        }
+
+        execs.add(postLSAction);
+        doPost(execs);
+        ParallelActionSet killCmds = new ParallelActionSet();
+        for (String ncHost : ncHosts) {
+            killCmds.add(new RemoteAsterixDriverKill(ncHost, username, sshKeyLocation));
+        }
+        //killCmds.add(new RemoteAsterixDriverKill(restHost, username, sshKeyLocation));
+        execs.add(killCmds);
+        execs.add(new StopAsterixManagixAction(managixHomePath, ASTERIX_INSTANCE_NAME));
+        if (statFile != null) {
+            ParallelActionSet collectIOActions = new ParallelActionSet();
+            for (String ncHost : ncHosts) {
+                collectIOActions.add(new AbstractRemoteExecutableAction(ncHost, username, sshKeyLocation) {
+
+                    @Override
+                    protected String getCommand() {
+                        String cmd = "cp " + statFile + " " + cluster.getLogDir();
+                        return cmd;
+                    }
+                });
+            }
+            execs.add(collectIOActions);
+        }
+
+        //collect profile information
+        //        if (ExperimentProfiler.PROFILE_MODE) {
+        //            if (!SpatialIndexProfiler.PROFILE_HOME_DIR.contentEquals(cluster.getLogDir())) {
+        //                ParallelActionSet collectProfileInfo = new ParallelActionSet();
+        //                for (String ncHost : ncHosts) {
+        //                    collectProfileInfo.add(new AbstractRemoteExecutableAction(ncHost, username, sshKeyLocation) {
+        //                        @Override
+        //                        protected String getCommand() {
+        //                            String cmd = "mv " + SpatialIndexProfiler.PROFILE_HOME_DIR + "*.txt " + cluster.getLogDir();
+        //                            return cmd;
+        //                        }
+        //                    });
+        //                }
+        //                execs.add(collectProfileInfo);
+        //            }
+        //        }
+
+        execs.add(new LogAsterixManagixAction(managixHomePath, ASTERIX_INSTANCE_NAME, localExperimentRoot
+                .resolve(LSMExperimentConstants.LOG_DIR + "-" + logDirSuffix).resolve(getName()).toString()));
+
+        if (getName().contains("SpatialIndexExperiment2") || getName().contains("SpatialIndexExperiment5")) {
+            //get query result file
+            SequentialActionList getQueryResultFileActions = new SequentialActionList();
+            final String queryResultFilePath = openStreetMapFilePath.substring(0,
+                    openStreetMapFilePath.lastIndexOf(File.separator))
+                    + File.separator + "QueryGenResult-*.txt";
+            for (final String qgenHost : dgenPairs.keySet()) {
+                getQueryResultFileActions.add(new AbstractRemoteExecutableAction(restHost, username, sshKeyLocation) {
+
+                    @Override
+                    protected String getCommand() {
+                        String cmd = "scp "
+                                + username
+                                + "@"
+                                + qgenHost
+                                + ":"
+                                + queryResultFilePath
+                                + " "
+                                + localExperimentRoot.resolve(LSMExperimentConstants.LOG_DIR + "-" + logDirSuffix)
+                                        .resolve(getName()).toString();
+                        return cmd;
+                    }
+                });
+            }
+            execs.add(getQueryResultFileActions);
+        }
+
+        e.addBody(execs);
+    }
+
+    protected Map<String, List<String>> readDatagenPairs(Path p) throws IOException {
+        Map<String, List<String>> dgenPairs = new HashMap<>();
+        Scanner s = new Scanner(p, StandardCharsets.UTF_8.name());
+        try {
+            while (s.hasNextLine()) {
+                String line = s.nextLine();
+                String[] pair = line.split("\\s+");
+                List<String> vals = dgenPairs.get(pair[0]);
+                if (vals == null) {
+                    vals = new ArrayList<>();
+                    dgenPairs.put(pair[0], vals);
+                }
+                vals.add(pair[1]);
+            }
+        } finally {
+            s.close();
+        }
+        return dgenPairs;
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/a70fba5c/asterix-experiments/src/main/java/org/apache/asterix/experiment/builder/AbstractLocalExperimentBuilder.java
----------------------------------------------------------------------
diff --git a/asterix-experiments/src/main/java/org/apache/asterix/experiment/builder/AbstractLocalExperimentBuilder.java b/asterix-experiments/src/main/java/org/apache/asterix/experiment/builder/AbstractLocalExperimentBuilder.java
new file mode 100644
index 0000000..eac4ac2
--- /dev/null
+++ b/asterix-experiments/src/main/java/org/apache/asterix/experiment/builder/AbstractLocalExperimentBuilder.java
@@ -0,0 +1,119 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.asterix.experiment.builder;
+
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+import org.apache.asterix.experiment.action.base.SequentialActionList;
+import org.apache.asterix.hyracks.bootstrap.CCApplicationEntryPoint;
+import org.apache.asterix.hyracks.bootstrap.NCApplicationEntryPoint;
+import org.apache.hyracks.control.cc.ClusterControllerService;
+import org.apache.hyracks.control.common.controllers.CCConfig;
+import org.apache.hyracks.control.common.controllers.NCConfig;
+import org.apache.hyracks.control.nc.NodeControllerService;
+
+public abstract class AbstractLocalExperimentBuilder extends AbstractExperimentBuilder {
+
+    private final int nNodeControllers;
+
+    protected AbstractLocalExperimentBuilder(String name, int nNodeControllers) {
+        super(name);
+        this.nNodeControllers = nNodeControllers;
+    }
+
+    protected abstract void addPre(SequentialActionList pre);
+
+    protected abstract void addPost(SequentialActionList post);
+
+//    @Override
+//    protected void prePost(SequentialExecutableSet pre, SequentialExecutableSet post) {
+//        int ccClientPort = 1098;
+//        int ccClusterPort = 1099;
+//        CCConfig ccConfig = new CCConfig();
+//        ccConfig.clusterNetIpAddress = "127.0.0.1";
+//        ccConfig.clientNetIpAddress = "127.0.0.1";
+//        ccConfig.clientNetPort = ccClientPort;
+//        ccConfig.clusterNetPort = ccClusterPort;
+//        ccConfig.defaultMaxJobAttempts = 0;
+//        ccConfig.resultTTL = 30000;
+//        ccConfig.resultSweepThreshold = 1000;
+//        ccConfig.appCCMainClass = CCApplicationEntryPoint.class.getName();
+//        final ClusterControllerService cc;
+//        try {
+//            cc = new ClusterControllerService(ccConfig);
+//        } catch (Exception e) {
+//            throw new IllegalArgumentException(e);
+//        }
+//
+//        final List<NodeControllerService> ncs = new ArrayList<>();
+//        for (int i = 0; i < nNodeControllers; ++i) {
+//            NCConfig ncConfig = new NCConfig();
+//            ncConfig.ccHost = "localhost";
+//            ncConfig.ccPort = ccClusterPort;
+//            ncConfig.clusterNetIPAddress = "127.0.0.1";
+//            ncConfig.dataIPAddress = "127.0.0.1";
+//            ncConfig.datasetIPAddress = "127.0.0.1";
+//            ncConfig.nodeId = "nc" + String.valueOf((i + 1));
+//            ncConfig.resultTTL = 30000;
+//            ncConfig.resultSweepThreshold = 1000;
+//            Path p0 = Paths.get(System.getProperty("java.io.tmpdir"), ncConfig.nodeId, "iodevice0");
+//            Path p1 = Paths.get(System.getProperty("java.io.tmpdir"), ncConfig.nodeId, "iodevice1");
+//            ncConfig.ioDevices = p0.toString() + "," + p1.toString();
+//            ncConfig.appNCMainClass = NCApplicationEntryPoint.class.getName();
+//            NodeControllerService nc;
+//            try {
+//                nc = new NodeControllerService(ncConfig);
+//            } catch (Exception e) {
+//                throw new IllegalArgumentException(e);
+//            }
+//            ncs.add(nc);
+//        }
+//
+//        pre.add(new AbstractExecutable() {
+//
+//            @Override
+//            protected void doExecute() throws Exception {
+//                cc.start();
+//                for (NodeControllerService nc : ncs) {
+//                    nc.start();
+//                }
+//            }
+//        });
+//
+//        post.add(new AbstractExecutable() {
+//
+//            @Override
+//            protected void doExecute() throws Exception {
+//                Collections.reverse(ncs);
+//                for (NodeControllerService nc : ncs) {
+//                    nc.stop();
+//                }
+//                cc.stop();
+//                System.exit(1);
+//            }
+//        });
+//        addPre(pre);
+//        addPost(post);
+//    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/a70fba5c/asterix-experiments/src/main/java/org/apache/asterix/experiment/builder/AbstractSpatialIndexExperiment2Builder.java
----------------------------------------------------------------------
diff --git a/asterix-experiments/src/main/java/org/apache/asterix/experiment/builder/AbstractSpatialIndexExperiment2Builder.java b/asterix-experiments/src/main/java/org/apache/asterix/experiment/builder/AbstractSpatialIndexExperiment2Builder.java
new file mode 100644
index 0000000..848ab7c
--- /dev/null
+++ b/asterix-experiments/src/main/java/org/apache/asterix/experiment/builder/AbstractSpatialIndexExperiment2Builder.java
@@ -0,0 +1,173 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.asterix.experiment.builder;
+
+import java.util.List;
+import java.util.Map;
+import java.util.Random;
+
+import org.apache.asterix.experiment.action.base.AbstractAction;
+import org.apache.asterix.experiment.action.base.ParallelActionSet;
+import org.apache.asterix.experiment.action.base.SequentialActionList;
+import org.apache.asterix.experiment.action.derived.AbstractRemoteExecutableAction;
+import org.apache.asterix.experiment.client.LSMExperimentSetRunner.LSMExperimentSetRunnerConfig;
+import org.apache.asterix.experiment.client.SpatialIndexExperiment2OrchestratorServer;
+import org.apache.commons.lang3.StringUtils;
+
+public abstract class AbstractSpatialIndexExperiment2Builder extends AbstractLSMBaseExperimentBuilder {
+
+    private static final long DOMAIN_SIZE = (1L << 32);
+
+    public static final long QUERY_BEGIN_ROUND = 6;
+
+    private static int N_PARTITIONS = 16;
+
+    private final int nIntervals;
+
+    private final String dataGenOrchHost;
+
+    private final int dataGenOrchPort;
+
+    private final String queryGenOrchHost;
+
+    private final int queryGenOrchPort;
+
+    private final long queryGenDuration;
+
+    protected final long dataInterval;
+
+    protected final int nQueryRuns;
+
+    protected final Random randGen;
+
+    protected final boolean isIndexOnlyPlan;
+
+    public AbstractSpatialIndexExperiment2Builder(String name, LSMExperimentSetRunnerConfig config,
+            String clusterConfigFileName, String ingestFileName, String dgenFileName, boolean isIndexOnlyPlan) {
+        super(name, config, clusterConfigFileName, ingestFileName, dgenFileName, null);
+        nIntervals = config.getNIntervals();
+        dataGenOrchHost = config.getOrchestratorHost();
+        dataGenOrchPort = config.getOrchestratorPort();
+        dataInterval = config.getDataInterval();
+        queryGenOrchHost = config.getQueryOrchestratorHost();
+        queryGenOrchPort = config.getQueryOrchestratorPort();
+        queryGenDuration = config.getQueryDuration();
+        this.nQueryRuns = config.getNQueryRuns();
+        this.randGen = new Random();
+        this.isIndexOnlyPlan = isIndexOnlyPlan;
+    }
+
+    @Override
+    protected void doBuildDataGen(SequentialActionList seq, Map<String, List<String>> dgenPairs) throws Exception {
+        int nDgens = 0;
+        for (List<String> v : dgenPairs.values()) {
+            nDgens += v.size();
+        }
+        final SpatialIndexExperiment2OrchestratorServer oServer = new SpatialIndexExperiment2OrchestratorServer(
+                dataGenOrchPort, nDgens, nIntervals, queryGenOrchPort, nDgens /*for now, query gen uses the same node as data gen*/);
+
+        seq.add(new AbstractAction() {
+
+            @Override
+            protected void doPerform() throws Exception {
+                oServer.start();
+            }
+        });
+
+        //prepare data gen runner and query gen runner
+        //note that dgenPairs.keySet() are used as query gen runners' ip address.
+        ParallelActionSet dataAndQueryGenActions = new ParallelActionSet();
+        int partition = 0;
+        for (String dgenHost : dgenPairs.keySet()) {
+            final List<String> rcvrs = dgenPairs.get(dgenHost);
+            final int p = partition;
+            //prepare data gen
+            dataAndQueryGenActions.add(new AbstractRemoteExecutableAction(dgenHost, username, sshKeyLocation) {
+
+                @Override
+                protected String getCommand() {
+                    String ipPortPairs = StringUtils.join(rcvrs.iterator(), " ");
+                    String binary = "JAVA_HOME=" + javaHomePath + " "
+                            + localExperimentRoot.resolve("bin").resolve("datagenrunner").toString();
+                    if (openStreetMapFilePath == null) {
+                        return StringUtils.join(new String[] { binary, "-rcbi",
+                                "" + recordCountPerBatchDuringIngestionOnly, "-rcbq",
+                                "" + recordCountPerBatchDuringQuery, "-dsti", "" + dataGenSleepTimeDuringIngestionOnly,
+                                "-dstq", "" + dataGenSleepTimeDuringQuery, "-si", "" + locationSampleInterval, "-p",
+                                "" + p, "-di", "" + dataInterval, "-ni", "" + nIntervals, "-qd", "" + queryGenDuration,
+                                "-oh", dataGenOrchHost, "-op", "" + dataGenOrchPort, ipPortPairs }, " ");
+                    } else {
+                        return StringUtils.join(new String[] { binary, "-rcbi",
+                                "" + recordCountPerBatchDuringIngestionOnly, "-rcbq",
+                                "" + recordCountPerBatchDuringQuery, "-dsti", "" + dataGenSleepTimeDuringIngestionOnly,
+                                "-dstq", "" + dataGenSleepTimeDuringQuery, "-si", "" + locationSampleInterval, "-of",
+                                openStreetMapFilePath, "-p", "" + p, "-di", "" + dataInterval, "-ni", "" + nIntervals,
+                                "-qd", "" + queryGenDuration, "-oh", dataGenOrchHost, "-op", "" + dataGenOrchPort,
+                                ipPortPairs }, " ");
+                    }
+                }
+            });
+
+            //prepare query gen
+            dataAndQueryGenActions.add(new AbstractRemoteExecutableAction(dgenHost, username, sshKeyLocation) {
+
+                @Override
+                protected String getCommand() {
+                    String ipPortPairs = StringUtils.join(rcvrs.iterator(), " ");
+                    String binary = "JAVA_HOME=" + javaHomePath + " "
+                            + localExperimentRoot.resolve("bin").resolve("querygenrunner").toString();
+                    if (openStreetMapFilePath == null) {
+                        if (isIndexOnlyPlan) {
+                            return StringUtils.join(new String[] { binary, "-iop", "-p", "" + p, "-qd",
+                                    "" + queryGenDuration, "-qoh", "" + queryGenOrchHost, "-qop",
+                                    "" + queryGenOrchPort, "-rh", restHost, "-rp", "" + restPort }, " ");
+                        } else {
+                            return StringUtils.join(new String[] { binary, "-p", "" + p, "-qd", "" + queryGenDuration,
+                                    "-qoh", "" + queryGenOrchHost, "-qop", "" + queryGenOrchPort, "-rh", restHost,
+                                    "-rp", "" + restPort }, " ");
+                        }
+                    } else {
+                        if (isIndexOnlyPlan) {
+                            return StringUtils.join(new String[] { binary, "-iop", "-of", openStreetMapFilePath, "-p",
+                                    "" + p, "-qd", "" + queryGenDuration, "-qoh", "" + queryGenOrchHost, "-qop",
+                                    "" + queryGenOrchPort, "-rh", restHost, "-rp", "" + restPort }, " ");
+                        } else {
+                            return StringUtils.join(new String[] { binary, "-of", openStreetMapFilePath, "-p", "" + p,
+                                    "-qd", "" + queryGenDuration, "-qoh", "" + queryGenOrchHost, "-qop",
+                                    "" + queryGenOrchPort, "-rh", restHost, "-rp", "" + restPort }, " ");
+                        }
+                    }
+                }
+            });
+
+            partition += rcvrs.size();
+        }
+        seq.add(dataAndQueryGenActions);
+
+        // wait until all dgen / queries are done
+        seq.add(new AbstractAction() {
+
+            @Override
+            protected void doPerform() throws Exception {
+                oServer.awaitFinished();
+            }
+        });
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/a70fba5c/asterix-experiments/src/main/java/org/apache/asterix/experiment/builder/AbstractSpatialIndexExperiment3PIdxLoadBuilder.java
----------------------------------------------------------------------
diff --git a/asterix-experiments/src/main/java/org/apache/asterix/experiment/builder/AbstractSpatialIndexExperiment3PIdxLoadBuilder.java b/asterix-experiments/src/main/java/org/apache/asterix/experiment/builder/AbstractSpatialIndexExperiment3PIdxLoadBuilder.java
new file mode 100644
index 0000000..6de1c73
--- /dev/null
+++ b/asterix-experiments/src/main/java/org/apache/asterix/experiment/builder/AbstractSpatialIndexExperiment3PIdxLoadBuilder.java
@@ -0,0 +1,293 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.asterix.experiment.builder;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Scanner;
+import java.util.Set;
+
+import javax.xml.bind.JAXBContext;
+import javax.xml.bind.Unmarshaller;
+
+import org.apache.asterix.event.schema.cluster.Cluster;
+import org.apache.asterix.experiment.action.base.ParallelActionSet;
+import org.apache.asterix.experiment.action.base.SequentialActionList;
+import org.apache.asterix.experiment.action.derived.AbstractRemoteExecutableAction;
+import org.apache.asterix.experiment.action.derived.ManagixActions.CreateAsterixManagixAction;
+import org.apache.asterix.experiment.action.derived.ManagixActions.DeleteAsterixManagixAction;
+import org.apache.asterix.experiment.action.derived.ManagixActions.LogAsterixManagixAction;
+import org.apache.asterix.experiment.action.derived.ManagixActions.StopAsterixManagixAction;
+import org.apache.asterix.experiment.action.derived.RemoteAsterixDriverKill;
+import org.apache.asterix.experiment.action.derived.RunAQLFileAction;
+import org.apache.asterix.experiment.action.derived.SleepAction;
+import org.apache.asterix.experiment.action.derived.TimedAction;
+import org.apache.asterix.experiment.client.LSMExperimentConstants;
+import org.apache.asterix.experiment.client.LSMExperimentSetRunner.LSMExperimentSetRunnerConfig;
+import org.apache.http.client.HttpClient;
+import org.apache.http.impl.client.DefaultHttpClient;
+
+/**
+ * This class is used to create experiments for spatial index static data evaluation, that is, no ingestion is involved.
+ * Also, there is no orchestration server involved in this experiment builder.
+ */
+public abstract class AbstractSpatialIndexExperiment3PIdxLoadBuilder extends AbstractExperimentBuilder {
+
+    private static final String ASTERIX_INSTANCE_NAME = "a1";
+
+    private final String logDirSuffix;
+
+    protected final HttpClient httpClient;
+
+    protected final String restHost;
+
+    protected final int restPort;
+
+    private final String managixHomePath;
+
+    protected final String javaHomePath;
+
+    protected final Path localExperimentRoot;
+
+    protected final String username;
+
+    protected final String sshKeyLocation;
+
+    private final int duration;
+
+    private final String clusterConfigFileName;
+
+    private final String ingestFileName;
+
+    protected final String dgenFileName;
+
+    private final String countFileName;
+
+    private final String statFile;
+
+    protected final SequentialActionList lsAction;
+
+    protected final String openStreetMapFilePath;
+
+    protected final int locationSampleInterval;
+
+    protected final String loadAQLFilePath;
+
+    public AbstractSpatialIndexExperiment3PIdxLoadBuilder(String name, LSMExperimentSetRunnerConfig config,
+            String clusterConfigFileName, String ingestFileName, String dgenFileName, String countFileName,
+            String loadAQLFileName) {
+        super(name);
+        this.logDirSuffix = config.getLogDirSuffix();
+        this.httpClient = new DefaultHttpClient();
+        this.restHost = config.getRESTHost();
+        this.restPort = config.getRESTPort();
+        this.managixHomePath = config.getManagixHome();
+        this.javaHomePath = config.getJavaHome();
+        this.localExperimentRoot = Paths.get(config.getLocalExperimentRoot());
+        this.username = config.getUsername();
+        this.sshKeyLocation = config.getSSHKeyLocation();
+        this.duration = config.getDuration();
+        this.clusterConfigFileName = clusterConfigFileName;
+        this.ingestFileName = ingestFileName;
+        this.dgenFileName = dgenFileName;
+        this.countFileName = countFileName;
+        this.statFile = config.getStatFile();
+        this.lsAction = new SequentialActionList();
+        this.openStreetMapFilePath = config.getOpenStreetMapFilePath();
+        this.locationSampleInterval = config.getLocationSampleInterval();
+        this.loadAQLFilePath = loadAQLFileName;
+    }
+
+    protected abstract void doBuildDDL(SequentialActionList seq);
+
+    protected void doPost(SequentialActionList seq) {
+    }
+
+    protected void doBuildDataGen(SequentialActionList seq, final Map<String, List<String>> dgenPairs) throws Exception {
+    }
+
+    @Override
+    protected void doBuild(Experiment e) throws Exception {
+        SequentialActionList execs = new SequentialActionList();
+
+        String clusterConfigPath = localExperimentRoot.resolve(LSMExperimentConstants.CONFIG_DIR)
+                .resolve(clusterConfigFileName).toString();
+        String asterixConfigPath = localExperimentRoot.resolve(LSMExperimentConstants.CONFIG_DIR)
+                .resolve(LSMExperimentConstants.ASTERIX_CONFIGURATION).toString();
+
+        //stop/delete/create instance
+        execs.add(new StopAsterixManagixAction(managixHomePath, ASTERIX_INSTANCE_NAME));
+        execs.add(new DeleteAsterixManagixAction(managixHomePath, ASTERIX_INSTANCE_NAME));
+        execs.add(new SleepAction(30000));
+        execs.add(new CreateAsterixManagixAction(managixHomePath, ASTERIX_INSTANCE_NAME, clusterConfigPath,
+                asterixConfigPath));
+
+        //ddl statements
+        execs.add(new SleepAction(15000));
+        // TODO: implement retry handler
+        execs.add(new RunAQLFileAction(httpClient, restHost, restPort, localExperimentRoot.resolve(
+                LSMExperimentConstants.AQL_DIR).resolve(LSMExperimentConstants.BASE_TYPES)));
+        doBuildDDL(execs);
+
+        //prepare io state action in NC node(s)
+        Map<String, List<String>> dgenPairs = readDatagenPairs(localExperimentRoot.resolve(
+                LSMExperimentConstants.DGEN_DIR).resolve(dgenFileName));
+        final Set<String> ncHosts = new HashSet<>();
+        for (List<String> ncHostList : dgenPairs.values()) {
+            for (String ncHost : ncHostList) {
+                ncHosts.add(ncHost.split(":")[0]);
+            }
+        }
+        if (statFile != null) {
+            ParallelActionSet ioCountActions = new ParallelActionSet();
+            for (String ncHost : ncHosts) {
+                ioCountActions.add(new AbstractRemoteExecutableAction(ncHost, username, sshKeyLocation) {
+
+                    @Override
+                    protected String getCommand() {
+                        String cmd = "screen -d -m sh -c \"sar -b -u 1 > " + statFile + "\"";
+                        return cmd;
+                    }
+                });
+            }
+            execs.add(ioCountActions);
+        }
+
+        //prepare post ls action
+        SequentialActionList postLSAction = new SequentialActionList();
+        File file = new File(clusterConfigPath);
+        JAXBContext ctx = JAXBContext.newInstance(Cluster.class);
+        Unmarshaller unmarshaller = ctx.createUnmarshaller();
+        final Cluster cluster = (Cluster) unmarshaller.unmarshal(file);
+        String[] storageRoots = cluster.getIodevices().split(",");
+        for (String ncHost : ncHosts) {
+            for (final String sRoot : storageRoots) {
+                lsAction.add(new AbstractRemoteExecutableAction(ncHost, username, sshKeyLocation) {
+                    @Override
+                    protected String getCommand() {
+                        return "ls -Rl " + sRoot;
+                    }
+                });
+                postLSAction.add(new AbstractRemoteExecutableAction(ncHost, username, sshKeyLocation) {
+                    @Override
+                    protected String getCommand() {
+                        return "ls -Rl " + sRoot;
+                    }
+                });
+
+            }
+        }
+
+        //---------- main experiment body begins -----------
+
+        //load data into pidx 
+        execs.add(new TimedAction(new RunAQLFileAction(httpClient, restHost, restPort, localExperimentRoot.resolve(
+                LSMExperimentConstants.AQL_DIR).resolve(loadAQLFilePath))));
+
+        //---------- main experiment body ends -----------
+
+        //kill io state action
+        if (statFile != null) {
+            ParallelActionSet ioCountKillActions = new ParallelActionSet();
+            for (String ncHost : ncHosts) {
+                ioCountKillActions.add(new AbstractRemoteExecutableAction(ncHost, username, sshKeyLocation) {
+
+                    @Override
+                    protected String getCommand() {
+                        String cmd = "screen -X -S `screen -list | grep Detached | awk '{print $1}'` quit";
+                        return cmd;
+                    }
+                });
+            }
+            execs.add(ioCountKillActions);
+        }
+
+        //total record count
+        execs.add(new SleepAction(10000));
+        if (countFileName != null) {
+            execs.add(new RunAQLFileAction(httpClient, restHost, restPort, localExperimentRoot.resolve(
+                    LSMExperimentConstants.AQL_DIR).resolve(countFileName)));
+        }
+
+        //add ls action
+        execs.add(postLSAction);
+
+        //kill asterix cc and nc
+        ParallelActionSet killCmds = new ParallelActionSet();
+        for (String ncHost : ncHosts) {
+            killCmds.add(new RemoteAsterixDriverKill(ncHost, username, sshKeyLocation));
+        }
+        killCmds.add(new RemoteAsterixDriverKill(restHost, username, sshKeyLocation));
+        execs.add(killCmds);
+
+        //stop asterix instance
+        execs.add(new StopAsterixManagixAction(managixHomePath, ASTERIX_INSTANCE_NAME));
+
+        //prepare to collect io state by putting the state file into asterix log dir
+        if (statFile != null) {
+            ParallelActionSet collectIOActions = new ParallelActionSet();
+            for (String ncHost : ncHosts) {
+                collectIOActions.add(new AbstractRemoteExecutableAction(ncHost, username, sshKeyLocation) {
+
+                    @Override
+                    protected String getCommand() {
+                        String cmd = "cp " + statFile + " " + cluster.getLogDir();
+                        return cmd;
+                    }
+                });
+            }
+            execs.add(collectIOActions);
+        }
+
+        //collect cc and nc logs
+        execs.add(new LogAsterixManagixAction(managixHomePath, ASTERIX_INSTANCE_NAME, localExperimentRoot
+                .resolve(LSMExperimentConstants.LOG_DIR + "-" + logDirSuffix).resolve(getName()).toString()));
+
+        e.addBody(execs);
+    }
+
+    protected Map<String, List<String>> readDatagenPairs(Path p) throws IOException {
+        Map<String, List<String>> dgenPairs = new HashMap<>();
+        Scanner s = new Scanner(p, StandardCharsets.UTF_8.name());
+        try {
+            while (s.hasNextLine()) {
+                String line = s.nextLine();
+                String[] pair = line.split("\\s+");
+                List<String> vals = dgenPairs.get(pair[0]);
+                if (vals == null) {
+                    vals = new ArrayList<>();
+                    dgenPairs.put(pair[0], vals);
+                }
+                vals.add(pair[1]);
+            }
+        } finally {
+            s.close();
+        }
+        return dgenPairs;
+    }
+}


Mime
View raw message