accumulo-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From mwa...@apache.org
Subject [1/2] accumulo-testing git commit: ACCUMULO-4510 Created 'accumulo-testing' command
Date Mon, 09 Jan 2017 22:12:05 GMT
Repository: accumulo-testing
Updated Branches:
  refs/heads/master ac5b271ca -> 4f8220640


ACCUMULO-4510 Created 'accumulo-testing' command

* Runs randomwalk locally or builds shaded jar that is submitted to YARN
* Added documentation for random walk


Project: http://git-wip-us.apache.org/repos/asf/accumulo-testing/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo-testing/commit/efaa7377
Tree: http://git-wip-us.apache.org/repos/asf/accumulo-testing/tree/efaa7377
Diff: http://git-wip-us.apache.org/repos/asf/accumulo-testing/diff/efaa7377

Branch: refs/heads/master
Commit: efaa7377c88a4716a7abf8f30e025ee7d68d6499
Parents: ac5b271
Author: Mike Walch <mwalch@apache.org>
Authored: Wed Dec 28 14:09:53 2016 -0500
Committer: Mike Walch <mwalch@apache.org>
Committed: Mon Jan 9 16:21:03 2017 -0500

----------------------------------------------------------------------
 .travis.yml                                     |  22 +++
 README.md                                       |  53 ++++++
 bin/accumulo-testing                            | 101 +++++++++++
 conf/.gitignore                                 |   2 +
 conf/accumulo-testing-env.sh.example            |  25 +++
 conf/accumulo-testing.properties.example        |  44 +++++
 conf/log4j.properties.example                   |  27 +++
 core/pom.xml                                    |   4 -
 .../apache/accumulo/testing/core/TestProps.java |  53 ++++++
 .../testing/core/randomwalk/Environment.java    |  49 +-----
 .../testing/core/randomwalk/Framework.java      |  22 +--
 .../testing/core/randomwalk/Module.java         |  16 +-
 .../core/randomwalk/multitable/CopyTable.java   |   3 +-
 .../randomwalk/sequential/MapRedVerify.java     |   3 +-
 .../main/resources/randomwalk/modules/Bulk.xml  |   2 +-
 .../resources/randomwalk/modules/Concurrent.xml |   2 +-
 .../randomwalk/modules/Conditional.xml          |   3 +-
 .../main/resources/randomwalk/modules/Image.xml |   2 +-
 .../resources/randomwalk/modules/MultiTable.xml |   2 +-
 .../resources/randomwalk/modules/Security.xml   |   2 +-
 .../resources/randomwalk/modules/Sequential.xml |   2 +-
 .../main/resources/randomwalk/modules/Shard.xml |   2 +-
 .../resources/randomwalk/modules/unit/Basic.xml |   2 +-
 .../randomwalk/modules/unit/Simple.xml          |   2 +-
 pom.xml                                         |  47 ++++++
 yarn/.gitignore                                 |   2 +
 yarn/pom.xml                                    | 110 ++++++++++++
 .../testing/yarn/YarnAccumuloTestRunner.java    | 169 +++++++++++++++++++
 yarn/src/main/resources/logback.xml             |  31 ++++
 29 files changed, 724 insertions(+), 80 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/efaa7377/.travis.yml
----------------------------------------------------------------------
diff --git a/.travis.yml b/.travis.yml
new file mode 100644
index 0000000..3688f4a
--- /dev/null
+++ b/.travis.yml
@@ -0,0 +1,22 @@
+#  Licensed to the Apache Software Foundation (ASF) under one or more
+#  contributor license agreements.  See the NOTICE file distributed with
+#  this work for additional information regarding copyright ownership.
+#  The ASF licenses this file to You under the Apache License, Version 2.0
+#  (the "License"); you may not use this file except in compliance with
+#  the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+language: java
+cache:
+  directories:
+    - $HOME/.m2
+jdk:
+  - oraclejdk8
+install: true
+script: mvn clean verify

http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/efaa7377/README.md
----------------------------------------------------------------------
diff --git a/README.md b/README.md
index 9b70451..49b38a9 100644
--- a/README.md
+++ b/README.md
@@ -1 +1,54 @@
 # Apache Accumulo Testing Suite
+
+The Apache Accumulo testing suite contains applications that test and verify the
+correctness of Accumulo.
+
+## Installation
+
+In order to run the Apache Accumulo testing suite, you will need Java 8 and Maven installed
+on your machine as well as an Accumulo instance to use for testing.
+
+1. First clone this repository.
+
+        git clone git@github.com:apache/accumulo-testing.git
+        cd accumulo-testing
+
+2. All configuation files for the test suite are in `conf/`. Only the `accumulo-testing.properties`
+   configuration file needs to be created and edited as all other configuration files are optional.
+   In `accumulo-testing.properites`, review the properties with `test.common.*` prefix as these are
+   used by all tests.
+
+        cd conf/
+        cp accumulo-testing.properties.example accumulo-testing.properties
+        vim accumulo-testing.properties
+
+3. Tests are run using the `accumulo-testing` command which is located in the `bin/`
+   directory. Run this command without any arguments to view its usage and see available tests.
+
+        ./bin/accumulo-testing
+
+## Random walk test
+
+The random walk test generates client behavior on an Apache Accumulo instance by randomly walking a
+graph of client operations. 
+
+Before running random walk, review the `test.common.*` and `test.randomwalk.*` properties in
+`accumulo-testing.properties` file. A test module must also be specified. See the [modules][modules]
+directory for a list of available ones.
+
+The command below will start a single random walker in a local process using the [Image.xml][image]
+module.
+
+        ./bin/accumulo-testing rw-local Image.xml
+
+If you would like to run multiple, distributed random walkers, run the command below to start random
+walkers in 5 containers in YARN using the Image.xml module.
+
+        ./bin/accumulo-testing rw-yarn 5 Image.xml
+
+This command will create an application in YARN and exit when test is completed. While its
+running, you can view logs for each random walker using the YARN resource manager. The YARN
+application can be killed at any time using `ctrl-c` or via the resource manager.
+
+[modules]: core/src/main/resources/randomwalk/modules
+[image]: core/src/main/resources/randomwalk/modules/Image.xml

http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/efaa7377/bin/accumulo-testing
----------------------------------------------------------------------
diff --git a/bin/accumulo-testing b/bin/accumulo-testing
new file mode 100755
index 0000000..e5c1063
--- /dev/null
+++ b/bin/accumulo-testing
@@ -0,0 +1,101 @@
+#! /usr/bin/env bash
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+bin_dir=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )
+at_home=$( cd "$( dirname "$bin_dir" )" && pwd )
+at_version=2.0.0-SNAPSHOT
+
+function print_usage() {
+  cat <<EOF
+
+Usage: accumulo-testing <command> (<argument>)
+
+Possible commands:
+  rw-local <module>         Runs randomwalk <module> in local java process
+  rw-yarn <num> <module>    Runs randomwalk <module> in <num> containers on YARN
+EOF
+}
+
+if [ -f "$at_home/conf/accumulo-testing-env.sh" ]; then
+  . "$at_home"/conf/accumulo-testing-env.sh
+else
+  . "$at_home"/conf/accumulo-testing-env.sh.example
+fi
+
+if [ -z "$ACCUMULO_VERSION" ]; then
+  echo "ERROR: ACCUMULO_VERSION must be set conf/accumulo-testing-env.sh"
+  exit 1
+fi
+if [ -z "$HADOOP_VERSION" ]; then
+  echo "ERROR: HADOOP_VERSION must be set conf/accumulo-testing-env.sh"
+  exit 1
+fi
+
+at_props="$at_home/conf/accumulo-testing.properties"
+if [ ! -f "$at_props" ]; then
+  echo "Please create and edit accumulo-testing.properties in $at_home/conf"
+  exit 1
+fi
+
+log4j_config="$at_home/conf/log4j.properties"
+if [ ! -f "$log4j_config" ]; then
+  log4j_config="$at_home/conf/log4j.properties.example"
+  if [ ! -f "$log4j_config" ]; then
+    echo "Could not find logj4.properties or log4j.properties.example in $at_home/conf"
+    exit 1
+  fi
+fi
+
+function build_shade_jar() {
+  export at_shaded_jar="$at_home/core/target/accumulo-testing-core-$at_version-shaded.jar"
+  if [ ! -f "$at_shaded_jar" ]; then
+    echo "Building $at_shaded_jar"
+    cd "$at_home" || exit 1
+    mvn clean package -P create-shade-jar -D skipTests -D accumulo.version="$ACCUMULO_VERSION" -D hadoop.version="$HADOOP_VERSION"
+  fi
+}
+
+randomwalk_main="org.apache.accumulo.testing.core.randomwalk.Framework"
+case "$1" in
+rw-local)
+  if [ -z "$2" ]; then
+    echo "ERROR: <module> needs to be set"
+    print_usage
+    exit 1
+  fi
+  build_shade_jar
+  java -Dlog4j.configuration="file:$log4j_config" -cp "$at_shaded_jar" "$randomwalk_main" "$at_props" "$2"
+  ;;
+rw-yarn)
+  if [ -z "$2" ]; then
+    echo "ERROR: <num> needs to be set"
+    print_usage
+    exit 1
+  fi
+  if [ -z "$3" ]; then
+    echo "ERROR: <module> needs to be set"
+    print_usage
+    exit 1
+  fi
+  build_shade_jar
+  mvn compile -P yarn-test-runner -D hadoop.version="$HADOOP_VERSION" -D exec.args="-t AccumuloRandomWalkTest -j $at_shaded_jar -m $randomwalk_main -n $2 -p $at_props -l $log4j_config -a ./accumulo-testing.properties $3"
+  ;;
+*)
+  echo "Unknown command: $1"
+  print_usage
+  exit 1
+esac

http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/efaa7377/conf/.gitignore
----------------------------------------------------------------------
diff --git a/conf/.gitignore b/conf/.gitignore
new file mode 100644
index 0000000..336f891
--- /dev/null
+++ b/conf/.gitignore
@@ -0,0 +1,2 @@
+/accumulo-testing.properties
+/accumulo-testing-env.sh

http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/efaa7377/conf/accumulo-testing-env.sh.example
----------------------------------------------------------------------
diff --git a/conf/accumulo-testing-env.sh.example b/conf/accumulo-testing-env.sh.example
new file mode 100644
index 0000000..24f8571
--- /dev/null
+++ b/conf/accumulo-testing-env.sh.example
@@ -0,0 +1,25 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Hadoop
+# ======
+test -z "$HADOOP_PREFIX" && export HADOOP_PREFIX=/path/to/hadoop
+test -z "$HADOOP_CONF_DIR" && export HADOOP_CONF_DIR=/path/to/hadoop/etc/hadoop
+
+# Accumulo
+# ========
+# Set the Accumulo version that should be included in the shaded jar
+export ACCUMULO_VERSION=`accumulo version`
+export HADOOP_VERSION=`hadoop version | head -n1 | awk '{print $2}'`

http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/efaa7377/conf/accumulo-testing.properties.example
----------------------------------------------------------------------
diff --git a/conf/accumulo-testing.properties.example b/conf/accumulo-testing.properties.example
new file mode 100644
index 0000000..05cbaf5
--- /dev/null
+++ b/conf/accumulo-testing.properties.example
@@ -0,0 +1,44 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+###################
+# Common properties
+###################
+
+# Accumulo instance name
+test.common.accumulo.instance=instance
+# Accumulo username
+test.common.accumulo.username=root
+# Accumulo password
+test.common.accumulo.password=secret
+# Accumulo keytab
+#test.common.accumulo.keytab=
+# Zookeeper connection string
+test.common.zookeepers=localhost:2181
+# Memory (in MB) given to each container (if running in YARN)
+test.common.yarn.container.memory.mb=1024
+# Number of cores given to each container (if running in YARN)
+test.common.yarn.container.cores=1
+
+########################
+# Random walk properties
+########################
+
+# Max memory for multi-table batch writer
+test.randomwalk.bw.max.mem=100000000
+# Max latency in milliseconds for multi-table batch writer
+test.randomwalk.bw.max.latency=600000
+# Number of write thread for multi-table batch writer
+test.randomwalk.bw.num.threads=4

http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/efaa7377/conf/log4j.properties.example
----------------------------------------------------------------------
diff --git a/conf/log4j.properties.example b/conf/log4j.properties.example
new file mode 100644
index 0000000..4cbfd34
--- /dev/null
+++ b/conf/log4j.properties.example
@@ -0,0 +1,27 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+log4j.rootLogger=INFO, CA
+log4j.appender.CA=org.apache.log4j.ConsoleAppender
+log4j.appender.CA.layout=org.apache.log4j.PatternLayout
+log4j.appender.CA.layout.ConversionPattern=%d{ISO8601} [%c] %-5p: %m%n
+
+log4j.logger.org.apache.accumulo=WARN
+log4j.logger.org.apache.accumulo.testing=DEBUG
+log4j.logger.org.apache.curator=ERROR
+log4j.logger.org.apache.hadoop=WARN
+log4j.logger.org.apache.hadoop.mapreduce=ERROR
+log4j.logger.org.apache.hadoop.util.NativeCodeLoader=ERROR
+log4j.logger.org.apache.zookeeper=ERROR

http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/efaa7377/core/pom.xml
----------------------------------------------------------------------
diff --git a/core/pom.xml b/core/pom.xml
index 83998d5..742ff89 100644
--- a/core/pom.xml
+++ b/core/pom.xml
@@ -31,10 +31,6 @@
 
   <dependencies>
     <dependency>
-      <groupId>com.beust</groupId>
-      <artifactId>jcommander</artifactId>
-    </dependency>
-    <dependency>
       <groupId>com.google.guava</groupId>
       <artifactId>guava</artifactId>
     </dependency>

http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/efaa7377/core/src/main/java/org/apache/accumulo/testing/core/TestProps.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/TestProps.java b/core/src/main/java/org/apache/accumulo/testing/core/TestProps.java
new file mode 100644
index 0000000..f8ce9ca
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/testing/core/TestProps.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.accumulo.testing.core;
+
+public class TestProps {
+
+  private static final String PREFIX = "test.";
+  private static final String RANDOMWALK = PREFIX + "randomwalk.";
+  private static final String COMMON = PREFIX + "common.";
+
+  /** Common properties **/
+  // Zookeeper connection string
+  public static final String ZOOKEEPERS = COMMON + "zookeepers";
+  // Accumulo instance name
+  public static final String ACCUMULO_INSTANCE = COMMON + "accumulo.instance";
+  // Accumulo username
+  public static final String ACCUMULO_USERNAME = COMMON + "accumulo.username";
+  // Accumulo password
+  public static final String ACCUMULO_PASSWORD = COMMON + "accumulo.password";
+  // Accumulo keytab
+  public static final String ACCUMULO_KEYTAB = COMMON + "accumulo.keytab";
+  // Memory (in MB) given to each YARN container
+  public static final String YARN_CONTAINER_MEMORY_MB = COMMON + "yarn.container.memory.mb";
+  // Number of cores given to each YARN container
+  public static final String YARN_CONTAINER_CORES = COMMON + "yarn.container.cores";
+
+
+  /** Random walk properties **/
+  // Number of random walker (if running in YARN)
+  public static final String RW_NUM_WALKERS = RANDOMWALK + "num.walkers";
+  // Max memory for multi-table batch writer
+  public static final String RW_BW_MAX_MEM = RANDOMWALK + "bw.max.mem";
+  // Max latency in milliseconds for multi-table batch writer
+  public static final String RW_BW_MAX_LATENCY = RANDOMWALK + "bw.max.latency";
+  // Number of write thread for multi-table batch writer
+  public static final String RW_BW_NUM_THREADS = RANDOMWALK + "bw.num.threads";
+
+}

http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/efaa7377/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/Environment.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/Environment.java b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/Environment.java
index 5684353..92a5de0 100644
--- a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/Environment.java
+++ b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/Environment.java
@@ -35,6 +35,7 @@ import org.apache.accumulo.core.client.ZooKeeperInstance;
 import org.apache.accumulo.core.client.security.tokens.AuthenticationToken;
 import org.apache.accumulo.core.client.security.tokens.KerberosToken;
 import org.apache.accumulo.core.client.security.tokens.PasswordToken;
+import org.apache.accumulo.testing.core.TestProps;
 import org.apache.hadoop.security.UserGroupInformation;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -44,38 +45,6 @@ import org.slf4j.LoggerFactory;
  * for creating client-side objects. This class is not thread-safe.
  */
 public class Environment {
-  /**
-   * The configuration property key for a username.
-   */
-  public static final String KEY_USERNAME = "USERNAME";
-  /**
-   * The configuration property key for a password.
-   */
-  public static final String KEY_PASSWORD = "PASSWORD";
-  /**
-   * The configuration property key for a keytab
-   */
-  public static final String KEY_KEYTAB = "KEYTAB";
-  /**
-   * The configuration property key for the instance name.
-   */
-  public static final String KEY_INSTANCE = "INSTANCE";
-  /**
-   * The configuration property key for the comma-separated list of ZooKeepers.
-   */
-  public static final String KEY_ZOOKEEPERS = "ZOOKEEPERS";
-  /**
-   * The configuration property key for the maximum memory for the multi-table batch writer.
-   */
-  public static final String KEY_MAX_MEM = "MAX_MEM";
-  /**
-   * The configuration property key for the maximum latency, in milliseconds, for the multi-table batch writer.
-   */
-  public static final String KEY_MAX_LATENCY = "MAX_LATENCY";
-  /**
-   * The configuration property key for the number of write threads for the multi-table batch writer.
-   */
-  public static final String KEY_NUM_THREADS = "NUM_THREADS";
 
   private static final Logger log = LoggerFactory.getLogger(Environment.class);
 
@@ -123,7 +92,7 @@ public class Environment {
    * @return username
    */
   public String getUserName() {
-    return p.getProperty(KEY_USERNAME);
+    return p.getProperty(TestProps.ACCUMULO_USERNAME);
   }
 
   /**
@@ -132,7 +101,7 @@ public class Environment {
    * @return password
    */
   public String getPassword() {
-    return p.getProperty(KEY_PASSWORD);
+    return p.getProperty(TestProps.ACCUMULO_PASSWORD);
   }
 
   /**
@@ -141,7 +110,7 @@ public class Environment {
    * @return path to keytab
    */
   public String getKeytab() {
-    return p.getProperty(KEY_KEYTAB);
+    return p.getProperty(TestProps.ACCUMULO_KEYTAB);
   }
 
   /**
@@ -186,8 +155,8 @@ public class Environment {
    */
   public Instance getInstance() {
     if (instance == null) {
-      String instance = p.getProperty(KEY_INSTANCE);
-      String zookeepers = p.getProperty(KEY_ZOOKEEPERS);
+      String instance = p.getProperty(TestProps.ACCUMULO_INSTANCE);
+      String zookeepers = p.getProperty(TestProps.ZOOKEEPERS);
       this.instance = new ZooKeeperInstance(ClientConfiguration.loadDefault().withInstance(instance).withZkHosts(zookeepers));
     }
     return instance;
@@ -216,9 +185,9 @@ public class Environment {
    */
   public MultiTableBatchWriter getMultiTableBatchWriter() throws AccumuloException, AccumuloSecurityException {
     if (mtbw == null) {
-      long maxMem = Long.parseLong(p.getProperty(KEY_MAX_MEM));
-      long maxLatency = Long.parseLong(p.getProperty(KEY_MAX_LATENCY));
-      int numThreads = Integer.parseInt(p.getProperty(KEY_NUM_THREADS));
+      long maxMem = Long.parseLong(p.getProperty(TestProps.RW_BW_MAX_MEM));
+      long maxLatency = Long.parseLong(p.getProperty(TestProps.RW_BW_MAX_LATENCY));
+      int numThreads = Integer.parseInt(p.getProperty(TestProps.RW_BW_NUM_THREADS));
       mtbw = getConnector().createMultiTableBatchWriter(
           new BatchWriterConfig().setMaxMemory(maxMem).setMaxLatency(maxLatency, TimeUnit.MILLISECONDS).setMaxWriteThreads(numThreads));
     }

http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/efaa7377/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/Framework.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/Framework.java b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/Framework.java
index 1a5700e..a9f1d68 100644
--- a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/Framework.java
+++ b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/Framework.java
@@ -16,15 +16,12 @@
  */
 package org.apache.accumulo.testing.core.randomwalk;
 
-import java.io.File;
 import java.io.FileInputStream;
 import java.util.HashMap;
 import java.util.Properties;
 
 import org.apache.log4j.Logger;
 
-import com.beust.jcommander.Parameter;
-
 public class Framework {
 
   private static final Logger log = Logger.getLogger(Framework.class);
@@ -63,7 +60,7 @@ public class Framework {
    *          Name of node
    * @return Node specified by id
    */
-  public Node getNode(String id) throws Exception {
+  Node getNode(String id) throws Exception {
 
     // check for node in nodes
     if (nodes.containsKey(id)) {
@@ -71,9 +68,9 @@ public class Framework {
     }
 
     // otherwise create and put in nodes
-    Node node = null;
+    Node node;
     if (id.endsWith(".xml")) {
-      node = new Module(new File("/randomwalk/modules/" + id));
+      node = new Module(id);
     } else {
       node = (Test) Class.forName(id).newInstance();
     }
@@ -81,13 +78,6 @@ public class Framework {
     return node;
   }
 
-  static class Opts extends org.apache.accumulo.core.cli.Help {
-    @Parameter(names = "--configDir", required = true, description = "directory containing the test configuration")
-    String configDir;
-    @Parameter(names = "--module", required = true, description = "the name of the module to run")
-    String module;
-  }
-
   public static void main(String[] args) throws Exception {
 
     if (args.length != 2) {
@@ -100,10 +90,12 @@ public class Framework {
     props.load(fis);
     fis.close();
 
+    log.info("Running random walk test with module: " + args[1]);
+
     State state = new State();
     Environment env = new Environment(props);
-    int retval = getInstance().run(args[1], state, env);
+    getInstance().run(args[1], state, env);
 
-    System.exit(retval);
+    log.info("Test finished");
   }
 }

http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/efaa7377/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/Module.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/Module.java b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/Module.java
index 1a3d059..3206906 100644
--- a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/Module.java
+++ b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/Module.java
@@ -18,7 +18,6 @@ package org.apache.accumulo.testing.core.randomwalk;
 
 import static java.nio.charset.StandardCharsets.UTF_8;
 
-import java.io.File;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.HashMap;
@@ -170,12 +169,12 @@ public class Module extends Node {
   private HashMap<String,String> prefixes = new HashMap<>();
   private HashMap<String,AdjList> adjMap = new HashMap<>();
   private HashMap<String,Set<String>> aliasMap = new HashMap<>();
-  private final File xmlFile;
+  private final String id;
   private String initNodeId;
   private Fixture fixture = null;
 
-  public Module(File xmlFile) throws Exception {
-    this.xmlFile = xmlFile;
+  public Module(String id) throws Exception {
+    this.id = id;
     loadFromXml();
   }
 
@@ -414,7 +413,7 @@ public class Module extends Node {
 
   @Override
   public String toString() {
-    return xmlFile.toString();
+    return id;
   }
 
   private String getFullName(String name) {
@@ -497,12 +496,13 @@ public class Module extends Node {
     dbf.setSchema(moduleSchema);
 
     // parse the document
+
     try {
       docbuilder = dbf.newDocumentBuilder();
-      d = docbuilder.parse(xmlFile);
+      d = docbuilder.parse(this.getClass().getResourceAsStream("/randomwalk/modules/" + id));
     } catch (Exception e) {
-      log.error("Failed to parse: " + xmlFile, e);
-      throw new Exception("Failed to parse: " + xmlFile);
+      log.error("Failed to parse xml at randomwalk/modules/" + id, e);
+      throw new Exception("Failed to parse xml at randomwalk/modules/" + id);
     }
 
     // parse packages

http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/efaa7377/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/multitable/CopyTable.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/multitable/CopyTable.java b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/multitable/CopyTable.java
index 6552161..b67f3d7 100644
--- a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/multitable/CopyTable.java
+++ b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/multitable/CopyTable.java
@@ -23,6 +23,7 @@ import java.util.TreeSet;
 
 import org.apache.accumulo.core.client.impl.Tables;
 import org.apache.accumulo.core.util.CachedConfiguration;
+import org.apache.accumulo.testing.core.TestProps;
 import org.apache.accumulo.testing.core.randomwalk.Environment;
 import org.apache.accumulo.testing.core.randomwalk.State;
 import org.apache.accumulo.testing.core.randomwalk.Test;
@@ -64,7 +65,7 @@ public class CopyTable extends Test {
     }
     args[4] = srcTableName;
     args[5] = env.getInstance().getInstanceName();
-    args[6] = env.getConfigProperty("ZOOKEEPERS");
+    args[6] = env.getConfigProperty(TestProps.ZOOKEEPERS);
     args[7] = dstTableName;
 
     log.debug("copying " + srcTableName + " to " + dstTableName);

http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/efaa7377/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/sequential/MapRedVerify.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/sequential/MapRedVerify.java b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/sequential/MapRedVerify.java
index 58d44d4..5113973 100644
--- a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/sequential/MapRedVerify.java
+++ b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/sequential/MapRedVerify.java
@@ -26,6 +26,7 @@ import org.apache.accumulo.core.data.Range;
 import org.apache.accumulo.core.data.Value;
 import org.apache.accumulo.core.security.Authorizations;
 import org.apache.accumulo.core.util.CachedConfiguration;
+import org.apache.accumulo.testing.core.TestProps;
 import org.apache.accumulo.testing.core.randomwalk.Environment;
 import org.apache.accumulo.testing.core.randomwalk.State;
 import org.apache.accumulo.testing.core.randomwalk.Test;
@@ -46,7 +47,7 @@ public class MapRedVerify extends Test {
     }
     args[4] = state.getString("seqTableName");
     args[5] = env.getInstance().getInstanceName();
-    args[6] = env.getConfigProperty("ZOOKEEPERS");
+    args[6] = env.getConfigProperty(TestProps.ZOOKEEPERS);
     args[7] = args[4] + "_MR";
 
     if (ToolRunner.run(CachedConfiguration.getInstance(), new MapRedVerifyTool(), args) != 0) {

http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/efaa7377/core/src/main/resources/randomwalk/modules/Bulk.xml
----------------------------------------------------------------------
diff --git a/core/src/main/resources/randomwalk/modules/Bulk.xml b/core/src/main/resources/randomwalk/modules/Bulk.xml
index 35e2a67..9ffa76a 100644
--- a/core/src/main/resources/randomwalk/modules/Bulk.xml
+++ b/core/src/main/resources/randomwalk/modules/Bulk.xml
@@ -17,7 +17,7 @@
 -->
 <module>
 
-<package prefix="bulk" value="org.apache.accumulo.test.randomwalk.bulk"/>
+<package prefix="bulk" value="org.apache.accumulo.testing.core.randomwalk.bulk"/>
 
 <init id="bulk.Setup"/>
 

http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/efaa7377/core/src/main/resources/randomwalk/modules/Concurrent.xml
----------------------------------------------------------------------
diff --git a/core/src/main/resources/randomwalk/modules/Concurrent.xml b/core/src/main/resources/randomwalk/modules/Concurrent.xml
index 36ea53c..ea96b27 100644
--- a/core/src/main/resources/randomwalk/modules/Concurrent.xml
+++ b/core/src/main/resources/randomwalk/modules/Concurrent.xml
@@ -17,7 +17,7 @@
 -->
 <module>
 
-<package prefix="ct" value="org.apache.accumulo.test.randomwalk.concurrent"/>
+<package prefix="ct" value="org.apache.accumulo.testing.core.randomwalk.concurrent"/>
 
 <fixture id="ct.ConcurrentFixture"/>
 

http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/efaa7377/core/src/main/resources/randomwalk/modules/Conditional.xml
----------------------------------------------------------------------
diff --git a/core/src/main/resources/randomwalk/modules/Conditional.xml b/core/src/main/resources/randomwalk/modules/Conditional.xml
index 54ff7ab..3c380dc 100644
--- a/core/src/main/resources/randomwalk/modules/Conditional.xml
+++ b/core/src/main/resources/randomwalk/modules/Conditional.xml
@@ -17,8 +17,7 @@
 -->
 <module>
 
-<package prefix="ct" value="org.apache.accumulo.test.randomwalk.conditional"/>
-
+<package prefix="ct" value="org.apache.accumulo.testing.core.randomwalk.conditional"/>
 
 <init id="ct.Setup"/>
 

http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/efaa7377/core/src/main/resources/randomwalk/modules/Image.xml
----------------------------------------------------------------------
diff --git a/core/src/main/resources/randomwalk/modules/Image.xml b/core/src/main/resources/randomwalk/modules/Image.xml
index 7561895..0b41d14 100644
--- a/core/src/main/resources/randomwalk/modules/Image.xml
+++ b/core/src/main/resources/randomwalk/modules/Image.xml
@@ -17,7 +17,7 @@
 -->
 <module>
 
-<package prefix="image" value="org.apache.accumulo.test.randomwalk.image"/>
+<package prefix="image" value="org.apache.accumulo.testing.core.randomwalk.image"/>
 
 <fixture id="image.ImageFixture"/>
 

http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/efaa7377/core/src/main/resources/randomwalk/modules/MultiTable.xml
----------------------------------------------------------------------
diff --git a/core/src/main/resources/randomwalk/modules/MultiTable.xml b/core/src/main/resources/randomwalk/modules/MultiTable.xml
index 55f6590..fa42c3f 100644
--- a/core/src/main/resources/randomwalk/modules/MultiTable.xml
+++ b/core/src/main/resources/randomwalk/modules/MultiTable.xml
@@ -17,7 +17,7 @@
 -->
 <module>
 
-<package prefix="mt" value="org.apache.accumulo.test.randomwalk.multitable"/>
+<package prefix="mt" value="org.apache.accumulo.testing.core.randomwalk.multitable"/>
 
 <fixture id="mt.MultiTableFixture"/>
 

http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/efaa7377/core/src/main/resources/randomwalk/modules/Security.xml
----------------------------------------------------------------------
diff --git a/core/src/main/resources/randomwalk/modules/Security.xml b/core/src/main/resources/randomwalk/modules/Security.xml
index 9e9ef9f..422b63e 100644
--- a/core/src/main/resources/randomwalk/modules/Security.xml
+++ b/core/src/main/resources/randomwalk/modules/Security.xml
@@ -12,7 +12,7 @@
 <module>
 
   <package prefix="security"
-    value="org.apache.accumulo.test.randomwalk.security" />
+    value="org.apache.accumulo.testing.core.randomwalk.security" />
 
   <fixture id="security.SecurityFixture" />
 

http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/efaa7377/core/src/main/resources/randomwalk/modules/Sequential.xml
----------------------------------------------------------------------
diff --git a/core/src/main/resources/randomwalk/modules/Sequential.xml b/core/src/main/resources/randomwalk/modules/Sequential.xml
index 454e75f..ce4abd0 100644
--- a/core/src/main/resources/randomwalk/modules/Sequential.xml
+++ b/core/src/main/resources/randomwalk/modules/Sequential.xml
@@ -17,7 +17,7 @@
 -->
 <module>
 
-<package prefix="seq" value="org.apache.accumulo.test.randomwalk.sequential"/>
+<package prefix="seq" value="org.apache.accumulo.testing.core.randomwalk.sequential"/>
 
 <fixture id="seq.SequentialFixture"/>
 

http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/efaa7377/core/src/main/resources/randomwalk/modules/Shard.xml
----------------------------------------------------------------------
diff --git a/core/src/main/resources/randomwalk/modules/Shard.xml b/core/src/main/resources/randomwalk/modules/Shard.xml
index eb23f37..e8e8654 100644
--- a/core/src/main/resources/randomwalk/modules/Shard.xml
+++ b/core/src/main/resources/randomwalk/modules/Shard.xml
@@ -17,7 +17,7 @@
 -->
 <module>
 
-<package prefix="shard" value="org.apache.accumulo.test.randomwalk.shard"/>
+<package prefix="shard" value="org.apache.accumulo.testing.core.randomwalk.shard"/>
 
 <fixture id="shard.ShardFixture"/>
 

http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/efaa7377/core/src/main/resources/randomwalk/modules/unit/Basic.xml
----------------------------------------------------------------------
diff --git a/core/src/main/resources/randomwalk/modules/unit/Basic.xml b/core/src/main/resources/randomwalk/modules/unit/Basic.xml
index 2dead02..f81f132 100644
--- a/core/src/main/resources/randomwalk/modules/unit/Basic.xml
+++ b/core/src/main/resources/randomwalk/modules/unit/Basic.xml
@@ -17,7 +17,7 @@
 -->
 <module>
 
-<package prefix="test" value="org.apache.accumulo.test.randomwalk.unit"/>
+<package prefix="test" value="org.apache.accumulo.testing.core.randomwalk.unit"/>
 
 <init id="test.CreateTable"/>
 

http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/efaa7377/core/src/main/resources/randomwalk/modules/unit/Simple.xml
----------------------------------------------------------------------
diff --git a/core/src/main/resources/randomwalk/modules/unit/Simple.xml b/core/src/main/resources/randomwalk/modules/unit/Simple.xml
index cad940e..6eb685a 100644
--- a/core/src/main/resources/randomwalk/modules/unit/Simple.xml
+++ b/core/src/main/resources/randomwalk/modules/unit/Simple.xml
@@ -17,7 +17,7 @@
 -->
 <module>
 
-<package prefix="test" value="org.apache.accumulo.test.randomwalk.unit"/>
+<package prefix="test" value="org.apache.accumulo.testing.core.randomwalk.unit"/>
 
 <init id="dummy.all"/>
 

http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/efaa7377/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index a2efdf4..2c72faf 100644
--- a/pom.xml
+++ b/pom.xml
@@ -34,6 +34,7 @@
 
   <modules>
     <module>core</module>
+    <module>yarn</module>
   </modules>
 
   <properties>
@@ -41,6 +42,7 @@
     <hadoop.version>2.6.4</hadoop.version>
     <zookeeper.version>3.4.6</zookeeper.version>
     <slf4j.version>1.7.21</slf4j.version>
+    <twill.version>0.9.0</twill.version>
     <maven.compiler.source>1.8</maven.compiler.source>
     <maven.compiler.target>1.8</maven.compiler.target>
   </properties>
@@ -58,6 +60,11 @@
         <version>1.48</version>
       </dependency>
       <dependency>
+        <groupId>org.apache.accumulo</groupId>
+        <artifactId>accumulo-testing-core</artifactId>
+        <version>${project.version}</version>
+      </dependency>
+      <dependency>
         <groupId>commons-configuration</groupId>
         <artifactId>commons-configuration</artifactId>
         <version>1.6</version>
@@ -93,6 +100,46 @@
         <version>${hadoop.version}</version>
       </dependency>
       <dependency>
+        <groupId>org.apache.hadoop</groupId>
+        <artifactId>hadoop-common</artifactId>
+        <version>${hadoop.version}</version>
+      </dependency>
+      <dependency>
+        <groupId>org.apache.hadoop</groupId>
+        <artifactId>hadoop-hdfs</artifactId>
+        <version>${hadoop.version}</version>
+      </dependency>
+      <dependency>
+        <groupId>org.apache.hadoop</groupId>
+        <artifactId>hadoop-yarn-api</artifactId>
+        <version>${hadoop.version}</version>
+      </dependency>
+      <dependency>
+        <groupId>org.apache.hadoop</groupId>
+        <artifactId>hadoop-yarn-client</artifactId>
+        <version>${hadoop.version}</version>
+      </dependency>
+      <dependency>
+        <groupId>org.apache.hadoop</groupId>
+        <artifactId>hadoop-yarn-common</artifactId>
+        <version>${hadoop.version}</version>
+      </dependency>
+      <dependency>
+        <groupId>org.apache.twill</groupId>
+        <artifactId>twill-api</artifactId>
+        <version>${twill.version}</version>
+      </dependency>
+      <dependency>
+        <groupId>org.apache.twill</groupId>
+        <artifactId>twill-ext</artifactId>
+        <version>${twill.version}</version>
+      </dependency>
+      <dependency>
+        <groupId>org.apache.twill</groupId>
+        <artifactId>twill-yarn</artifactId>
+        <version>${twill.version}</version>
+      </dependency>
+      <dependency>
         <groupId>org.apache.zookeeper</groupId>
         <artifactId>zookeeper</artifactId>
         <version>${zookeeper.version}</version>

http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/efaa7377/yarn/.gitignore
----------------------------------------------------------------------
diff --git a/yarn/.gitignore b/yarn/.gitignore
new file mode 100644
index 0000000..17bb010
--- /dev/null
+++ b/yarn/.gitignore
@@ -0,0 +1,2 @@
+/target/
+/*.iml

http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/efaa7377/yarn/pom.xml
----------------------------------------------------------------------
diff --git a/yarn/pom.xml b/yarn/pom.xml
new file mode 100644
index 0000000..650c3b9
--- /dev/null
+++ b/yarn/pom.xml
@@ -0,0 +1,110 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  Licensed to the Apache Software Foundation (ASF) under one or more
+  contributor license agreements.  See the NOTICE file distributed with
+  this work for additional information regarding copyright ownership.
+  The ASF licenses this file to You under the Apache License, Version 2.0
+  (the "License"); you may not use this file except in compliance with
+  the License.  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+  <modelVersion>4.0.0</modelVersion>
+
+  <parent>
+    <groupId>org.apache.accumulo</groupId>
+    <artifactId>accumulo-testing</artifactId>
+    <version>2.0.0-SNAPSHOT</version>
+  </parent>
+
+  <artifactId>accumulo-testing-yarn</artifactId>
+  <packaging>jar</packaging>
+
+  <name>Apache Accumulo Testing YARN</name>
+
+  <dependencies>
+    <dependency>
+      <groupId>com.beust</groupId>
+      <artifactId>jcommander</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>com.google.guava</groupId>
+      <artifactId>guava</artifactId>
+      <!-- Twill requires version 13.0.1 -->
+      <version>13.0.1</version>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.accumulo</groupId>
+      <artifactId>accumulo-testing-core</artifactId>
+      <exclusions>
+        <exclusion>
+          <groupId>log4j</groupId>
+          <artifactId>log4j</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>org.slf4j</groupId>
+          <artifactId>slf4j-log4j12</artifactId>
+        </exclusion>
+      </exclusions>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-yarn-api</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.twill</groupId>
+      <artifactId>twill-api</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.twill</groupId>
+      <artifactId>twill-ext</artifactId>
+      <exclusions>
+        <exclusion>
+          <groupId>log4j</groupId>
+          <artifactId>log4j</artifactId>
+        </exclusion>
+      </exclusions>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.twill</groupId>
+      <artifactId>twill-yarn</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>org.slf4j</groupId>
+      <artifactId>slf4j-api</artifactId>
+    </dependency>
+  </dependencies>
+
+  <profiles>
+    <profile>
+      <id>yarn-test-runner</id>
+      <build>
+        <plugins>
+          <plugin>
+            <groupId>org.codehaus.mojo</groupId>
+            <artifactId>exec-maven-plugin</artifactId>
+            <executions>
+              <execution>
+                <goals>
+                  <goal>java</goal>
+                </goals>
+                <phase>compile</phase>
+                <configuration>
+                  <mainClass>org.apache.accumulo.testing.yarn.YarnAccumuloTestRunner</mainClass>
+                </configuration>
+              </execution>
+            </executions>
+          </plugin>
+        </plugins>
+      </build>
+    </profile>
+  </profiles>
+
+</project>

http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/efaa7377/yarn/src/main/java/org/apache/accumulo/testing/yarn/YarnAccumuloTestRunner.java
----------------------------------------------------------------------
diff --git a/yarn/src/main/java/org/apache/accumulo/testing/yarn/YarnAccumuloTestRunner.java b/yarn/src/main/java/org/apache/accumulo/testing/yarn/YarnAccumuloTestRunner.java
new file mode 100644
index 0000000..e50dbb4
--- /dev/null
+++ b/yarn/src/main/java/org/apache/accumulo/testing/yarn/YarnAccumuloTestRunner.java
@@ -0,0 +1,169 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.testing.yarn;
+
+import com.beust.jcommander.JCommander;
+import com.beust.jcommander.Parameter;
+import com.google.common.base.Preconditions;
+import org.apache.accumulo.testing.core.TestProps;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.twill.api.ResourceSpecification;
+import org.apache.twill.api.TwillApplication;
+import org.apache.twill.api.TwillController;
+import org.apache.twill.api.TwillRunnerService;
+import org.apache.twill.api.TwillSpecification;
+import org.apache.twill.ext.BundledJarRunnable;
+import org.apache.twill.ext.BundledJarRunner;
+import org.apache.twill.yarn.YarnTwillRunnerService;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Properties;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+public class YarnAccumuloTestRunner {
+
+  private static final Logger LOG = LoggerFactory.getLogger(YarnAccumuloTestRunner.class);
+
+  private static class YarnTestApp implements TwillApplication {
+
+    private TestRunnerOpts opts;
+    private Properties props;
+
+    YarnTestApp(TestRunnerOpts opts, Properties props) {
+      this.opts = opts;
+      this.props = props;
+    }
+
+    @Override
+    public TwillSpecification configure() {
+
+      int numCores = Integer.valueOf(props.getProperty(TestProps.YARN_CONTAINER_CORES));
+      int memory = Integer.valueOf(props.getProperty(TestProps.YARN_CONTAINER_MEMORY_MB));
+
+      ResourceSpecification resourceSpec = ResourceSpecification.Builder.with()
+          .setVirtualCores(numCores).setMemory(memory, ResourceSpecification.SizeUnit.MEGA)
+          .setInstances(opts.numContainers).build();
+
+      File jarFile = new File(opts.jarPath);
+      File testProps = new File(opts.testProps);
+      File log4jProps = new File(opts.logProps);
+
+      return TwillSpecification.Builder.with()
+          .setName(opts.testName)
+          .withRunnable()
+          .add("BundledJarRunnable", new BundledJarRunnable(), resourceSpec)
+          .withLocalFiles()
+          .add(jarFile.getName(), jarFile.toURI(), false)
+          .add(testProps.getName(), testProps.toURI())
+          .add(log4jProps.getName(), log4jProps.toURI())
+          .apply()
+          .anyOrder()
+          .build();
+    }
+  }
+
+  private static class TestRunnerOpts {
+
+    @Parameter(names={"--testName", "-t"}, required = true,  description = "Test name")
+    String testName;
+
+    @Parameter(names={"--numContainers", "-n"}, required = true,  description = "Test name")
+    int numContainers;
+
+    @Parameter(names={"--jar", "-j"}, required = true, description = "Bundled jar path")
+    String jarPath;
+
+    @Parameter(names={"--main", "-m"}, required = true, description = "Main class")
+    String mainClass;
+
+    @Parameter(names={"--testProps", "-p"}, required = true, description = "Test properties path")
+    String testProps;
+
+    @Parameter(names={"--logProps", "-l"}, required = true, description = "Log properties path")
+    String logProps;
+
+    @Parameter(names={"--args", "-a"}, variableArity = true, description = "Main class args")
+    List<String> mainArgs = new ArrayList<>();
+  }
+
+  private static void verifyPath(String path) {
+    File f = new File(path);
+    Preconditions.checkState(f.exists());
+    Preconditions.checkState(f.canRead());
+  }
+
+  public static void main(String[] args) throws Exception {
+
+    TestRunnerOpts opts = new TestRunnerOpts();
+    new JCommander(opts, args);
+
+    verifyPath(opts.jarPath);
+    verifyPath(opts.testProps);
+    verifyPath(opts.logProps);
+
+    String[] mainArgs = opts.mainArgs.stream().toArray(String[]::new);
+    BundledJarRunner.Arguments arguments = new BundledJarRunner.Arguments(opts.jarPath, "/lib",
+                                                                          opts.mainClass, mainArgs);
+
+    Properties props = new Properties();
+    FileInputStream fis = new FileInputStream(opts.testProps);
+    props.load(fis);
+    fis.close();
+    String zookeepers = props.getProperty(TestProps.ZOOKEEPERS);
+
+    final TwillRunnerService twillRunner = new YarnTwillRunnerService(new YarnConfiguration(),
+                                                                      zookeepers);
+    twillRunner.start();
+
+    final TwillController controller = twillRunner.prepare(
+        new YarnTestApp(opts, props))
+        .addJVMOptions("-Dlog4j.configuration=file:$PWD/" + new File(opts.logProps).getName())
+        .withArguments("BundledJarRunnable", arguments.toArray())
+        .start();
+
+    final AtomicBoolean done = new AtomicBoolean(false);
+
+    Runtime.getRuntime().addShutdownHook(new Thread(() -> {
+      try {
+        if (!done.get()) {
+          controller.kill();
+        }
+      } finally {
+        twillRunner.stop();
+      }
+    }));
+
+    LOG.info("Waiting for {} to finish in YARN...", opts.testName);
+    LOG.info("Press ctrl-c to kill {} in YARN", opts.testName);
+
+    try {
+      controller.awaitTerminated();
+      done.set(true);
+    } catch (ExecutionException e) {
+      LOG.error("Exception during execution", e);
+      throw e;
+    }
+    LOG.info("{} finished", opts.testName);
+  }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/efaa7377/yarn/src/main/resources/logback.xml
----------------------------------------------------------------------
diff --git a/yarn/src/main/resources/logback.xml b/yarn/src/main/resources/logback.xml
new file mode 100644
index 0000000..7a8e3b8
--- /dev/null
+++ b/yarn/src/main/resources/logback.xml
@@ -0,0 +1,31 @@
+<!--
+  Licensed to the Apache Software Foundation (ASF) under one or more contributor license
+  agreements. See the NOTICE file distributed with this work for additional information regarding
+  copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the
+  "License"); you may not use this file except in compliance with the License. You may obtain a
+  copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software distributed under the License
+  is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+  or implied. See the License for the specific language governing permissions and limitations under
+  the License.
+-->
+<configuration>
+
+  <appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
+    <!-- encoders are assigned the type
+         ch.qos.logback.classic.encoder.PatternLayoutEncoder by default -->
+    <encoder>
+      <pattern>%d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n</pattern>
+    </encoder>
+  </appender>
+  
+  <logger name="org.apache.twill" level="warn"/>
+  <logger name="org.apache.twill.yarn.YarnTwillRunnerService" level="info"/>
+
+  <root level="info">
+    <appender-ref ref="STDOUT" />
+  </root>
+</configuration>


Mime
View raw message