accumulo-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From mwa...@apache.org
Subject [4/5] accumulo-testing git commit: ACCUMULO-4510 Moved remaining external test code from Accumulo
Date Wed, 25 Jan 2017 18:21:53 GMT
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/0d97273c/core/src/main/java/org/apache/accumulo/testing/core/merkle/cli/ManualComparison.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/merkle/cli/ManualComparison.java b/core/src/main/java/org/apache/accumulo/testing/core/merkle/cli/ManualComparison.java
new file mode 100644
index 0000000..87489f6
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/testing/core/merkle/cli/ManualComparison.java
@@ -0,0 +1,95 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.testing.core.merkle.cli;
+
+import java.util.Iterator;
+import java.util.Map.Entry;
+
+import org.apache.accumulo.core.cli.ClientOpts;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.security.Authorizations;
+
+import com.beust.jcommander.Parameter;
+
+/**
+ * Accepts two table names and enumerates all key-values pairs in both checking for correctness. All differences between the two tables will be printed to the
+ * console.
+ */
+public class ManualComparison {
+
+  public static class ManualComparisonOpts extends ClientOpts {
+    @Parameter(names = {"--table1"}, required = true, description = "First table")
+    public String table1;
+
+    @Parameter(names = {"--table2"}, required = true, description = "First table")
+    public String table2;
+  }
+
+  public static void main(String[] args) throws Exception {
+    ManualComparisonOpts opts = new ManualComparisonOpts();
+    opts.parseArgs("ManualComparison", args);
+
+    Connector conn = opts.getConnector();
+
+    Scanner s1 = conn.createScanner(opts.table1, Authorizations.EMPTY), s2 = conn.createScanner(opts.table2, Authorizations.EMPTY);
+    Iterator<Entry<Key,Value>> iter1 = s1.iterator(), iter2 = s2.iterator();
+    boolean incrementFirst = true, incrementSecond = true;
+
+    Entry<Key,Value> entry1 = iter1.next(), entry2 = iter2.next();
+    while (iter1.hasNext() && iter2.hasNext()) {
+      if (incrementFirst) {
+        entry1 = iter1.next();
+      }
+      if (incrementSecond) {
+        entry2 = iter2.next();
+      }
+      incrementFirst = false;
+      incrementSecond = false;
+
+      if (!entry1.equals(entry2)) {
+
+        if (entry1.getKey().compareTo(entry2.getKey()) < 0) {
+          System.out.println("Exist in original " + entry1);
+          incrementFirst = true;
+        } else if (entry2.getKey().compareTo(entry1.getKey()) < 0) {
+          System.out.println("Exist in replica " + entry2);
+          incrementSecond = true;
+        } else {
+          System.out.println("Differ... " + entry1 + " " + entry2);
+          incrementFirst = true;
+          incrementSecond = true;
+        }
+      } else {
+        incrementFirst = true;
+        incrementSecond = true;
+      }
+    }
+
+    System.out.println("\nExtra entries from " + opts.table1);
+    while (iter1.hasNext()) {
+      System.out.println(iter1.next());
+    }
+
+    System.out.println("\nExtra entries from " + opts.table2);
+    while (iter2.hasNext()) {
+      System.out.println(iter2.next());
+    }
+  }
+}

http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/0d97273c/core/src/main/java/org/apache/accumulo/testing/core/merkle/ingest/RandomWorkload.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/merkle/ingest/RandomWorkload.java b/core/src/main/java/org/apache/accumulo/testing/core/merkle/ingest/RandomWorkload.java
new file mode 100644
index 0000000..440b009
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/testing/core/merkle/ingest/RandomWorkload.java
@@ -0,0 +1,120 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.testing.core.merkle.ingest;
+
+import java.util.Random;
+
+import org.apache.accumulo.core.cli.BatchWriterOpts;
+import org.apache.accumulo.core.cli.ClientOnDefaultTable;
+import org.apache.accumulo.core.client.BatchWriter;
+import org.apache.accumulo.core.client.BatchWriterConfig;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.Value;
+import org.apache.hadoop.io.Text;
+
+import com.beust.jcommander.Parameter;
+
+/**
+ * Generates some random data with a given percent of updates to be deletes.
+ */
+public class RandomWorkload {
+  public static final String DEFAULT_TABLE_NAME = "randomWorkload";
+
+  public static class RandomWorkloadOpts extends ClientOnDefaultTable {
+    @Parameter(names = {"-n", "--num"}, required = true, description = "Num records to write")
+    public long numRecords;
+
+    @Parameter(names = {"-r", "--rows"}, required = true, description = "Range of rows that can be generated")
+    public int rowMax;
+
+    @Parameter(names = {"-cf", "--colfams"}, required = true, description = "Range of column families that can be generated")
+    public int cfMax;
+
+    @Parameter(names = {"-cq", "--colquals"}, required = true, description = "Range of column qualifiers that can be generated")
+    public int cqMax;
+
+    @Parameter(names = {"-d", "--deletes"}, required = false, description = "Percentage of updates that should be deletes")
+    public int deletePercent = 5;
+
+    public RandomWorkloadOpts() {
+      super(DEFAULT_TABLE_NAME);
+    }
+
+    public RandomWorkloadOpts(String tableName) {
+      super(tableName);
+    }
+  }
+
+  public void run(RandomWorkloadOpts opts, BatchWriterConfig cfg) throws Exception {
+    run(opts.getConnector(), opts.getTableName(), cfg, opts.numRecords, opts.rowMax, opts.cfMax, opts.cqMax, opts.deletePercent);
+  }
+
+  public void run(final Connector conn, final String tableName, final BatchWriterConfig cfg, final long numRecords, int rowMax, int cfMax, int cqMax,
+      int deletePercent) throws Exception {
+
+    final Random rowRand = new Random(12345);
+    final Random cfRand = new Random(12346);
+    final Random cqRand = new Random(12347);
+    final Random deleteRand = new Random(12348);
+    long valueCounter = 0l;
+
+    if (!conn.tableOperations().exists(tableName)) {
+      conn.tableOperations().create(tableName);
+    }
+
+    BatchWriter bw = conn.createBatchWriter(tableName, cfg);
+    try {
+      final Text row = new Text(), cf = new Text(), cq = new Text();
+      final Value value = new Value();
+      for (long i = 0; i < numRecords; i++) {
+        row.set(Integer.toString(rowRand.nextInt(rowMax)));
+        cf.set(Integer.toString(cfRand.nextInt(cfMax)));
+        cq.set(Integer.toString(cqRand.nextInt(cqMax)));
+
+        Mutation m = new Mutation(row);
+
+        // Choose a random value between [0,100)
+        int deleteValue = deleteRand.nextInt(100);
+
+        // putDelete if the value we chose is less than our delete percentage
+        if (deleteValue < deletePercent) {
+          m.putDelete(cf, cq);
+        } else {
+          value.set(Long.toString(valueCounter).getBytes());
+          m.put(cf, cq, valueCounter, value);
+        }
+
+        bw.addMutation(m);
+
+        valueCounter++;
+      }
+    } finally {
+      bw.close();
+    }
+  }
+
+  public static void main(String[] args) throws Exception {
+    RandomWorkloadOpts opts = new RandomWorkloadOpts();
+    BatchWriterOpts bwOpts = new BatchWriterOpts();
+    opts.parseArgs(RandomWorkload.class.getSimpleName(), args, bwOpts);
+
+    RandomWorkload rw = new RandomWorkload();
+
+    rw.run(opts, bwOpts.getBatchWriterConfig());
+  }
+}

http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/0d97273c/core/src/main/java/org/apache/accumulo/testing/core/merkle/package-info.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/merkle/package-info.java b/core/src/main/java/org/apache/accumulo/testing/core/merkle/package-info.java
new file mode 100644
index 0000000..5b27e4b
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/testing/core/merkle/package-info.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/**
+ * A <a href="http://en.wikipedia.org/wiki/Merkle_tree">Merkle tree</a> is a hash tree and can be used to evaluate equality over large
+ * files with the ability to ascertain what portions of the files differ. Each leaf of the Merkle tree is some hash of a
+ * portion of the file, with each leaf corresponding to some "range" within the source file. As such, if all leaves are
+ * considered as ranges of the source file, the "sum" of all leaves creates a contiguous range over the entire file.
+ * <p>
+ * The parent of any nodes (typically, a binary tree; however this is not required) is the concatenation of the hashes of
+ * the children. We can construct a full tree by walking up the tree, creating parents from children, until we have a root
+ * node. To check equality of two files that each have a merkle tree built, we can very easily compare the value of at the
+ * root of the Merkle tree to know whether or not the files are the same.
+ * <p>
+ * Additionally, in the situation where we have two files with we expect to be the same but are not, we can walk back down
+ * the tree, finding subtrees that are equal and subtrees that are not. Subtrees that are equal correspond to portions of
+ * the files which are identical, where subtrees that are not equal correspond to discrepancies between the two files.
+ * <p>
+ * We can apply this concept to Accumulo, treating a table as a file, and ranges within a file as an Accumulo Range. We can
+ * then compute the hashes over each of these Ranges and compute the entire Merkle tree to determine if two tables are
+ * equivalent.
+ *
+ * @since 1.7.0
+ */
+package org.apache.accumulo.testing.core.merkle;
+

http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/0d97273c/core/src/main/java/org/apache/accumulo/testing/core/merkle/skvi/DigestIterator.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/merkle/skvi/DigestIterator.java b/core/src/main/java/org/apache/accumulo/testing/core/merkle/skvi/DigestIterator.java
new file mode 100644
index 0000000..ab7ad43
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/testing/core/merkle/skvi/DigestIterator.java
@@ -0,0 +1,149 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.testin.core.merkle.skvi;
+
+import java.io.ByteArrayOutputStream;
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.security.MessageDigest;
+import java.security.NoSuchAlgorithmException;
+import java.util.Collection;
+import java.util.Map;
+
+import org.apache.accumulo.core.data.ByteSequence;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.iterators.IteratorEnvironment;
+import org.apache.accumulo.core.iterators.SortedKeyValueIterator;
+
+/**
+ * {@link SortedKeyValueIterator} which attempts to compute a hash over some range of Key-Value pairs.
+ * <p>
+ * For the purposes of constructing a Merkle tree, this class will only generate a meaningful result if the (Batch)Scanner will compute a single digest over a
+ * Range. If the (Batch)Scanner stops and restarts in the middle of a session, incorrect values will be returned and the merkle tree will be invalid.
+ */
+public class DigestIterator implements SortedKeyValueIterator<Key,Value> {
+  public static final String HASH_NAME_KEY = "hash.name";
+
+  private MessageDigest digest;
+  private Key topKey;
+  private Value topValue;
+  private SortedKeyValueIterator<Key,Value> source;
+
+  @Override
+  public void init(SortedKeyValueIterator<Key,Value> source, Map<String,String> options, IteratorEnvironment env) throws IOException {
+    String hashName = options.get(HASH_NAME_KEY);
+    if (null == hashName) {
+      throw new IOException(HASH_NAME_KEY + " must be provided as option");
+    }
+
+    try {
+      this.digest = MessageDigest.getInstance(hashName);
+    } catch (NoSuchAlgorithmException e) {
+      throw new IOException(e);
+    }
+
+    this.topKey = null;
+    this.topValue = null;
+    this.source = source;
+  }
+
+  @Override
+  public boolean hasTop() {
+    return null != topKey;
+  }
+
+  @Override
+  public void next() throws IOException {
+    // We can't call next() if we already consumed it all
+    if (!this.source.hasTop()) {
+      this.topKey = null;
+      this.topValue = null;
+      return;
+    }
+
+    this.source.next();
+
+    consume();
+  }
+
+  @Override
+  public void seek(Range range, Collection<ByteSequence> columnFamilies, boolean inclusive) throws IOException {
+    this.source.seek(range, columnFamilies, inclusive);
+
+    consume();
+  }
+
+  protected void consume() throws IOException {
+    digest.reset();
+    ByteArrayOutputStream baos = new ByteArrayOutputStream();
+    DataOutputStream dos = new DataOutputStream(baos);
+
+    if (!this.source.hasTop()) {
+      this.topKey = null;
+      this.topValue = null;
+
+      return;
+    }
+
+    Key lastKeySeen = null;
+    while (this.source.hasTop()) {
+      baos.reset();
+
+      Key currentKey = this.source.getTopKey();
+      lastKeySeen = currentKey;
+
+      currentKey.write(dos);
+      this.source.getTopValue().write(dos);
+
+      digest.update(baos.toByteArray());
+
+      this.source.next();
+    }
+
+    this.topKey = lastKeySeen;
+    this.topValue = new Value(digest.digest());
+  }
+
+  @Override
+  public Key getTopKey() {
+    return topKey;
+  }
+
+  @Override
+  public Value getTopValue() {
+    return topValue;
+  }
+
+  @Override
+  public SortedKeyValueIterator<Key,Value> deepCopy(IteratorEnvironment env) {
+    DigestIterator copy = new DigestIterator();
+    try {
+      copy.digest = MessageDigest.getInstance(digest.getAlgorithm());
+    } catch (NoSuchAlgorithmException e) {
+      throw new RuntimeException(e);
+    }
+
+    copy.topKey = this.topKey;
+    copy.topValue = this.topValue;
+    copy.source = this.source.deepCopy(env);
+
+    return copy;
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/0d97273c/core/src/main/java/org/apache/accumulo/testing/core/scalability/Ingest.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/scalability/Ingest.java b/core/src/main/java/org/apache/accumulo/testing/core/scalability/Ingest.java
new file mode 100644
index 0000000..1a61eea
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/testing/core/scalability/Ingest.java
@@ -0,0 +1,143 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.testing.core.scalability;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+
+import java.util.Random;
+import java.util.UUID;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.accumulo.core.client.BatchWriter;
+import org.apache.accumulo.core.client.BatchWriterConfig;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.MutationsRejectedException;
+import org.apache.accumulo.core.client.TableNotFoundException;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.security.ColumnVisibility;
+import org.apache.accumulo.testing.core.continuous.ContinuousIngest;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class Ingest extends ScaleTest {
+
+  private static final Logger log = LoggerFactory.getLogger(Ingest.class);
+
+  @Override
+  public void setup() {
+
+    Connector conn = getConnector();
+    String tableName = getTestProperty("TABLE");
+
+    // delete existing table
+    if (conn.tableOperations().exists(tableName)) {
+      System.out.println("Deleting existing table: " + tableName);
+      try {
+        conn.tableOperations().delete(tableName);
+      } catch (Exception e) {
+        log.error("Failed to delete table '" + tableName + "'.", e);
+      }
+    }
+
+    // create table
+    try {
+      conn.tableOperations().create(tableName);
+      conn.tableOperations().addSplits(tableName, calculateSplits());
+      conn.tableOperations().setProperty(tableName, "table.split.threshold", "256M");
+    } catch (Exception e) {
+      log.error("Failed to create table '" + tableName + "'.", e);
+    }
+
+  }
+
+  @Override
+  public void client() {
+
+    Connector conn = getConnector();
+    String tableName = getTestProperty("TABLE");
+
+    // get batch writer configuration
+    long maxMemory = Long.parseLong(getTestProperty("MAX_MEMORY"));
+    long maxLatency = Long.parseLong(getTestProperty("MAX_LATENCY"));
+    int maxWriteThreads = Integer.parseInt(getTestProperty("NUM_THREADS"));
+
+    // create batch writer
+    BatchWriter bw = null;
+    try {
+      bw = conn.createBatchWriter(tableName, new BatchWriterConfig().setMaxMemory(maxMemory).setMaxLatency(maxLatency, TimeUnit.MILLISECONDS)
+          .setMaxWriteThreads(maxWriteThreads));
+    } catch (TableNotFoundException e) {
+      log.error("Table '" + tableName + "' not found.", e);
+      System.exit(-1);
+    }
+
+    // configure writing
+    Random r = new Random();
+    String ingestInstanceId = UUID.randomUUID().toString();
+    long numIngestEntries = Long.parseLong(getTestProperty("NUM_ENTRIES"));
+    long minRow = 0L;
+    long maxRow = 9223372036854775807L;
+    int maxColF = 32767;
+    int maxColQ = 32767;
+    long count = 0;
+    long totalBytes = 0;
+
+    ColumnVisibility cv = new ColumnVisibility();
+
+    // start timer
+    startTimer();
+
+    // write specified number of entries
+    while (count < numIngestEntries) {
+      count++;
+      long rowId = ContinuousIngest.genLong(minRow, maxRow, r);
+      Mutation m = ContinuousIngest.genMutation(rowId, r.nextInt(maxColF), r.nextInt(maxColQ), cv, ingestInstanceId.getBytes(UTF_8), count, null, false);
+      totalBytes += m.numBytes();
+      try {
+        bw.addMutation(m);
+      } catch (MutationsRejectedException e) {
+        log.error("Mutations rejected.", e);
+        System.exit(-1);
+      }
+    }
+
+    // close writer
+    try {
+      bw.close();
+    } catch (MutationsRejectedException e) {
+      log.error("Could not close BatchWriter due to mutations being rejected.", e);
+      System.exit(-1);
+    }
+
+    // stop timer
+    stopTimer(count, totalBytes);
+  }
+
+  @Override
+  public void teardown() {
+
+    Connector conn = getConnector();
+    String tableName = getTestProperty("TABLE");
+
+    try {
+      conn.tableOperations().delete(tableName);
+    } catch (Exception e) {
+      log.error("Failed to delete table '" + tableName + "'", e);
+    }
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/0d97273c/core/src/main/java/org/apache/accumulo/testing/core/scalability/Run.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/scalability/Run.java b/core/src/main/java/org/apache/accumulo/testing/core/scalability/Run.java
new file mode 100644
index 0000000..f7f7458
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/testing/core/scalability/Run.java
@@ -0,0 +1,97 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.testing.core.scalability;
+
+import java.io.FileInputStream;
+import java.net.InetAddress;
+import java.util.Properties;
+
+import org.apache.accumulo.core.cli.Help;
+import org.apache.accumulo.core.util.CachedConfiguration;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.beust.jcommander.Parameter;
+
+public class Run {
+
+  private static final Logger log = LoggerFactory.getLogger(Run.class);
+
+  static class Opts extends Help {
+    @Parameter(names = "--testId", required = true)
+    String testId;
+    @Parameter(names = "--action", required = true, description = "one of 'setup', 'teardown' or 'client'")
+    String action;
+    @Parameter(names = "--count", description = "number of tablet servers", required = true)
+    int numTabletServers;
+  }
+
+  public static void main(String[] args) throws Exception {
+
+    final String sitePath = "/tmp/scale-site.conf";
+    final String testPath = "/tmp/scale-test.conf";
+    Opts opts = new Opts();
+    opts.parseArgs(Run.class.getName(), args);
+
+    Configuration conf = CachedConfiguration.getInstance();
+    FileSystem fs;
+    fs = FileSystem.get(conf);
+
+    fs.copyToLocalFile(new Path("/accumulo-scale/conf/site.conf"), new Path(sitePath));
+    fs.copyToLocalFile(new Path(String.format("/accumulo-scale/conf/%s.conf", opts.testId)), new Path(testPath));
+
+    // load configuration file properties
+    Properties scaleProps = new Properties();
+    Properties testProps = new Properties();
+    try {
+      FileInputStream fis = new FileInputStream(sitePath);
+      try {
+        scaleProps.load(fis);
+      } finally {
+        fis.close();
+      }
+      fis = new FileInputStream(testPath);
+      try {
+        testProps.load(fis);
+      } finally {
+        fis.close();
+      }
+    } catch (Exception e) {
+      log.error("Error loading config file.", e);
+    }
+
+    ScaleTest test = (ScaleTest) Class.forName(String.format("org.apache.accumulo.test.scalability.%s", opts.testId)).newInstance();
+
+    test.init(scaleProps, testProps, opts.numTabletServers);
+
+    if (opts.action.equalsIgnoreCase("setup")) {
+      test.setup();
+    } else if (opts.action.equalsIgnoreCase("client")) {
+      InetAddress addr = InetAddress.getLocalHost();
+      String host = addr.getHostName();
+      fs.createNewFile(new Path("/accumulo-scale/clients/" + host));
+      test.client();
+      fs.copyFromLocalFile(new Path("/tmp/scale.out"), new Path("/accumulo-scale/results/" + host));
+    } else if (opts.action.equalsIgnoreCase("teardown")) {
+      test.teardown();
+    }
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/0d97273c/core/src/main/java/org/apache/accumulo/testing/core/scalability/ScaleTest.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/scalability/ScaleTest.java b/core/src/main/java/org/apache/accumulo/testing/core/scalability/ScaleTest.java
new file mode 100644
index 0000000..a78f39e
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/testing/core/scalability/ScaleTest.java
@@ -0,0 +1,88 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.testing.core.scalability;
+
+import java.util.Properties;
+import java.util.TreeSet;
+
+import org.apache.accumulo.core.client.AccumuloException;
+import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.core.client.ClientConfiguration;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.ZooKeeperInstance;
+import org.apache.accumulo.core.client.security.tokens.PasswordToken;
+import org.apache.hadoop.io.Text;
+
+public abstract class ScaleTest {
+
+  private Connector conn;
+  private Properties scaleProps;
+  private Properties testProps;
+  private int numTabletServers;
+  private long startTime;
+
+  public void init(Properties scaleProps, Properties testProps, int numTabletServers) throws AccumuloException, AccumuloSecurityException {
+
+    this.scaleProps = scaleProps;
+    this.testProps = testProps;
+    this.numTabletServers = numTabletServers;
+
+    // get properties to create connector
+    String instanceName = this.scaleProps.getProperty("INSTANCE_NAME");
+    String zookeepers = this.scaleProps.getProperty("ZOOKEEPERS");
+    String user = this.scaleProps.getProperty("USER");
+    String password = this.scaleProps.getProperty("PASSWORD");
+    System.out.println(password);
+
+    conn = new ZooKeeperInstance(new ClientConfiguration().withInstance(instanceName).withZkHosts(zookeepers)).getConnector(user, new PasswordToken(password));
+  }
+
+  protected void startTimer() {
+    startTime = System.currentTimeMillis();
+  }
+
+  protected void stopTimer(long numEntries, long numBytes) {
+    long endTime = System.currentTimeMillis();
+    System.out.printf("ELAPSEDMS %d %d %d%n", endTime - startTime, numEntries, numBytes);
+  }
+
+  public abstract void setup();
+
+  public abstract void client();
+
+  public abstract void teardown();
+
+  public TreeSet<Text> calculateSplits() {
+    int numSplits = numTabletServers - 1;
+    long distance = (Long.MAX_VALUE / numTabletServers) + 1;
+    long split = distance;
+    TreeSet<Text> keys = new TreeSet<>();
+    for (int i = 0; i < numSplits; i++) {
+      keys.add(new Text(String.format("%016x", split)));
+      split += distance;
+    }
+    return keys;
+  }
+
+  public Connector getConnector() {
+    return conn;
+  }
+
+  public String getTestProperty(String key) {
+    return testProps.getProperty(key);
+  }
+}

http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/0d97273c/core/src/main/java/org/apache/accumulo/testing/core/stress/DataWriter.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/stress/DataWriter.java b/core/src/main/java/org/apache/accumulo/testing/core/stress/DataWriter.java
new file mode 100644
index 0000000..e7158e2
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/testing/core/stress/DataWriter.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.test.stress.random;
+
+import org.apache.accumulo.core.client.BatchWriter;
+import org.apache.accumulo.core.client.MutationsRejectedException;
+
+public class DataWriter extends Stream<Void> {
+  private final BatchWriter writer;
+  private final RandomMutations mutations;
+
+  public DataWriter(BatchWriter writer, RandomMutations mutations) {
+    this.writer = writer;
+    this.mutations = mutations;
+  }
+
+  @Override
+  public Void next() {
+    try {
+      writer.addMutation(mutations.next());
+    } catch (MutationsRejectedException e) {
+      throw new RuntimeException(e);
+    }
+    return null;
+  }
+
+  @Override
+  public void finalize() {
+    try {
+      this.writer.close();
+    } catch (MutationsRejectedException e) {
+      System.err.println("Error closing batch writer.");
+      e.printStackTrace();
+    }
+  }
+}

http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/0d97273c/core/src/main/java/org/apache/accumulo/testing/core/stress/IntArgValidator.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/stress/IntArgValidator.java b/core/src/main/java/org/apache/accumulo/testing/core/stress/IntArgValidator.java
new file mode 100644
index 0000000..5a5ad3e
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/testing/core/stress/IntArgValidator.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.accumulo.test.stress.random;
+
+import static com.google.common.base.Preconditions.checkArgument;
+import static java.util.Objects.requireNonNull;
+
+import com.beust.jcommander.IValueValidator;
+import com.beust.jcommander.ParameterException;
+
+public class IntArgValidator implements IValueValidator<Integer> {
+
+  @Override
+  public void validate(String name, Integer value) throws ParameterException {
+    requireNonNull(value);
+    checkArgument(value > 0);
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/0d97273c/core/src/main/java/org/apache/accumulo/testing/core/stress/RandomByteArrays.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/stress/RandomByteArrays.java b/core/src/main/java/org/apache/accumulo/testing/core/stress/RandomByteArrays.java
new file mode 100644
index 0000000..405fabb
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/testing/core/stress/RandomByteArrays.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.test.stress.random;
+
+/**
+ * A stream that will create random byte arrays as it is looped over.
+ */
+public class RandomByteArrays extends Stream<byte[]> {
+  private final RandomWithinRange random_arrays;
+
+  public RandomByteArrays(RandomWithinRange random_arrays) {
+    this.random_arrays = random_arrays;
+  }
+
+  @Override
+  public byte[] next() {
+    return random_arrays.next_bytes();
+  }
+}

http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/0d97273c/core/src/main/java/org/apache/accumulo/testing/core/stress/RandomMutations.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/stress/RandomMutations.java b/core/src/main/java/org/apache/accumulo/testing/core/stress/RandomMutations.java
new file mode 100644
index 0000000..db5da55
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/testing/core/stress/RandomMutations.java
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.test.stress.random;
+
+import org.apache.accumulo.core.data.Mutation;
+
+public class RandomMutations extends Stream<Mutation> {
+  private final RandomByteArrays rows, column_families, column_qualifiers, values;
+  private final RandomWithinRange row_widths;
+  private final int max_cells_per_mutation;
+  private byte[] current_row;
+  private int cells_remaining_in_row;
+
+  public RandomMutations(RandomByteArrays rows, RandomByteArrays column_families, RandomByteArrays column_qualifiers, RandomByteArrays values,
+      RandomWithinRange row_widths, int max_cells_per_mutation) {
+    this.rows = rows;
+    this.column_families = column_families;
+    this.column_qualifiers = column_qualifiers;
+    this.values = values;
+    this.row_widths = row_widths;
+    this.max_cells_per_mutation = (max_cells_per_mutation > 0 ? max_cells_per_mutation : Integer.MAX_VALUE);
+
+    current_row = null;
+    cells_remaining_in_row = 0;
+  }
+
+  // TODO should we care about timestamps?
+  @Override
+  public Mutation next() {
+    if (cells_remaining_in_row == 0) {
+      current_row = rows.next();
+      cells_remaining_in_row = row_widths.next();
+    }
+    Mutation m = new Mutation(current_row);
+    final int cells = Math.min(cells_remaining_in_row, max_cells_per_mutation);
+    for (int i = 1; i <= cells; i++) {
+      m.put(column_families.next(), column_qualifiers.next(), values.next());
+    }
+    cells_remaining_in_row -= cells;
+    return m;
+  }
+}

http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/0d97273c/core/src/main/java/org/apache/accumulo/testing/core/stress/RandomWithinRange.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/stress/RandomWithinRange.java b/core/src/main/java/org/apache/accumulo/testing/core/stress/RandomWithinRange.java
new file mode 100644
index 0000000..06cea28
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/testing/core/stress/RandomWithinRange.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.test.stress.random;
+
+import static com.google.common.base.Preconditions.checkArgument;
+
+import java.util.Random;
+
+/**
+ * Class that returns positive integers between some minimum and maximum.
+ *
+ */
+public class RandomWithinRange {
+  private final Random random;
+  private final int min, max;
+
+  public RandomWithinRange(int seed, int min, int max) {
+    this(new Random(seed), min, max);
+  }
+
+  public RandomWithinRange(Random random, int min, int max) {
+    checkArgument(min > 0, "Min must be positive.");
+    checkArgument(max >= min, "Max must be greater than or equal to min.");
+    this.random = random;
+    this.min = min;
+    this.max = max;
+  }
+
+  public int next() {
+    if (min == max) {
+      return min;
+    } else {
+      // we pick a random number that's between 0 and (max - min), then add
+      // min as an offset to get a random number that's [min, max)
+      return random.nextInt(max - min) + min;
+    }
+  }
+
+  public byte[] next_bytes() {
+    byte[] b = new byte[next()];
+    random.nextBytes(b);
+    return b;
+  }
+}

http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/0d97273c/core/src/main/java/org/apache/accumulo/testing/core/stress/Scan.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/stress/Scan.java b/core/src/main/java/org/apache/accumulo/testing/core/stress/Scan.java
new file mode 100644
index 0000000..3e8d5fd
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/testing/core/stress/Scan.java
@@ -0,0 +1,121 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.test.stress.random;
+
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.Random;
+
+import org.apache.accumulo.core.client.AccumuloException;
+import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.client.TableNotFoundException;
+import org.apache.accumulo.core.client.admin.TableOperations;
+import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.hadoop.io.Text;
+
+import com.google.common.collect.Lists;
+
+public class Scan {
+
+  public static void main(String[] args) throws Exception {
+    ScanOpts opts = new ScanOpts();
+    opts.parseArgs(Scan.class.getName(), args);
+
+    Connector connector = opts.getConnector();
+    Scanner scanner = connector.createScanner(opts.getTableName(), new Authorizations());
+
+    if (opts.isolate) {
+      scanner.enableIsolation();
+    }
+
+    Random tablet_index_generator = new Random(opts.scan_seed);
+
+    LoopControl scanning_condition = opts.continuous ? new ContinuousLoopControl() : new IterativeLoopControl(opts.scan_iterations);
+
+    while (scanning_condition.keepScanning()) {
+      Range range = pickRange(connector.tableOperations(), opts.getTableName(), tablet_index_generator);
+      scanner.setRange(range);
+      if (opts.batch_size > 0) {
+        scanner.setBatchSize(opts.batch_size);
+      }
+      try {
+        consume(scanner);
+      } catch (Exception e) {
+        System.err.println(String.format("Exception while scanning range %s. Check the state of Accumulo for errors.", range));
+        throw e;
+      }
+    }
+  }
+
+  public static void consume(Iterable<?> iterable) {
+    Iterator<?> itr = iterable.iterator();
+    while (itr.hasNext()) {
+      itr.next();
+    }
+  }
+
+  public static Range pickRange(TableOperations tops, String table, Random r) throws TableNotFoundException, AccumuloSecurityException, AccumuloException {
+    ArrayList<Text> splits = Lists.newArrayList(tops.listSplits(table));
+    if (splits.isEmpty()) {
+      return new Range();
+    } else {
+      int index = r.nextInt(splits.size());
+      Text endRow = splits.get(index);
+      Text startRow = index == 0 ? null : splits.get(index - 1);
+      return new Range(startRow, false, endRow, true);
+    }
+  }
+
+  /*
+   * These interfaces + implementations are used to determine how many times the scanner should look up a random tablet and scan it.
+   */
+  static interface LoopControl {
+    public boolean keepScanning();
+  }
+
+  // Does a finite number of iterations
+  static class IterativeLoopControl implements LoopControl {
+    private final int max;
+    private int current;
+
+    public IterativeLoopControl(int max) {
+      this.max = max;
+      this.current = 0;
+    }
+
+    @Override
+    public boolean keepScanning() {
+      if (current < max) {
+        ++current;
+        return true;
+      } else {
+        return false;
+      }
+    }
+  }
+
+  // Does an infinite number of iterations
+  static class ContinuousLoopControl implements LoopControl {
+    @Override
+    public boolean keepScanning() {
+      return true;
+    }
+  }
+}

http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/0d97273c/core/src/main/java/org/apache/accumulo/testing/core/stress/ScanOpts.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/stress/ScanOpts.java b/core/src/main/java/org/apache/accumulo/testing/core/stress/ScanOpts.java
new file mode 100644
index 0000000..e3f73f7
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/testing/core/stress/ScanOpts.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.test.stress.random;
+
+import org.apache.accumulo.core.cli.ClientOnDefaultTable;
+
+import com.beust.jcommander.Parameter;
+
+class ScanOpts extends ClientOnDefaultTable {
+  @Parameter(names = "--isolate", description = "true to turn on scan isolation, false to turn off. default is false.")
+  boolean isolate = false;
+
+  @Parameter(names = "--num-iterations", description = "number of scan iterations")
+  int scan_iterations = 1024;
+
+  @Parameter(names = "--continuous", description = "continuously scan the table. note that this overrides --num-iterations")
+  boolean continuous;
+
+  @Parameter(names = "--scan-seed", description = "seed for randomly choosing tablets to scan")
+  int scan_seed = 1337;
+
+  @Parameter(names = "--scan-batch-size", description = "scanner batch size")
+  int batch_size = -1;
+
+  public ScanOpts() {
+    this(WriteOptions.DEFAULT_TABLE);
+  }
+
+  public ScanOpts(String table) {
+    super(table);
+  }
+}

http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/0d97273c/core/src/main/java/org/apache/accumulo/testing/core/stress/Stream.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/stress/Stream.java b/core/src/main/java/org/apache/accumulo/testing/core/stress/Stream.java
new file mode 100644
index 0000000..72b31e5
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/testing/core/stress/Stream.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.test.stress.random;
+
+import java.util.Iterator;
+
+/**
+ * Base class to model an infinite stream of data. A stream implements an iterator whose {{@link #hasNext()} method will always return true.
+ *
+ */
+public abstract class Stream<T> implements Iterator<T> {
+
+  @Override
+  public final boolean hasNext() {
+    return true;
+  }
+
+  @Override
+  public abstract T next();
+
+  @Override
+  public final void remove() {
+    throw new UnsupportedOperationException();
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/0d97273c/core/src/main/java/org/apache/accumulo/testing/core/stress/Write.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/stress/Write.java b/core/src/main/java/org/apache/accumulo/testing/core/stress/Write.java
new file mode 100644
index 0000000..ea6f164
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/testing/core/stress/Write.java
@@ -0,0 +1,77 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.test.stress.random;
+
+import org.apache.accumulo.core.cli.BatchWriterOpts;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.TableExistsException;
+import org.apache.accumulo.core.client.TableNotFoundException;
+
+public class Write {
+
+  public static void main(String[] args) throws Exception {
+    WriteOptions opts = new WriteOptions();
+    BatchWriterOpts batch_writer_opts = new BatchWriterOpts();
+    opts.parseArgs(Write.class.getName(), args, batch_writer_opts);
+
+    opts.check();
+
+    Connector c = opts.getConnector();
+
+    if (opts.clear_table && c.tableOperations().exists(opts.getTableName())) {
+      try {
+        c.tableOperations().delete(opts.getTableName());
+      } catch (TableNotFoundException e) {
+        System.err.println("Couldn't delete the table because it doesn't exist any more.");
+      }
+    }
+
+    if (!c.tableOperations().exists(opts.getTableName())) {
+      try {
+        c.tableOperations().create(opts.getTableName());
+      } catch (TableExistsException e) {
+        System.err.println("Couldn't create table ourselves, but that's ok. Continuing.");
+      }
+    }
+
+    long writeDelay = opts.write_delay;
+    if (writeDelay < 0) {
+      writeDelay = 0;
+    }
+
+    DataWriter dw = new DataWriter(c.createBatchWriter(opts.getTableName(), batch_writer_opts.getBatchWriterConfig()), new RandomMutations(
+    // rows
+        new RandomByteArrays(new RandomWithinRange(opts.row_seed, opts.rowMin(), opts.rowMax())),
+        // cfs
+        new RandomByteArrays(new RandomWithinRange(opts.cf_seed, opts.cfMin(), opts.cfMax())),
+        // cqs
+        new RandomByteArrays(new RandomWithinRange(opts.cq_seed, opts.cqMin(), opts.cqMax())),
+        // vals
+        new RandomByteArrays(new RandomWithinRange(opts.value_seed, opts.valueMin(), opts.valueMax())),
+        // number of cells per row
+        new RandomWithinRange(opts.row_width_seed, opts.rowWidthMin(), opts.rowWidthMax()),
+        // max cells per mutation
+        opts.max_cells_per_mutation));
+
+    while (true) {
+      dw.next();
+      if (writeDelay > 0) {
+        Thread.sleep(writeDelay);
+      }
+    }
+  }
+}

http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/0d97273c/core/src/main/java/org/apache/accumulo/testing/core/stress/WriteOptions.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/stress/WriteOptions.java b/core/src/main/java/org/apache/accumulo/testing/core/stress/WriteOptions.java
new file mode 100644
index 0000000..f92a9eb
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/testing/core/stress/WriteOptions.java
@@ -0,0 +1,169 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.test.stress.random;
+
+import org.apache.accumulo.core.cli.ClientOnDefaultTable;
+
+import com.beust.jcommander.Parameter;
+
+class WriteOptions extends ClientOnDefaultTable {
+  static final String DEFAULT_TABLE = "stress_test";
+  static final int DEFAULT_MIN = 1, DEFAULT_MAX = 128, DEFAULT_SPREAD = DEFAULT_MAX - DEFAULT_MIN;
+
+  @Parameter(validateValueWith = IntArgValidator.class, names = "--min-row-size", description = "minimum row size")
+  Integer row_min;
+
+  @Parameter(validateValueWith = IntArgValidator.class, names = "--max-row-size", description = "maximum row size")
+  Integer row_max;
+
+  @Parameter(validateValueWith = IntArgValidator.class, names = "--min-cf-size", description = "minimum column family size")
+  Integer cf_min;
+
+  @Parameter(validateValueWith = IntArgValidator.class, names = "--max-cf-size", description = "maximum column family size")
+  Integer cf_max;
+
+  @Parameter(validateValueWith = IntArgValidator.class, names = "--min-cq-size", description = "minimum column qualifier size")
+  Integer cq_min;
+
+  @Parameter(validateValueWith = IntArgValidator.class, names = "--max-cq-size", description = "maximum column qualifier size")
+  Integer cq_max;
+
+  @Parameter(validateValueWith = IntArgValidator.class, names = "--min-value-size", description = "minimum value size")
+  Integer value_min;
+
+  @Parameter(validateValueWith = IntArgValidator.class, names = "--max-value-size", description = "maximum value size")
+  Integer value_max;
+
+  @Parameter(validateValueWith = IntArgValidator.class, names = "--min-row-width", description = "minimum row width")
+  Integer row_width_min;
+
+  @Parameter(validateValueWith = IntArgValidator.class, names = "--max-row-width", description = "maximum row width")
+  Integer row_width_max;
+
+  @Parameter(names = "--clear-table", description = "clears the table before ingesting")
+  boolean clear_table;
+
+  @Parameter(names = "--row-seed", description = "seed for generating rows")
+  int row_seed = 87;
+
+  @Parameter(names = "--cf-seed", description = "seed for generating column families")
+  int cf_seed = 7;
+
+  @Parameter(names = "--cq-seed", description = "seed for generating column qualifiers")
+  int cq_seed = 43;
+
+  @Parameter(names = "--value-seed", description = "seed for generating values")
+  int value_seed = 99;
+
+  @Parameter(names = "--row-width-seed", description = "seed for generating the number of cells within a row (a row's \"width\")")
+  int row_width_seed = 444;
+
+  @Parameter(names = "--max-cells-per-mutation", description = "maximum number of cells per mutation; non-positive value implies no limit")
+  int max_cells_per_mutation = -1;
+
+  @Parameter(names = "--write-delay", description = "milliseconds to wait between writes")
+  long write_delay = 0L;
+
+  public WriteOptions(String table) {
+    super(table);
+  }
+
+  public WriteOptions() {
+    this(DEFAULT_TABLE);
+  }
+
+  private static int minOrDefault(Integer ref) {
+    return ref == null ? DEFAULT_MIN : ref;
+  }
+
+  private static int calculateMax(Integer min_ref, Integer max_ref) {
+    if (max_ref == null) {
+      if (min_ref == null) {
+        return DEFAULT_MAX;
+      } else {
+        return min_ref + DEFAULT_SPREAD;
+      }
+    } else {
+      return max_ref;
+    }
+  }
+
+  public void check() {
+    checkPair("ROW", row_min, row_max);
+    checkPair("COLUMN FAMILY", cf_min, cf_max);
+    checkPair("COLUMN QUALIFIER", cq_min, cq_max);
+    checkPair("VALUE", value_min, value_max);
+  }
+
+  public void checkPair(String label, Integer min_ref, Integer max_ref) {
+    // we've already asserted that the numbers will either be
+    // 1) null
+    // 2) positive
+    // need to verify that they're coherent here
+
+    if (min_ref == null && max_ref != null) {
+      // we don't support just specifying a max yet
+      throw new IllegalArgumentException(String.format("[%s] Maximum value supplied, but no minimum. Must supply a minimum with a maximum value.", label));
+    } else if (min_ref != null && max_ref != null) {
+      // if a user supplied lower and upper bounds, we need to verify
+      // that min <= max
+      if (min_ref.compareTo(max_ref) > 0) {
+        throw new IllegalArgumentException(String.format("[%s] Min value (%d) is greater than max value (%d)", label, min_ref, max_ref));
+      }
+    }
+  }
+
+  public int rowMin() {
+    return minOrDefault(row_min);
+  }
+
+  public int rowMax() {
+    return calculateMax(row_min, row_max);
+  }
+
+  public int cfMin() {
+    return minOrDefault(cf_min);
+  }
+
+  public int cfMax() {
+    return calculateMax(cf_min, cf_max);
+  }
+
+  public int cqMin() {
+    return minOrDefault(cq_min);
+  }
+
+  public int cqMax() {
+    return calculateMax(cq_min, cq_max);
+  }
+
+  public int valueMin() {
+    return minOrDefault(value_min);
+  }
+
+  public int valueMax() {
+    return calculateMax(value_min, value_max);
+  }
+
+  public int rowWidthMin() {
+    return minOrDefault(row_width_min);
+  }
+
+  public int rowWidthMax() {
+    return calculateMax(row_width_min, row_width_max);
+  }
+}

http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/0d97273c/core/src/main/java/org/apache/accumulo/testing/core/stress/package-info.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/stress/package-info.java b/core/src/main/java/org/apache/accumulo/testing/core/stress/package-info.java
new file mode 100644
index 0000000..fdbf72e
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/testing/core/stress/package-info.java
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/**
+ * This package contains utility classes designed to test Accumulo when large cells are being written. This is an attempt to observe the behavior Accumulo
+ * displays when compacting and reading these cells.
+ *
+ * There are two components to this package: {@link org.apache.accumulo.test.stress.random.Write} and {@link org.apache.accumulo.test.stress.random.Scan}.
+ *
+ * The {@link org.apache.accumulo.test.stress.random.Write} provides facilities for writing random sized cells. Users can configure minimum and maximum
+ * sized portions of a cell. The portions users can configure are the row, column family, column qualifier and value. Note that the sizes are uniformly
+ * distributed between the minimum and maximum values. See {@link org.apache.accumulo.test.stress.random.WriteOptions} for available options and default sizing
+ * information.
+ *
+ * The Scan provides users with the ability to query tables generated by the Write. It will pick a tablet at random and scan the entire range. The
+ * amount of times this process is done is user configurable. By default, it happens 1,024 times. Users can also specify whether or not the scan should be
+ * isolated or not.
+ *
+ * There is no shared state intended by either of these services. This allows multiple clients to be run in parallel, either on the same host or distributed
+ * across hosts.
+ */
+package org.apache.accumulo.test.stress.random;
+

http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/0d97273c/test/agitator/.gitignore
----------------------------------------------------------------------
diff --git a/test/agitator/.gitignore b/test/agitator/.gitignore
new file mode 100644
index 0000000..3429b01
--- /dev/null
+++ b/test/agitator/.gitignore
@@ -0,0 +1,3 @@
+*~
+*.ini
+*.pyc

http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/0d97273c/test/agitator/README.md
----------------------------------------------------------------------
diff --git a/test/agitator/README.md b/test/agitator/README.md
new file mode 100644
index 0000000..8abb74c
--- /dev/null
+++ b/test/agitator/README.md
@@ -0,0 +1,39 @@
+<!--
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements.  See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License.  You may obtain a copy of the License at 
+ 
+    http://www.apache.org/licenses/LICENSE-2.0
+ 
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+-->
+
+Agitator: randomly kill processes
+===========================
+
+The agitator is used to randomly select processes for termination during
+system test.
+
+Configure the agitator using the example agitator.ini file provided.
+
+Create a list of hosts to be agitated:
+
+	$ cp ../../../conf/tservers hosts
+	$ echo master >> hosts
+	$ echo namenode >> hosts
+
+The agitator can be used to kill and restart any part of the accumulo
+ecosystem: zookeepers, namenode, datanodes, tablet servers and master.
+You can choose to agitate them all with "--all"
+
+	$ ./agitator.py --all --hosts=hosts --config=agitator.ini --log DEBUG
+
+You will need to be able to ssh, without passwords, to all your hosts as 
+the user that can kill and start the services.

http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/0d97273c/test/agitator/agitator.ini.example
----------------------------------------------------------------------
diff --git a/test/agitator/agitator.ini.example b/test/agitator/agitator.ini.example
new file mode 100644
index 0000000..3512561
--- /dev/null
+++ b/test/agitator/agitator.ini.example
@@ -0,0 +1,56 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+[DEFAULT]
+install=%(env.pwd)s/../../../..
+user=%(env.user)s
+
+[agitator]
+kill=kill -9
+ssh=ssh -q -A -o StrictHostKeyChecking=no
+sleep=300
+sleep.restart=30
+sleep.jitter=30
+
+[accumulo]
+home=%(install)s/accumulo
+tserver.kill.min=1
+tserver.kill.max=1
+tserver.frequency=0.8
+
+master.kill.min=1
+master.kill.max=1
+master.frequency=0.1
+
+gc.kill.min=1
+gc.kill.max=1
+gc.frequency=0.1
+
+[hadoop]
+home=%(install)s/hadoop
+bin=%(home)s/bin
+datanode.frequency=0.8
+datanode.kill.min=1
+datanode.kill.max=1
+namenode.frequency=0.05
+namenode.kill.min=1
+namenode.kill.max=1
+secondarynamenode.frequency=0.05
+secondarynamenode.kill.min=1
+secondarynamenode.kill.max=1
+
+[zookeeper]
+home=%(install)s/zookeeper
+frequency=0.05

http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/0d97273c/test/agitator/agitator.py
----------------------------------------------------------------------
diff --git a/test/agitator/agitator.py b/test/agitator/agitator.py
new file mode 100755
index 0000000..db94546
--- /dev/null
+++ b/test/agitator/agitator.py
@@ -0,0 +1,241 @@
+#! /usr/bin/python
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import random
+import logging
+import ConfigParser
+
+# add the environment variables as default settings
+import os
+defaults=dict([('env.' + k, v) for k, v in os.environ.iteritems()])
+config = ConfigParser.ConfigParser(defaults)
+
+# things you can do to a particular kind of process
+class Proc:
+   program = 'Unknown'
+   _frequencyToKill = 1.0
+
+   def start(self, host):
+       pass
+
+   def find(self, host):
+       pass
+
+   def numberToKill(self):
+       return (1, 1)
+
+   def frequencyToKill(self):
+       return self._frequencyToKill
+
+   def user(self):
+       return config.get(self.program, 'user')
+
+   def kill(self, host, pid):
+      kill = config.get('agitator', 'kill').split()
+      code, stdout, stderr = self.runOn(host, kill + [pid])
+      if code != 0:
+         raise logging.warn("Unable to kill %d on %s (%s)", pid, host, stderr)
+
+   def runOn(self, host, cmd):
+      ssh = config.get('agitator', 'ssh').split()
+      return self.run(ssh + ["%s@%s" % (self.user(), host)] + cmd)
+
+   def run(self, cmd):
+      import subprocess
+      cmd = map(str, cmd)
+      logging.debug('Running %s', cmd)
+      p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+      stdout, stderr = p.communicate()
+      if stdout.strip():
+         logging.debug("%s", stdout.strip())
+      if stderr.strip():
+         logging.error("%s", stderr.strip())
+      if p.returncode != 0:
+         logging.error("Problem running %s", ' '.join(cmd))
+      return p.returncode, stdout, stderr
+
+   def __repr__(self):
+      return self.program
+
+class Zookeeper(Proc):
+   program = 'zookeeper'
+   def __init__(self):
+      self._frequencyToKill = config.getfloat(self.program, 'frequency')
+
+   def start(self, host):
+      self.runOn(host, [config.get(self.program, 'home') + '/bin/zkServer.sh start'])
+
+   def find(self, host):
+     code, stdout, stderr = self.runOn(host, ['pgrep -f [Q]uorumPeerMain || true'])
+     return map(int, [line for line in stdout.split("\n") if line])
+
+class Hadoop(Proc):
+   section = 'hadoop'
+   def __init__(self, program):
+      self.program = program
+      self._frequencyToKill = config.getfloat(self.section, program + '.frequency')
+      self.minimumToKill = config.getint(self.section, program + '.kill.min')
+      self.maximumToKill = config.getint(self.section, program + '.kill.max')
+
+   def start(self, host):
+      binDir = config.get(self.section, 'bin')
+      self.runOn(host, ['nohup %s/hdfs %s < /dev/null >/dev/null 2>&1 &' %(binDir, self.program)])
+     
+   def find(self, host):
+      code, stdout, stderr = self.runOn(host, ["pgrep -f 'proc[_]%s' || true" % (self.program,)])
+      return map(int, [line for line in stdout.split("\n") if line])
+
+   def numberToKill(self):
+      return (self.minimumToKill, self.maximumToKill)
+
+   def user(self):
+      return config.get(self.section, 'user')
+
+class Accumulo(Hadoop):
+   section = 'accumulo'
+   def start(self, host):
+      home = config.get(self.section, 'home')
+      self.runOn(host, ['nohup %s/bin/accumulo %s </dev/null >/dev/null 2>&1 & ' %(home, self.program)])
+
+   def find(self, host):
+      code, stdout, stderr = self.runOn(host, ["pgrep -f 'app[=]%s' || true" % self.program])
+      return map(int, [line for line in stdout.split("\n") if line])
+
+def fail(msg):
+   import sys
+   logging.critical(msg)
+   sys.exit(1)
+
+def jitter(n):
+   return random.random() * n - n / 2
+
+def sleep(n):
+   if n > 0:
+       logging.info("Sleeping %.2f", n)
+       import time
+       time.sleep(n)
+
+def agitate(hosts, procs):
+   starters = []
+
+   logging.info("Agitating %s on %d hosts" % (procs, len(hosts)))
+
+   section = 'agitator'
+
+   # repeatedly...
+   while True:
+      if starters:
+         # start up services that were previously killed
+         t = max(0, config.getfloat(section, 'sleep.restart') + jitter(config.getfloat(section, 'sleep.jitter')))
+         sleep(t)
+         for host, proc in starters:
+            logging.info('Starting %s on %s', proc, host)
+            proc.start(host)
+         starters = []
+
+      # wait some time
+      t = max(0, config.getfloat(section, 'sleep') + jitter(config.getfloat(section, 'sleep.jitter')))
+      sleep(t)
+
+      # for some processes
+      for p in procs:
+
+         # roll dice: should it be killed?
+         if random.random() < p.frequencyToKill():
+
+            # find them
+            from multiprocessing import Pool
+            def finder(host):
+               return host, p.find(host)
+            with Pool(5) as pool:
+               result = pool.map(finder, hosts)
+            candidates = {}
+            for host, pids in result:
+               if pids:
+                  candidates[host] = pids
+
+            # how many?
+            minKill, maxKill = p.numberToKill()
+            count = min(random.randrange(minKill, maxKill + 1), len(candidates))
+
+            # pick the victims
+            doomedHosts = random.sample(candidates.keys(), count)
+
+            # kill them
+            logging.info("Killing %s on %s", p, doomedHosts)
+            for doomedHost in doomedHosts:
+               pids = candidates[doomedHost]
+               if not pids:
+                  logging.error("Unable to kill any %s on %s: no processes of that type are running", p, doomedHost)
+               else:
+                  pid = random.choice(pids)
+                  logging.debug("Killing %s (%d) on %s", p, pid, doomedHost)
+                  p.kill(doomedHost, pid)
+                  # remember to restart them later
+                  starters.append((doomedHost, p))
+
+def main():
+   import argparse
+   parser = argparse.ArgumentParser(description='Kill random processes')
+   parser.add_argument('--log', help='set the log level', default='INFO')
+   parser.add_argument('--namenodes', help='randomly kill namenodes', action="store_true")
+   parser.add_argument('--secondary', help='randomly kill secondary namenode', action="store_true")
+   parser.add_argument('--datanodes', help='randomly kill datanodes', action="store_true")
+   parser.add_argument('--tservers', help='randomly kill tservers', action="store_true")
+   parser.add_argument('--masters', help='randomly kill masters', action="store_true")
+   parser.add_argument('--zookeepers', help='randomly kill zookeepers', action="store_true")
+   parser.add_argument('--gc', help='randomly kill the file garbage collector', action="store_true")
+   parser.add_argument('--all', 
+                       help='kill any of the tservers, masters, datanodes, namenodes or zookeepers', 
+                       action='store_true')
+   parser.add_argument('--hosts', type=argparse.FileType('r'), required=True)
+   parser.add_argument('--config', type=argparse.FileType('r'), required=True)
+   args = parser.parse_args()
+
+   config.readfp(args.config)
+
+   level = getattr(logging, args.log.upper(), None)
+   if isinstance(level, int):
+      logging.basicConfig(level=level)
+
+   procs = []
+   def addIf(flag, proc):
+       if flag or args.all:
+          procs.append(proc)
+
+   addIf(args.namenodes,  Hadoop('namenode'))
+   addIf(args.datanodes,  Hadoop('datanode'))
+   addIf(args.secondary,  Hadoop('secondarynamenode'))
+   addIf(args.tservers,   Accumulo('tserver'))
+   addIf(args.masters,    Accumulo('master'))
+   addIf(args.gc,         Accumulo('gc'))
+   addIf(args.zookeepers, Zookeeper())
+   if len(procs) == 0:
+       fail("No processes to agitate!\n")
+
+   hosts = []
+   for line in args.hosts.readlines():
+       line = line.strip()
+       if line and line[0] != '#':
+           hosts.append(line)
+   if not hosts:
+       fail('No hosts to agitate!\n')
+
+   agitate(hosts, procs)
+
+if __name__ == '__main__':
+   main()

http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/0d97273c/test/agitator/hosts.example
----------------------------------------------------------------------
diff --git a/test/agitator/hosts.example b/test/agitator/hosts.example
new file mode 100644
index 0000000..63fb8bb
--- /dev/null
+++ b/test/agitator/hosts.example
@@ -0,0 +1,16 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+localhost

http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/0d97273c/test/bench/README.md
----------------------------------------------------------------------
diff --git a/test/bench/README.md b/test/bench/README.md
new file mode 100644
index 0000000..0929bc3
--- /dev/null
+++ b/test/bench/README.md
@@ -0,0 +1,61 @@
+<!--
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements.  See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+-->
+Benchmark Tests
+===============
+
+Running the Benchmarks
+----------------------
+
+Syntax for running run.py:
+
+> `$ ./run.py [-l -v <log_level> -s <run_speed> -u <user> -p <password> -i <instance>] [Benchmark1 ... BenchmarkN]`
+
+Specifying a specific benchmark or set of benchmarks runs only those, while
+not specifying any runs all benchmarks.
+
+`-l` lists the benchmarks that will be run  
+`-v <run_speed>` can either be slow, medium or fast  
+`-s <log_level>` is a number representing the verbosity of the debugging output: 10 is debug, 20 is info, 30 is warning, etc.  
+`-u <user>` user to use when connecting with accumulo.  If not set you will be prompted to input it.  
+`-p <password>` password to use when connecting with accumulo.  If not set you will be prompted to input it.  
+`-z <zookeepers>` comma delimited lit of zookeeper host:port pairs to use when connecting with accumulo.  If not set you will be prompted to input it.  
+`-i <instance>` instance to use when connecting with accumulo.  If not set you will be prompted to input it.  
+
+The Benchmarks
+--------------
+
+Values in a 3-tuple are the slow,medium,fast speeds at which you can run the benchmarks.
+
+* CloudStone1: Test the speed at which we can check that accumulo is up and we can reach all the tservers. Lower is better.  
+* CloudStone2: Ingest 10000,100000,1000000 rows of values 50 bytes on every tserver.  Higher is better.  
+* CloudStone3: Ingest 1000,5000,10000 rows of values 1024,8192,65535 bytes on every tserver.  Higher is better.  
+* CloudStone4 (TeraSort): Ingests 10000,10000000,10000000000 rows. Lower score is better.  
+* CloudStone5: Creates 100,500,1000 tables named TestTableX and then deletes them. Lower is better.  
+* CloudStone6: Creates a table with 400, 800, 1000 splits.  Lower is better.  
+
+Terasort
+--------
+
+The 4th Benchmark is Terasort.  Run the benchmarks with speed 'slow' to do a full terasort.
+
+Misc
+----
+
+These benchmarks create tables in accumulo named 'test_ingest' and 'CloudIngestTest'.  These tables are deleted
+at the end of the benchmarks. The benchmarks will also alter user auths while it runs. It is recommended that
+a benchmark user is created.
+

http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/0d97273c/test/bench/cloudstone1/__init__.py
----------------------------------------------------------------------
diff --git a/test/bench/cloudstone1/__init__.py b/test/bench/cloudstone1/__init__.py
new file mode 100755
index 0000000..09697dc
--- /dev/null
+++ b/test/bench/cloudstone1/__init__.py
@@ -0,0 +1,15 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+

http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/0d97273c/test/bench/cloudstone1/cloudstone1.py
----------------------------------------------------------------------
diff --git a/test/bench/cloudstone1/cloudstone1.py b/test/bench/cloudstone1/cloudstone1.py
new file mode 100755
index 0000000..309ef9c
--- /dev/null
+++ b/test/bench/cloudstone1/cloudstone1.py
@@ -0,0 +1,44 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import unittest
+
+import time
+
+from lib import cloudshell
+from lib.Benchmark import Benchmark
+from lib.tservers import runAll
+from lib.path import accumulo
+
+class CloudStone1(Benchmark):
+
+    def shortDescription(self):
+        return 'Test the speed at which we can check that accumulo is up '\
+               'and we can reach all the tservers. Lower is better.'
+
+    def runTest(self):
+        code, out, err = cloudshell.run(self.username, self.password, 'table accumulo.metadata\nscan\n')
+        self.assertEqual(code, 0, "Could not scan the metadata table. %s %s" % (out, err))
+        results = runAll('echo help | %s shell -u %s -p %s' %
+                         (accumulo('bin', 'accumulo'), self.username, self.password))
+                         
+    def setSpeed(self, speed):
+        "We want to override this method but no speed can be set"
+
+def suite():
+    result = unittest.TestSuite([
+        CloudStone1(),
+        ])
+    return result

http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/0d97273c/test/bench/cloudstone2/__init__.py
----------------------------------------------------------------------
diff --git a/test/bench/cloudstone2/__init__.py b/test/bench/cloudstone2/__init__.py
new file mode 100755
index 0000000..09697dc
--- /dev/null
+++ b/test/bench/cloudstone2/__init__.py
@@ -0,0 +1,15 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+

http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/0d97273c/test/bench/cloudstone2/cloudstone2.py
----------------------------------------------------------------------
diff --git a/test/bench/cloudstone2/cloudstone2.py b/test/bench/cloudstone2/cloudstone2.py
new file mode 100755
index 0000000..996e5ef
--- /dev/null
+++ b/test/bench/cloudstone2/cloudstone2.py
@@ -0,0 +1,49 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+import unittest
+
+from lib import cloudshell
+from lib.IngestBenchmark import IngestBenchmark
+
+class CloudStone2(IngestBenchmark):
+    "TestIngest one million small records on each tserver"
+    
+    _size = 50
+    _count = 1000000
+
+    def size(self):
+        return self._size
+
+    def count(self):
+        return self._count
+    
+    def setSpeed(self, speed):
+        if speed == "fast":
+            self._size = 50
+            self._count = 10000
+        elif speed == "medium":
+            self._size = 50
+            self._count = 100000         
+        elif speed == "slow":
+            self._size = 50
+            self._count = 1000000
+
+def suite():
+    result = unittest.TestSuite([
+        CloudStone2(),
+        ])
+    return result

http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/0d97273c/test/bench/cloudstone3/__init__.py
----------------------------------------------------------------------
diff --git a/test/bench/cloudstone3/__init__.py b/test/bench/cloudstone3/__init__.py
new file mode 100755
index 0000000..09697dc
--- /dev/null
+++ b/test/bench/cloudstone3/__init__.py
@@ -0,0 +1,15 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+


Mime
View raw message