accumulo-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From mwa...@apache.org
Subject [2/7] accumulo-testing git commit: ACCUMULO-4510 Adding Randomwalk code from Accumulo
Date Tue, 03 Jan 2017 20:55:51 GMT
http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/ac5b271c/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/shard/CompactFilter.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/shard/CompactFilter.java b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/shard/CompactFilter.java
new file mode 100644
index 0000000..eacd36b
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/shard/CompactFilter.java
@@ -0,0 +1,94 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.testing.core.randomwalk.shard;
+
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map.Entry;
+import java.util.Properties;
+import java.util.Random;
+
+import org.apache.accumulo.core.client.BatchScanner;
+import org.apache.accumulo.core.client.IteratorSetting;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.iterators.user.RegExFilter;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.testing.core.randomwalk.Environment;
+import org.apache.accumulo.testing.core.randomwalk.State;
+import org.apache.accumulo.testing.core.randomwalk.Test;
+import org.apache.hadoop.io.Text;
+
+/**
+ * Test deleting documents by using a compaction filter iterator
+ */
+public class CompactFilter extends Test {
+
+  @Override
+  public void visit(State state, Environment env, Properties props) throws Exception {
+    String indexTableName = (String) state.get("indexTableName");
+    String docTableName = (String) state.get("docTableName");
+    Random rand = (Random) state.get("rand");
+
+    String deleteChar = Integer.toHexString(rand.nextInt(16)) + "";
+    String regex = "^[0-9a-f][" + deleteChar + "].*";
+
+    ArrayList<IteratorSetting> documentFilters = new ArrayList<>();
+
+    IteratorSetting is = new IteratorSetting(21, "ii", RegExFilter.class);
+    RegExFilter.setRegexs(is, regex, null, null, null, false);
+    RegExFilter.setNegate(is, true);
+    documentFilters.add(is);
+
+    long t1 = System.currentTimeMillis();
+    env.getConnector().tableOperations().compact(docTableName, null, null, documentFilters, true, true);
+    long t2 = System.currentTimeMillis();
+    long t3 = t2 - t1;
+
+    ArrayList<IteratorSetting> indexFilters = new ArrayList<>();
+
+    is = new IteratorSetting(21, RegExFilter.class);
+    RegExFilter.setRegexs(is, null, null, regex, null, false);
+    RegExFilter.setNegate(is, true);
+    indexFilters.add(is);
+
+    t1 = System.currentTimeMillis();
+    env.getConnector().tableOperations().compact(indexTableName, null, null, indexFilters, true, true);
+    t2 = System.currentTimeMillis();
+
+    log.debug("Filtered documents using compaction iterators " + regex + " " + (t3) + " " + (t2 - t1));
+
+    BatchScanner bscanner = env.getConnector().createBatchScanner(docTableName, new Authorizations(), 10);
+
+    List<Range> ranges = new ArrayList<>();
+    for (int i = 0; i < 16; i++) {
+      ranges.add(Range.prefix(new Text(Integer.toHexString(i) + "" + deleteChar)));
+    }
+
+    bscanner.setRanges(ranges);
+    Iterator<Entry<Key,Value>> iter = bscanner.iterator();
+
+    if (iter.hasNext()) {
+      throw new Exception("Saw unexpected document " + iter.next().getKey());
+    }
+
+    bscanner.close();
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/ac5b271c/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/shard/Delete.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/shard/Delete.java b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/shard/Delete.java
new file mode 100644
index 0000000..e2c8bea
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/shard/Delete.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.testing.core.randomwalk.shard;
+
+import java.util.Map.Entry;
+import java.util.Properties;
+import java.util.Random;
+
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.testing.core.randomwalk.Environment;
+import org.apache.accumulo.testing.core.randomwalk.State;
+import org.apache.accumulo.testing.core.randomwalk.Test;
+
+public class Delete extends Test {
+
+  @Override
+  public void visit(State state, Environment env, Properties props) throws Exception {
+    String indexTableName = (String) state.get("indexTableName");
+    String dataTableName = (String) state.get("docTableName");
+    int numPartitions = (Integer) state.get("numPartitions");
+    Random rand = (Random) state.get("rand");
+
+    Entry<Key,Value> entry = Search.findRandomDocument(state, env, dataTableName, rand);
+    if (entry == null)
+      return;
+
+    String docID = entry.getKey().getRow().toString();
+    String doc = entry.getValue().toString();
+
+    Insert.unindexDocument(env.getMultiTableBatchWriter().getBatchWriter(indexTableName), doc, docID, numPartitions);
+
+    Mutation m = new Mutation(docID);
+    m.putDelete("doc", "");
+
+    env.getMultiTableBatchWriter().getBatchWriter(dataTableName).addMutation(m);
+
+    log.debug("Deleted document " + docID);
+
+    env.getMultiTableBatchWriter().flush();
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/ac5b271c/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/shard/DeleteSomeDocs.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/shard/DeleteSomeDocs.java b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/shard/DeleteSomeDocs.java
new file mode 100644
index 0000000..2b790bd
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/shard/DeleteSomeDocs.java
@@ -0,0 +1,80 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.testing.core.randomwalk.shard;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Properties;
+import java.util.Random;
+
+import org.apache.accumulo.core.client.BatchDeleter;
+import org.apache.accumulo.core.client.BatchWriterConfig;
+import org.apache.accumulo.core.client.IteratorSetting;
+import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.iterators.user.RegExFilter;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.testing.core.randomwalk.Environment;
+import org.apache.accumulo.testing.core.randomwalk.State;
+import org.apache.accumulo.testing.core.randomwalk.Test;
+
+//a test created to test the batch deleter
+public class DeleteSomeDocs extends Test {
+
+  @Override
+  public void visit(State state, Environment env, Properties props) throws Exception {
+    // delete documents that where the document id matches a given pattern from doc and index table
+    // using the batch deleter
+
+    Random rand = (Random) state.get("rand");
+    String indexTableName = (String) state.get("indexTableName");
+    String dataTableName = (String) state.get("docTableName");
+
+    ArrayList<String> patterns = new ArrayList<>();
+
+    for (Object key : props.keySet())
+      if (key instanceof String && ((String) key).startsWith("pattern"))
+        patterns.add(props.getProperty((String) key));
+
+    String pattern = patterns.get(rand.nextInt(patterns.size()));
+    BatchWriterConfig bwc = new BatchWriterConfig();
+    BatchDeleter ibd = env.getConnector().createBatchDeleter(indexTableName, Authorizations.EMPTY, 8, bwc);
+    ibd.setRanges(Collections.singletonList(new Range()));
+
+    IteratorSetting iterSettings = new IteratorSetting(100, RegExFilter.class);
+    RegExFilter.setRegexs(iterSettings, null, null, pattern, null, false);
+
+    ibd.addScanIterator(iterSettings);
+
+    ibd.delete();
+
+    ibd.close();
+
+    BatchDeleter dbd = env.getConnector().createBatchDeleter(dataTableName, Authorizations.EMPTY, 8, bwc);
+    dbd.setRanges(Collections.singletonList(new Range()));
+
+    iterSettings = new IteratorSetting(100, RegExFilter.class);
+    RegExFilter.setRegexs(iterSettings, pattern, null, null, null, false);
+
+    dbd.addScanIterator(iterSettings);
+
+    dbd.delete();
+
+    dbd.close();
+
+    log.debug("Deleted documents w/ id matching '" + pattern + "'");
+  }
+}

http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/ac5b271c/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/shard/DeleteWord.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/shard/DeleteWord.java b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/shard/DeleteWord.java
new file mode 100644
index 0000000..544c35e
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/shard/DeleteWord.java
@@ -0,0 +1,97 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.testing.core.randomwalk.shard;
+
+import java.util.ArrayList;
+import java.util.Map.Entry;
+import java.util.Properties;
+import java.util.Random;
+
+import org.apache.accumulo.core.client.BatchScanner;
+import org.apache.accumulo.core.client.BatchWriter;
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.testing.core.randomwalk.Environment;
+import org.apache.accumulo.testing.core.randomwalk.State;
+import org.apache.accumulo.testing.core.randomwalk.Test;
+import org.apache.hadoop.io.Text;
+
+/**
+ * Delete all documents containing a particular word.
+ *
+ */
+
+public class DeleteWord extends Test {
+
+  @Override
+  public void visit(State state, Environment env, Properties props) throws Exception {
+    String indexTableName = (String) state.get("indexTableName");
+    String docTableName = (String) state.get("docTableName");
+    int numPartitions = (Integer) state.get("numPartitions");
+    Random rand = (Random) state.get("rand");
+
+    String wordToDelete = Insert.generateRandomWord(rand);
+
+    // use index to find all documents containing word
+    Scanner scanner = env.getConnector().createScanner(indexTableName, Authorizations.EMPTY);
+    scanner.fetchColumnFamily(new Text(wordToDelete));
+
+    ArrayList<Range> documentsToDelete = new ArrayList<>();
+
+    for (Entry<Key,Value> entry : scanner)
+      documentsToDelete.add(new Range(entry.getKey().getColumnQualifier()));
+
+    if (documentsToDelete.size() > 0) {
+      // use a batch scanner to fetch all documents
+      BatchScanner bscanner = env.getConnector().createBatchScanner(docTableName, Authorizations.EMPTY, 8);
+      bscanner.setRanges(documentsToDelete);
+
+      BatchWriter ibw = env.getMultiTableBatchWriter().getBatchWriter(indexTableName);
+      BatchWriter dbw = env.getMultiTableBatchWriter().getBatchWriter(docTableName);
+
+      int count = 0;
+
+      for (Entry<Key,Value> entry : bscanner) {
+        String docID = entry.getKey().getRow().toString();
+        String doc = entry.getValue().toString();
+
+        Insert.unindexDocument(ibw, doc, docID, numPartitions);
+
+        Mutation m = new Mutation(docID);
+        m.putDelete("doc", "");
+
+        dbw.addMutation(m);
+        count++;
+      }
+
+      bscanner.close();
+
+      env.getMultiTableBatchWriter().flush();
+
+      if (count != documentsToDelete.size()) {
+        throw new Exception("Batch scanner did not return expected number of docs " + count + " " + documentsToDelete.size());
+      }
+    }
+
+    log.debug("Deleted " + documentsToDelete.size() + " documents containing " + wordToDelete);
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/ac5b271c/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/shard/ExportIndex.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/shard/ExportIndex.java b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/shard/ExportIndex.java
new file mode 100644
index 0000000..d52198b
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/shard/ExportIndex.java
@@ -0,0 +1,118 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.testing.core.randomwalk.shard;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+
+import java.io.BufferedReader;
+import java.io.InputStreamReader;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map.Entry;
+import java.util.Properties;
+
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.util.CachedConfiguration;
+import org.apache.accumulo.testing.core.randomwalk.Environment;
+import org.apache.accumulo.testing.core.randomwalk.State;
+import org.apache.accumulo.testing.core.randomwalk.Test;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.FileUtil;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.Text;
+
+/**
+ *
+ */
+public class ExportIndex extends Test {
+
+  @Override
+  public void visit(State state, Environment env, Properties props) throws Exception {
+
+    String indexTableName = (String) state.get("indexTableName");
+    String tmpIndexTableName = indexTableName + "_tmp";
+
+    String exportDir = "/tmp/shard_export/" + indexTableName;
+    String copyDir = "/tmp/shard_export/" + tmpIndexTableName;
+
+    FileSystem fs = FileSystem.get(CachedConfiguration.getInstance());
+
+    fs.delete(new Path("/tmp/shard_export/" + indexTableName), true);
+    fs.delete(new Path("/tmp/shard_export/" + tmpIndexTableName), true);
+
+    // disable spits, so that splits can be compared later w/o worrying one table splitting and the other not
+    env.getConnector().tableOperations().setProperty(indexTableName, Property.TABLE_SPLIT_THRESHOLD.getKey(), "20G");
+
+    long t1 = System.currentTimeMillis();
+
+    env.getConnector().tableOperations().flush(indexTableName, null, null, true);
+    env.getConnector().tableOperations().offline(indexTableName);
+
+    long t2 = System.currentTimeMillis();
+
+    env.getConnector().tableOperations().exportTable(indexTableName, exportDir);
+
+    long t3 = System.currentTimeMillis();
+
+    // copy files
+    BufferedReader reader = new BufferedReader(new InputStreamReader(fs.open(new Path(exportDir, "distcp.txt")), UTF_8));
+    String file = null;
+    while ((file = reader.readLine()) != null) {
+      Path src = new Path(file);
+      Path dest = new Path(new Path(copyDir), src.getName());
+      FileUtil.copy(fs, src, fs, dest, false, true, CachedConfiguration.getInstance());
+    }
+
+    reader.close();
+
+    long t4 = System.currentTimeMillis();
+
+    env.getConnector().tableOperations().online(indexTableName);
+    env.getConnector().tableOperations().importTable(tmpIndexTableName, copyDir);
+
+    long t5 = System.currentTimeMillis();
+
+    fs.delete(new Path(exportDir), true);
+    fs.delete(new Path(copyDir), true);
+
+    HashSet<Text> splits1 = new HashSet<>(env.getConnector().tableOperations().listSplits(indexTableName));
+    HashSet<Text> splits2 = new HashSet<>(env.getConnector().tableOperations().listSplits(tmpIndexTableName));
+
+    if (!splits1.equals(splits2))
+      throw new Exception("Splits not equals " + indexTableName + " " + tmpIndexTableName);
+
+    HashMap<String,String> props1 = new HashMap<>();
+    for (Entry<String,String> entry : env.getConnector().tableOperations().getProperties(indexTableName))
+      props1.put(entry.getKey(), entry.getValue());
+
+    HashMap<String,String> props2 = new HashMap<>();
+    for (Entry<String,String> entry : env.getConnector().tableOperations().getProperties(tmpIndexTableName))
+      props2.put(entry.getKey(), entry.getValue());
+
+    if (!props1.equals(props2))
+      throw new Exception("Props not equals " + indexTableName + " " + tmpIndexTableName);
+
+    // unset the split threshold
+    env.getConnector().tableOperations().removeProperty(indexTableName, Property.TABLE_SPLIT_THRESHOLD.getKey());
+    env.getConnector().tableOperations().removeProperty(tmpIndexTableName, Property.TABLE_SPLIT_THRESHOLD.getKey());
+
+    log.debug("Imported " + tmpIndexTableName + " from " + indexTableName + " flush: " + (t2 - t1) + "ms export: " + (t3 - t2) + "ms copy:" + (t4 - t3)
+        + "ms import:" + (t5 - t4) + "ms");
+
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/ac5b271c/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/shard/Flush.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/shard/Flush.java b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/shard/Flush.java
new file mode 100644
index 0000000..e6ac574
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/shard/Flush.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.testing.core.randomwalk.shard;
+
+import java.util.Properties;
+import java.util.Random;
+
+import org.apache.accumulo.testing.core.randomwalk.Environment;
+import org.apache.accumulo.testing.core.randomwalk.State;
+import org.apache.accumulo.testing.core.randomwalk.Test;
+
+public class Flush extends Test {
+
+  @Override
+  public void visit(State state, Environment env, Properties props) throws Exception {
+    String indexTableName = (String) state.get("indexTableName");
+    String dataTableName = (String) state.get("docTableName");
+    Random rand = (Random) state.get("rand");
+
+    String table;
+
+    if (rand.nextDouble() < .5)
+      table = indexTableName;
+    else
+      table = dataTableName;
+
+    env.getConnector().tableOperations().flush(table, null, null, true);
+    log.debug("Flushed " + table);
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/ac5b271c/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/shard/Grep.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/shard/Grep.java b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/shard/Grep.java
new file mode 100644
index 0000000..5892626
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/shard/Grep.java
@@ -0,0 +1,97 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.testing.core.randomwalk.shard;
+
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.Map.Entry;
+import java.util.Properties;
+import java.util.Random;
+
+import org.apache.accumulo.core.client.BatchScanner;
+import org.apache.accumulo.core.client.IteratorSetting;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.iterators.user.IntersectingIterator;
+import org.apache.accumulo.core.iterators.user.RegExFilter;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.testing.core.randomwalk.Environment;
+import org.apache.accumulo.testing.core.randomwalk.State;
+import org.apache.accumulo.testing.core.randomwalk.Test;
+import org.apache.hadoop.io.Text;
+
+public class Grep extends Test {
+
+  @Override
+  public void visit(State state, Environment env, Properties props) throws Exception {
+    // pick a few randoms words... grep for those words and search the index
+    // ensure both return the same set of documents
+
+    String indexTableName = (String) state.get("indexTableName");
+    String dataTableName = (String) state.get("docTableName");
+    Random rand = (Random) state.get("rand");
+
+    Text words[] = new Text[rand.nextInt(4) + 2];
+
+    for (int i = 0; i < words.length; i++) {
+      words[i] = new Text(Insert.generateRandomWord(rand));
+    }
+
+    BatchScanner bs = env.getConnector().createBatchScanner(indexTableName, Authorizations.EMPTY, 16);
+    IteratorSetting ii = new IteratorSetting(20, "ii", IntersectingIterator.class.getName());
+    IntersectingIterator.setColumnFamilies(ii, words);
+    bs.addScanIterator(ii);
+    bs.setRanges(Collections.singleton(new Range()));
+
+    HashSet<Text> documentsFoundInIndex = new HashSet<>();
+
+    for (Entry<Key,Value> entry2 : bs) {
+      documentsFoundInIndex.add(entry2.getKey().getColumnQualifier());
+    }
+
+    bs.close();
+
+    bs = env.getConnector().createBatchScanner(dataTableName, Authorizations.EMPTY, 16);
+
+    for (int i = 0; i < words.length; i++) {
+      IteratorSetting more = new IteratorSetting(20 + i, "ii" + i, RegExFilter.class);
+      RegExFilter.setRegexs(more, null, null, null, "(^|(.*\\s))" + words[i] + "($|(\\s.*))", false);
+      bs.addScanIterator(more);
+    }
+
+    bs.setRanges(Collections.singleton(new Range()));
+
+    HashSet<Text> documentsFoundByGrep = new HashSet<>();
+
+    for (Entry<Key,Value> entry2 : bs) {
+      documentsFoundByGrep.add(entry2.getKey().getRow());
+    }
+
+    bs.close();
+
+    if (!documentsFoundInIndex.equals(documentsFoundByGrep)) {
+      throw new Exception("Set of documents found not equal for words " + Arrays.asList(words).toString() + " " + documentsFoundInIndex + " "
+          + documentsFoundByGrep);
+    }
+
+    log.debug("Grep and index agree " + Arrays.asList(words).toString() + " " + documentsFoundInIndex.size());
+
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/ac5b271c/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/shard/Insert.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/shard/Insert.java b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/shard/Insert.java
new file mode 100644
index 0000000..1b15323
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/shard/Insert.java
@@ -0,0 +1,136 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.testing.core.randomwalk.shard;
+
+import java.util.HashSet;
+import java.util.Properties;
+import java.util.Random;
+
+import org.apache.accumulo.core.client.AccumuloException;
+import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.core.client.BatchWriter;
+import org.apache.accumulo.core.client.TableNotFoundException;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.testing.core.randomwalk.Environment;
+import org.apache.accumulo.testing.core.randomwalk.State;
+import org.apache.accumulo.testing.core.randomwalk.Test;
+
+public class Insert extends Test {
+
+  static final int NUM_WORDS = 100000;
+  static final int MIN_WORDS_PER_DOC = 10;
+  static final int MAX_WORDS_PER_DOC = 3000;
+
+  @Override
+  public void visit(State state, Environment env, Properties props) throws Exception {
+    String indexTableName = (String) state.get("indexTableName");
+    String dataTableName = (String) state.get("docTableName");
+    int numPartitions = (Integer) state.get("numPartitions");
+    Random rand = (Random) state.get("rand");
+    long nextDocID = (Long) state.get("nextDocID");
+
+    BatchWriter dataWriter = env.getMultiTableBatchWriter().getBatchWriter(dataTableName);
+    BatchWriter indexWriter = env.getMultiTableBatchWriter().getBatchWriter(indexTableName);
+
+    String docID = insertRandomDocument(nextDocID++, dataWriter, indexWriter, indexTableName, dataTableName, numPartitions, rand);
+
+    log.debug("Inserted document " + docID);
+
+    state.set("nextDocID", Long.valueOf(nextDocID));
+  }
+
+  static String insertRandomDocument(long did, BatchWriter dataWriter, BatchWriter indexWriter, String indexTableName, String dataTableName, int numPartitions,
+      Random rand) throws TableNotFoundException, Exception, AccumuloException, AccumuloSecurityException {
+    String doc = createDocument(rand);
+
+    String docID = new StringBuilder(String.format("%016x", did)).reverse().toString();
+
+    saveDocument(dataWriter, docID, doc);
+    indexDocument(indexWriter, doc, docID, numPartitions);
+
+    return docID;
+  }
+
+  static void saveDocument(BatchWriter bw, String docID, String doc) throws Exception {
+
+    Mutation m = new Mutation(docID);
+    m.put("doc", "", doc);
+
+    bw.addMutation(m);
+  }
+
+  static String createDocument(Random rand) {
+    StringBuilder sb = new StringBuilder();
+
+    int numWords = rand.nextInt(MAX_WORDS_PER_DOC - MIN_WORDS_PER_DOC) + MIN_WORDS_PER_DOC;
+
+    for (int i = 0; i < numWords; i++) {
+      String word = generateRandomWord(rand);
+
+      if (i > 0)
+        sb.append(" ");
+
+      sb.append(word);
+    }
+
+    return sb.toString();
+  }
+
+  static String generateRandomWord(Random rand) {
+    return Integer.toString(rand.nextInt(NUM_WORDS), Character.MAX_RADIX);
+  }
+
+  static String genPartition(int partition) {
+    return String.format("%06x", Math.abs(partition));
+  }
+
+  static void indexDocument(BatchWriter bw, String doc, String docId, int numPartitions) throws Exception {
+    indexDocument(bw, doc, docId, numPartitions, false);
+  }
+
+  static void unindexDocument(BatchWriter bw, String doc, String docId, int numPartitions) throws Exception {
+    indexDocument(bw, doc, docId, numPartitions, true);
+  }
+
+  static void indexDocument(BatchWriter bw, String doc, String docId, int numPartitions, boolean delete) throws Exception {
+
+    String[] tokens = doc.split("\\W+");
+
+    String partition = genPartition(doc.hashCode() % numPartitions);
+
+    Mutation m = new Mutation(partition);
+
+    HashSet<String> tokensSeen = new HashSet<>();
+
+    for (String token : tokens) {
+      token = token.toLowerCase();
+
+      if (!tokensSeen.contains(token)) {
+        tokensSeen.add(token);
+        if (delete)
+          m.putDelete(token, docId);
+        else
+          m.put(token, docId, new Value(new byte[0]));
+      }
+    }
+
+    if (m.size() > 0)
+      bw.addMutation(m);
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/ac5b271c/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/shard/Merge.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/shard/Merge.java b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/shard/Merge.java
new file mode 100644
index 0000000..106ab3b
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/shard/Merge.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.testing.core.randomwalk.shard;
+
+import java.util.Collection;
+import java.util.Properties;
+import java.util.SortedSet;
+import java.util.TreeSet;
+
+import org.apache.accumulo.testing.core.randomwalk.Environment;
+import org.apache.accumulo.testing.core.randomwalk.State;
+import org.apache.accumulo.testing.core.randomwalk.Test;
+import org.apache.hadoop.io.Text;
+
+public class Merge extends Test {
+
+  @Override
+  public void visit(State state, Environment env, Properties props) throws Exception {
+    String indexTableName = (String) state.get("indexTableName");
+
+    Collection<Text> splits = env.getConnector().tableOperations().listSplits(indexTableName);
+    SortedSet<Text> splitSet = new TreeSet<>(splits);
+    log.debug("merging " + indexTableName);
+    env.getConnector().tableOperations().merge(indexTableName, null, null);
+    org.apache.accumulo.core.util.Merge merge = new org.apache.accumulo.core.util.Merge();
+    merge.mergomatic(env.getConnector(), indexTableName, null, null, 256 * 1024 * 1024, true);
+    splits = env.getConnector().tableOperations().listSplits(indexTableName);
+    if (splits.size() > splitSet.size()) {
+      // throw an excpetion so that test will die an no further changes to table will occur...
+      // this way table is left as is for debugging.
+      throw new Exception("There are more tablets after a merge: " + splits.size() + " was " + splitSet.size());
+    }
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/ac5b271c/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/shard/Reindex.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/shard/Reindex.java b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/shard/Reindex.java
new file mode 100644
index 0000000..ac0c872
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/shard/Reindex.java
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.testing.core.randomwalk.shard;
+
+import java.util.Map.Entry;
+import java.util.Properties;
+import java.util.Random;
+
+import org.apache.accumulo.core.client.BatchWriter;
+import org.apache.accumulo.core.client.BatchWriterConfig;
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.testing.core.randomwalk.Environment;
+import org.apache.accumulo.testing.core.randomwalk.State;
+import org.apache.accumulo.testing.core.randomwalk.Test;
+
+public class Reindex extends Test {
+
+  @Override
+  public void visit(State state, Environment env, Properties props) throws Exception {
+    String indexTableName = (String) state.get("indexTableName");
+    String tmpIndexTableName = indexTableName + "_tmp";
+    String docTableName = (String) state.get("docTableName");
+    int numPartitions = (Integer) state.get("numPartitions");
+
+    Random rand = (Random) state.get("rand");
+
+    ShardFixture.createIndexTable(this.log, state, env, "_tmp", rand);
+
+    Scanner scanner = env.getConnector().createScanner(docTableName, Authorizations.EMPTY);
+    BatchWriter tbw = env.getConnector().createBatchWriter(tmpIndexTableName, new BatchWriterConfig());
+
+    int count = 0;
+
+    for (Entry<Key,Value> entry : scanner) {
+      String docID = entry.getKey().getRow().toString();
+      String doc = entry.getValue().toString();
+
+      Insert.indexDocument(tbw, doc, docID, numPartitions);
+
+      count++;
+    }
+
+    tbw.close();
+
+    log.debug("Reindexed " + count + " documents into " + tmpIndexTableName);
+
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/ac5b271c/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/shard/Search.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/shard/Search.java b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/shard/Search.java
new file mode 100644
index 0000000..c07397d
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/shard/Search.java
@@ -0,0 +1,105 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.testing.core.randomwalk.shard;
+
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.Map.Entry;
+import java.util.Properties;
+import java.util.Random;
+
+import org.apache.accumulo.core.client.BatchScanner;
+import org.apache.accumulo.core.client.IteratorSetting;
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.iterators.user.IntersectingIterator;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.testing.core.randomwalk.Environment;
+import org.apache.accumulo.testing.core.randomwalk.State;
+import org.apache.accumulo.testing.core.randomwalk.Test;
+import org.apache.hadoop.io.Text;
+
+public class Search extends Test {
+
+  @Override
+  public void visit(State state, Environment env, Properties props) throws Exception {
+    String indexTableName = (String) state.get("indexTableName");
+    String dataTableName = (String) state.get("docTableName");
+
+    Random rand = (Random) state.get("rand");
+
+    Entry<Key,Value> entry = findRandomDocument(state, env, dataTableName, rand);
+    if (entry == null)
+      return;
+
+    Text docID = entry.getKey().getRow();
+    String doc = entry.getValue().toString();
+
+    String[] tokens = doc.split("\\W+");
+    int numSearchTerms = rand.nextInt(6);
+    if (numSearchTerms < 2)
+      numSearchTerms = 2;
+
+    HashSet<String> searchTerms = new HashSet<>();
+    while (searchTerms.size() < numSearchTerms)
+      searchTerms.add(tokens[rand.nextInt(tokens.length)]);
+
+    Text columns[] = new Text[searchTerms.size()];
+    int index = 0;
+    for (String term : searchTerms) {
+      columns[index++] = new Text(term);
+    }
+
+    log.debug("Looking up terms " + searchTerms + " expect to find " + docID);
+
+    BatchScanner bs = env.getConnector().createBatchScanner(indexTableName, Authorizations.EMPTY, 10);
+    IteratorSetting ii = new IteratorSetting(20, "ii", IntersectingIterator.class);
+    IntersectingIterator.setColumnFamilies(ii, columns);
+    bs.addScanIterator(ii);
+    bs.setRanges(Collections.singleton(new Range()));
+
+    boolean sawDocID = false;
+
+    for (Entry<Key,Value> entry2 : bs) {
+      if (entry2.getKey().getColumnQualifier().equals(docID)) {
+        sawDocID = true;
+        break;
+      }
+    }
+
+    bs.close();
+
+    if (!sawDocID)
+      throw new Exception("Did not see doc " + docID + " in index.  terms:" + searchTerms + " " + indexTableName + " " + dataTableName);
+  }
+
+  static Entry<Key,Value> findRandomDocument(State state, Environment env, String dataTableName, Random rand) throws Exception {
+    Scanner scanner = env.getConnector().createScanner(dataTableName, Authorizations.EMPTY);
+    scanner.setBatchSize(1);
+    scanner.setRange(new Range(Integer.toString(rand.nextInt(0xfffffff), 16), null));
+
+    Iterator<Entry<Key,Value>> iter = scanner.iterator();
+    if (!iter.hasNext())
+      return null;
+
+    return iter.next();
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/ac5b271c/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/shard/ShardFixture.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/shard/ShardFixture.java b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/shard/ShardFixture.java
new file mode 100644
index 0000000..e20f64d
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/shard/ShardFixture.java
@@ -0,0 +1,137 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.testing.core.randomwalk.shard;
+
+import java.net.InetAddress;
+import java.util.Random;
+import java.util.SortedSet;
+import java.util.TreeSet;
+
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.MultiTableBatchWriter;
+import org.apache.accumulo.core.client.MutationsRejectedException;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.testing.core.randomwalk.Environment;
+import org.apache.accumulo.testing.core.randomwalk.Fixture;
+import org.apache.accumulo.testing.core.randomwalk.State;
+import org.apache.hadoop.io.Text;
+import org.apache.log4j.Logger;
+
+public class ShardFixture extends Fixture {
+
+  static SortedSet<Text> genSplits(long max, int numTablets, String format) {
+
+    int numSplits = numTablets - 1;
+    long distance = max / numTablets;
+    long split = distance;
+
+    TreeSet<Text> splits = new TreeSet<>();
+
+    for (int i = 0; i < numSplits; i++) {
+      splits.add(new Text(String.format(format, split)));
+      split += distance;
+    }
+
+    return splits;
+  }
+
+  static void createIndexTable(Logger log, State state, Environment env, String suffix, Random rand) throws Exception {
+    Connector conn = env.getConnector();
+    String name = (String) state.get("indexTableName") + suffix;
+    int numPartitions = (Integer) state.get("numPartitions");
+    boolean enableCache = (Boolean) state.get("cacheIndex");
+    conn.tableOperations().create(name);
+
+    String tableId = conn.tableOperations().tableIdMap().get(name);
+    log.info("Created index table " + name + "(id:" + tableId + ")");
+
+    SortedSet<Text> splits = genSplits(numPartitions, rand.nextInt(numPartitions) + 1, "%06x");
+    conn.tableOperations().addSplits(name, splits);
+
+    log.info("Added " + splits.size() + " splits to " + name);
+
+    if (enableCache) {
+      conn.tableOperations().setProperty(name, Property.TABLE_INDEXCACHE_ENABLED.getKey(), "true");
+      conn.tableOperations().setProperty(name, Property.TABLE_BLOCKCACHE_ENABLED.getKey(), "true");
+
+      log.info("Enabled caching for table " + name);
+    }
+  }
+
+  @Override
+  public void setUp(State state, Environment env) throws Exception {
+    String hostname = InetAddress.getLocalHost().getHostName().replaceAll("[-.]", "_");
+    String pid = env.getPid();
+
+    Random rand = new Random();
+
+    int numPartitions = rand.nextInt(90) + 10;
+
+    state.set("indexTableName", String.format("ST_index_%s_%s_%d", hostname, pid, System.currentTimeMillis()));
+    state.set("docTableName", String.format("ST_docs_%s_%s_%d", hostname, pid, System.currentTimeMillis()));
+    state.set("numPartitions", Integer.valueOf(numPartitions));
+    state.set("cacheIndex", rand.nextDouble() < .5);
+    state.set("rand", rand);
+    state.set("nextDocID", Long.valueOf(0));
+
+    Connector conn = env.getConnector();
+
+    createIndexTable(this.log, state, env, "", rand);
+
+    String docTableName = (String) state.get("docTableName");
+    conn.tableOperations().create(docTableName);
+
+    String tableId = conn.tableOperations().tableIdMap().get(docTableName);
+    log.info("Created doc table " + docTableName + " (id:" + tableId + ")");
+
+    SortedSet<Text> splits = genSplits(0xff, rand.nextInt(32) + 1, "%02x");
+    conn.tableOperations().addSplits(docTableName, splits);
+
+    log.info("Added " + splits.size() + " splits to " + docTableName);
+
+    if (rand.nextDouble() < .5) {
+      conn.tableOperations().setProperty((String) state.get("docTableName"), Property.TABLE_BLOOM_ENABLED.getKey(), "true");
+      log.info("Enabled bloom filters for table " + (String) state.get("docTableName"));
+    }
+  }
+
+  @Override
+  public void tearDown(State state, Environment env) throws Exception {
+    // We have resources we need to clean up
+    if (env.isMultiTableBatchWriterInitialized()) {
+      MultiTableBatchWriter mtbw = env.getMultiTableBatchWriter();
+      try {
+        mtbw.close();
+      } catch (MutationsRejectedException e) {
+        log.error("Ignoring mutations that weren't flushed", e);
+      }
+
+      // Reset the MTBW on the state to null
+      env.resetMultiTableBatchWriter();
+    }
+
+    Connector conn = env.getConnector();
+
+    log.info("Deleting index and doc tables");
+
+    conn.tableOperations().delete((String) state.get("indexTableName"));
+    conn.tableOperations().delete((String) state.get("docTableName"));
+
+    log.debug("Exiting shard test");
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/ac5b271c/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/shard/SortTool.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/shard/SortTool.java b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/shard/SortTool.java
new file mode 100644
index 0000000..04c96f4
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/shard/SortTool.java
@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.testing.core.randomwalk.shard;
+
+import java.util.Collection;
+
+import org.apache.accumulo.core.client.mapreduce.AccumuloFileOutputFormat;
+import org.apache.accumulo.core.client.mapreduce.lib.partition.KeyRangePartitioner;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Value;
+import org.apache.hadoop.conf.Configured;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapreduce.Job;
+import org.apache.hadoop.mapreduce.lib.input.SequenceFileInputFormat;
+import org.apache.hadoop.util.Tool;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class SortTool extends Configured implements Tool {
+  protected final Logger log = LoggerFactory.getLogger(this.getClass());
+  private String outputDir;
+  private String seqFile;
+  private String splitFile;
+  private Collection<Text> splits;
+
+  public SortTool(String seqFile, String outputDir, String splitFile, Collection<Text> splits) {
+    this.outputDir = outputDir;
+    this.seqFile = seqFile;
+    this.splitFile = splitFile;
+    this.splits = splits;
+  }
+
+  @Override
+  public int run(String[] args) throws Exception {
+    Job job = Job.getInstance(getConf(), this.getClass().getSimpleName());
+    job.setJarByClass(this.getClass());
+
+    if (job.getJar() == null) {
+      log.error("M/R requires a jar file!  Run mvn package.");
+      return 1;
+    }
+
+    job.setInputFormatClass(SequenceFileInputFormat.class);
+    SequenceFileInputFormat.setInputPaths(job, seqFile);
+
+    job.setPartitionerClass(KeyRangePartitioner.class);
+    KeyRangePartitioner.setSplitFile(job, splitFile);
+
+    job.setMapOutputKeyClass(Key.class);
+    job.setMapOutputValueClass(Value.class);
+
+    job.setNumReduceTasks(splits.size() + 1);
+
+    job.setOutputFormatClass(AccumuloFileOutputFormat.class);
+    AccumuloFileOutputFormat.setOutputPath(job, new Path(outputDir));
+
+    job.waitForCompletion(true);
+    return job.isSuccessful() ? 0 : 1;
+  }
+}

http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/ac5b271c/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/shard/Split.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/shard/Split.java b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/shard/Split.java
new file mode 100644
index 0000000..bef5104
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/shard/Split.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.testing.core.randomwalk.shard;
+
+import java.util.Properties;
+import java.util.Random;
+import java.util.SortedSet;
+
+import org.apache.accumulo.testing.core.randomwalk.Environment;
+import org.apache.accumulo.testing.core.randomwalk.State;
+import org.apache.accumulo.testing.core.randomwalk.Test;
+import org.apache.hadoop.io.Text;
+
+public class Split extends Test {
+
+  @Override
+  public void visit(State state, Environment env, Properties props) throws Exception {
+    String indexTableName = (String) state.get("indexTableName");
+    int numPartitions = (Integer) state.get("numPartitions");
+    Random rand = (Random) state.get("rand");
+
+    SortedSet<Text> splitSet = ShardFixture.genSplits(numPartitions, rand.nextInt(numPartitions) + 1, "%06x");
+    log.debug("adding splits " + indexTableName);
+    env.getConnector().tableOperations().addSplits(indexTableName, splitSet);
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/ac5b271c/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/shard/VerifyIndex.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/shard/VerifyIndex.java b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/shard/VerifyIndex.java
new file mode 100644
index 0000000..caba1d7
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/shard/VerifyIndex.java
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.testing.core.randomwalk.shard;
+
+import java.util.Iterator;
+import java.util.Map.Entry;
+import java.util.Properties;
+
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.PartialKey;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.testing.core.randomwalk.Environment;
+import org.apache.accumulo.testing.core.randomwalk.State;
+import org.apache.accumulo.testing.core.randomwalk.Test;
+
+public class VerifyIndex extends Test {
+
+  @Override
+  public void visit(State state, Environment env, Properties props) throws Exception {
+
+    String indexTableName = (String) state.get("indexTableName");
+    String tmpIndexTableName = indexTableName + "_tmp";
+
+    // scan new and old index and verify identical
+    Scanner indexScanner1 = env.getConnector().createScanner(tmpIndexTableName, Authorizations.EMPTY);
+    Scanner indexScanner2 = env.getConnector().createScanner(indexTableName, Authorizations.EMPTY);
+
+    Iterator<Entry<Key,Value>> iter = indexScanner2.iterator();
+
+    int count = 0;
+
+    for (Entry<Key,Value> entry : indexScanner1) {
+      if (!iter.hasNext())
+        throw new Exception("index rebuild mismatch " + entry.getKey() + " " + indexTableName);
+
+      Key key1 = entry.getKey();
+      Key key2 = iter.next().getKey();
+
+      if (!key1.equals(key2, PartialKey.ROW_COLFAM_COLQUAL))
+        throw new Exception("index rebuild mismatch " + key1 + " " + key2 + " " + indexTableName + " " + tmpIndexTableName);
+      count++;
+      if (count % 1000 == 0)
+        makingProgress();
+    }
+
+    if (iter.hasNext())
+      throw new Exception("index rebuild mismatch " + iter.next().getKey() + " " + tmpIndexTableName);
+
+    log.debug("Verified " + count + " index entries ");
+
+    env.getConnector().tableOperations().delete(indexTableName);
+    env.getConnector().tableOperations().rename(tmpIndexTableName, indexTableName);
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/ac5b271c/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/unit/CreateTable.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/unit/CreateTable.java b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/unit/CreateTable.java
new file mode 100644
index 0000000..df1df59
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/unit/CreateTable.java
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.accumulo.testing.core.randomwalk.unit;
+
+import java.util.Properties;
+
+import org.apache.accumulo.testing.core.randomwalk.Environment;
+import org.apache.accumulo.testing.core.randomwalk.State;
+import org.apache.accumulo.testing.core.randomwalk.Test;
+
+public class CreateTable extends Test {
+
+  @Override
+  public void visit(State state, Environment env, Properties props) throws Exception {}
+}

http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/ac5b271c/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/unit/DeleteTable.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/unit/DeleteTable.java b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/unit/DeleteTable.java
new file mode 100644
index 0000000..f7226cb
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/unit/DeleteTable.java
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.testing.core.randomwalk.unit;
+
+import java.util.Properties;
+
+import org.apache.accumulo.testing.core.randomwalk.Environment;
+import org.apache.accumulo.testing.core.randomwalk.State;
+import org.apache.accumulo.testing.core.randomwalk.Test;
+
+public class DeleteTable extends Test {
+
+  @Override
+  public void visit(State state, Environment env, Properties props) throws Exception {}
+}

http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/ac5b271c/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/unit/Ingest.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/unit/Ingest.java b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/unit/Ingest.java
new file mode 100644
index 0000000..5681402
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/unit/Ingest.java
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.testing.core.randomwalk.unit;
+
+import java.util.Properties;
+
+import org.apache.accumulo.testing.core.randomwalk.Environment;
+import org.apache.accumulo.testing.core.randomwalk.State;
+import org.apache.accumulo.testing.core.randomwalk.Test;
+
+public class Ingest extends Test {
+
+  @Override
+  public void visit(State state, Environment env, Properties props) throws Exception {}
+}

http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/ac5b271c/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/unit/Scan.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/unit/Scan.java b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/unit/Scan.java
new file mode 100644
index 0000000..c677cf9
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/unit/Scan.java
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.testing.core.randomwalk.unit;
+
+import java.util.Properties;
+
+import org.apache.accumulo.testing.core.randomwalk.Environment;
+import org.apache.accumulo.testing.core.randomwalk.State;
+import org.apache.accumulo.testing.core.randomwalk.Test;
+
+public class Scan extends Test {
+
+  @Override
+  public void visit(State state, Environment env, Properties props) throws Exception {}
+}

http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/ac5b271c/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/unit/Verify.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/unit/Verify.java b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/unit/Verify.java
new file mode 100644
index 0000000..95acf4f
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/testing/core/randomwalk/unit/Verify.java
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.testing.core.randomwalk.unit;
+
+import java.util.Properties;
+
+import org.apache.accumulo.testing.core.randomwalk.Environment;
+import org.apache.accumulo.testing.core.randomwalk.State;
+import org.apache.accumulo.testing.core.randomwalk.Test;
+
+public class Verify extends Test {
+
+  @Override
+  public void visit(State state, Environment env, Properties props) throws Exception {}
+}

http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/ac5b271c/core/src/main/resources/randomwalk/module.xsd
----------------------------------------------------------------------
diff --git a/core/src/main/resources/randomwalk/module.xsd b/core/src/main/resources/randomwalk/module.xsd
new file mode 100644
index 0000000..bcdaaae
--- /dev/null
+++ b/core/src/main/resources/randomwalk/module.xsd
@@ -0,0 +1,69 @@
+<xsd:schema xmlns:xsd="http://www.w3.org/2001/XMLSchema">
+<!--
+  Licensed to the Apache Software Foundation (ASF) under one or more
+  contributor license agreements.  See the NOTICE file distributed with
+  this work for additional information regarding copyright ownership.
+  The ASF licenses this file to You under the Apache License, Version 2.0
+  (the "License"); you may not use this file except in compliance with
+  the License.  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+-->
+
+  <xsd:element name="module" type="ModuleType"/>
+
+  <xsd:complexType name="ModuleType">
+    <xsd:sequence>
+      <xsd:element name="package" type="PrefixType" minOccurs="0" maxOccurs="unbounded"/>
+      <xsd:element name="fixture" type="InitType" minOccurs="0" maxOccurs="1"/>
+      <xsd:element name="init" type="InitType"/>
+      <xsd:element name="node" type="NodeType" minOccurs="1" maxOccurs="unbounded"/>
+   </xsd:sequence>
+  </xsd:complexType>
+
+  <xsd:complexType name="PrefixType">
+    <xsd:attribute name="prefix" type="xsd:string"/>
+    <xsd:attribute name="value" type="xsd:string"/>
+  </xsd:complexType>
+
+  <xsd:complexType name="InitType">
+    <xsd:attribute name="id" type="xsd:string"/>
+    <xsd:attribute name="maxHops" type="xsd:nonNegativeInteger"/>
+    <xsd:attribute name="maxSec" type="xsd:nonNegativeInteger"/>
+    <xsd:attribute name="teardown" type="xsd:boolean"/>
+  </xsd:complexType>
+
+  <xsd:complexType name="NodeType">
+    <xsd:sequence>
+      <xsd:element name="alias" type="AliasType" minOccurs="0" maxOccurs="unbounded"/>
+      <xsd:element name="property" type="PropertyType" minOccurs="0" maxOccurs="unbounded"/>
+      <xsd:element name="edge" type="EdgeType" minOccurs="1" maxOccurs="unbounded"/>
+    </xsd:sequence>
+    <xsd:attribute name="id" type="xsd:string"/>
+    <xsd:attribute name="src" type="xsd:string"/>
+    <xsd:attribute name="maxHops" type="xsd:nonNegativeInteger"/>
+    <xsd:attribute name="maxSec" type="xsd:nonNegativeInteger"/>
+    <xsd:attribute name="teardown" type="xsd:boolean"/>
+  </xsd:complexType>
+
+  <xsd:complexType name="EdgeType">
+    <xsd:attribute name="id" type="xsd:string"/>
+    <xsd:attribute name="weight" type="xsd:positiveInteger"/>
+  </xsd:complexType>
+
+  <xsd:complexType name="AliasType">
+    <xsd:attribute name="name" type="xsd:string"/>
+  </xsd:complexType>
+  
+  <xsd:complexType name="PropertyType">
+    <xsd:attribute name="key" type="xsd:string"/>
+    <xsd:attribute name="value" type="xsd:string"/>
+  </xsd:complexType>
+
+</xsd:schema>

http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/ac5b271c/core/src/main/resources/randomwalk/modules/All.xml
----------------------------------------------------------------------
diff --git a/core/src/main/resources/randomwalk/modules/All.xml b/core/src/main/resources/randomwalk/modules/All.xml
new file mode 100644
index 0000000..be5c815
--- /dev/null
+++ b/core/src/main/resources/randomwalk/modules/All.xml
@@ -0,0 +1,65 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  Licensed to the Apache Software Foundation (ASF) under one or more
+  contributor license agreements.  See the NOTICE file distributed with
+  this work for additional information regarding copyright ownership.
+  The ASF licenses this file to You under the Apache License, Version 2.0
+  (the "License"); you may not use this file except in compliance with
+  the License.  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+-->
+<module>
+
+<init id="dummy.ToAll"/>
+
+<node id="Image.xml">
+  <edge id="dummy.ToAll" weight="1"/>
+</node>
+
+<node id="dummy.ToAll">
+  <edge id="Image.xml" weight="1"/>
+  <edge id="Sequential.xml" weight="1"/>
+  <edge id="MultiTable.xml" weight="1"/>
+  <edge id="Shard.xml" weight="1"/>
+  <edge id="Concurrent.xml" weight="1"/>
+  <edge id="Conditional.xml" weight="1"/>
+  <edge id="Security.xml" weight="1"/>
+  <edge id="Bulk.xml" weight="1"/>
+</node>
+
+<node id="Sequential.xml">
+  <edge id="dummy.ToAll" weight="1"/>
+</node>
+
+<node id="MultiTable.xml">
+  <edge id="dummy.ToAll" weight="1"/>
+</node>
+
+<node id="Shard.xml">
+  <edge id="dummy.ToAll" weight="1"/>
+</node>
+
+<node id="Concurrent.xml">
+  <edge id="dummy.ToAll" weight="1"/>
+</node>
+
+<node id="Conditional.xml">
+  <edge id="dummy.ToAll" weight="1"/>
+</node>
+
+<node id="Security.xml">
+  <edge id="dummy.ToAll" weight="1"/>
+</node>
+
+<node id="Bulk.xml">
+  <edge id="dummy.ToAll" weight="1"/>
+</node>
+
+</module>

http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/ac5b271c/core/src/main/resources/randomwalk/modules/Bulk.xml
----------------------------------------------------------------------
diff --git a/core/src/main/resources/randomwalk/modules/Bulk.xml b/core/src/main/resources/randomwalk/modules/Bulk.xml
new file mode 100644
index 0000000..35e2a67
--- /dev/null
+++ b/core/src/main/resources/randomwalk/modules/Bulk.xml
@@ -0,0 +1,61 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  Licensed to the Apache Software Foundation (ASF) under one or more
+  contributor license agreements.  See the NOTICE file distributed with
+  this work for additional information regarding copyright ownership.
+  The ASF licenses this file to You under the Apache License, Version 2.0
+  (the "License"); you may not use this file except in compliance with
+  the License.  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+-->
+<module>
+
+<package prefix="bulk" value="org.apache.accumulo.test.randomwalk.bulk"/>
+
+<init id="bulk.Setup"/>
+
+<node id="bulk.Setup">
+  <edge id="bulk.BulkPlusOne" weight="1"/>
+</node>
+
+<node id="bulk.BulkPlusOne">
+  <edge id="bulk.Merge" weight="10"/>
+  <edge id="bulk.Split" weight="10"/>
+  <edge id="bulk.Compact" weight="10"/>
+  <edge id="bulk.ConsistencyCheck" weight="25"/>
+  <edge id="bulk.BulkMinusOne" weight="10"/>
+</node>
+
+<node id="bulk.BulkMinusOne">
+  <edge id="bulk.BulkPlusOne" weight="200"/>
+  <edge id="bulk.Verify" weight="1"/>
+</node>
+
+<node id="bulk.Merge">
+  <edge id="bulk.BulkMinusOne" weight="1"/>
+</node>
+
+<node id="bulk.Split">
+  <edge id="bulk.BulkMinusOne" weight="1"/>
+</node>
+
+<node id="bulk.Compact">
+  <edge id="bulk.BulkMinusOne" weight="1"/>
+</node>
+
+<node id="bulk.ConsistencyCheck">
+  <edge id="bulk.BulkMinusOne" weight="1"/>
+</node>
+
+<node id="bulk.Verify">
+  <edge id="END" weight="1"/>
+</node>
+
+</module>

http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/ac5b271c/core/src/main/resources/randomwalk/modules/Concurrent.xml
----------------------------------------------------------------------
diff --git a/core/src/main/resources/randomwalk/modules/Concurrent.xml b/core/src/main/resources/randomwalk/modules/Concurrent.xml
new file mode 100644
index 0000000..36ea53c
--- /dev/null
+++ b/core/src/main/resources/randomwalk/modules/Concurrent.xml
@@ -0,0 +1,181 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  Licensed to the Apache Software Foundation (ASF) under one or more
+  contributor license agreements.  See the NOTICE file distributed with
+  this work for additional information regarding copyright ownership.
+  The ASF licenses this file to You under the Apache License, Version 2.0
+  (the "License"); you may not use this file except in compliance with
+  the License.  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+-->
+<module>
+
+<package prefix="ct" value="org.apache.accumulo.test.randomwalk.concurrent"/>
+
+<fixture id="ct.ConcurrentFixture"/>
+
+<init id="ct.Setup"/>
+
+<node id="dummy.ToAll">
+  <edge id="ct.CreateTable" weight="1000"/>
+  <edge id="ct.BatchWrite" weight="1000"/>
+  <edge id="ct.BatchScan" weight="1000"/>
+  <edge id="ct.CloneTable" weight="1000"/>
+  <edge id="ct.DeleteTable" weight="1000"/>
+  <edge id="ct.RenameTable" weight="1000"/>
+  <edge id="ct.ScanTable" weight="500"/>
+  <edge id="ct.Replication" weight="500"/>
+  <edge id="ct.IsolatedScan" weight="500"/>
+  <edge id="ct.AddSplits" weight="1000"/>
+  <edge id="ct.ListSplits" weight="1000"/>
+  <edge id="ct.OfflineTable" weight="1000"/>
+  <edge id="ct.Merge" weight="1000"/>
+  <edge id="ct.Compact" weight="1000"/>
+  <edge id="ct.BulkImport" weight="1000"/>
+  <edge id="ct.DeleteRange" weight="1000"/>
+  <edge id="ct.CreateUser" weight="1000"/>
+  <edge id="ct.DropUser" weight="1000"/>
+  <edge id="ct.ChangeAuthorizations" weight="1000"/>
+  <edge id="ct.ChangePermissions" weight="1000"/>
+  <edge id="ct.CheckPermission" weight="1000"/>
+  <edge id="ct.StopTabletServer" weight="100"/>
+  <edge id="ct.StartAll" weight="1000"/>
+  <edge id="ct.Shutdown" weight="10"/>
+  <edge id="ct.Config" weight="1000"/>
+  <edge id="ct.CreateNamespace" weight="1000"/>
+  <edge id="ct.DeleteNamespace" weight="100"/>
+  <edge id="ct.RenameNamespace" weight="100"/>
+  <edge id="ct.Apocalypse" weight="10"/>
+  <edge id="END" weight="1"/>
+</node>
+
+<node id="ct.Setup">
+  <property key="numTables" value="9"/>
+  <property key="numNamespaces" value="2"/>
+  <edge id="ct.CreateTable" weight="1"/>
+</node>
+
+<node id="ct.BatchScan">
+  <edge id="dummy.ToAll" weight="1"/>
+</node>
+
+<node id="ct.BatchWrite">
+  <edge id="dummy.ToAll" weight="1"/>
+</node>
+
+<node id="ct.CloneTable">
+  <edge id="dummy.ToAll" weight="1"/>
+</node>
+
+<node id="ct.CreateTable">
+  <edge id="dummy.ToAll" weight="1"/>
+</node>
+
+<node id="ct.DeleteTable">
+  <edge id="dummy.ToAll" weight="1"/>
+</node>
+ 
+<node id="ct.RenameTable">
+  <edge id="dummy.ToAll" weight="1"/>
+</node>
+
+<node id="ct.ScanTable">
+  <edge id="dummy.ToAll" weight="1"/>
+</node>
+
+<node id="ct.Replication">
+  <edge id="dummy.ToAll" weight="1"/>
+</node>
+
+<node id="ct.IsolatedScan">
+  <edge id="dummy.ToAll" weight="1"/>
+</node>
+
+<node id="ct.AddSplits">
+  <edge id="dummy.ToAll" weight="1"/>
+</node>
+
+<node id="ct.ListSplits">
+  <edge id="dummy.ToAll" weight="1"/>
+</node>
+
+<node id="ct.OfflineTable">
+  <edge id="dummy.ToAll" weight="1"/>
+</node>
+
+<node id="ct.Merge">
+  <edge id="dummy.ToAll" weight="1"/>
+</node>
+
+<node id="ct.Compact">
+  <edge id="dummy.ToAll" weight="1"/>
+</node>
+
+<node id="ct.BulkImport">
+  <edge id="dummy.ToAll" weight="1"/>
+</node>
+
+<node id="ct.DeleteRange">
+  <edge id="dummy.ToAll" weight="1"/>
+</node>
+
+<node id="ct.CreateUser">
+  <edge id="dummy.ToAll" weight="1"/>
+</node>
+
+<node id="ct.DropUser">
+  <edge id="dummy.ToAll" weight="1"/>
+</node>
+
+<node id="ct.ChangeAuthorizations">
+  <edge id="dummy.ToAll" weight="1"/>
+</node>
+
+<node id="ct.ChangePermissions">
+  <edge id="dummy.ToAll" weight="1"/>
+</node>
+
+<node id="ct.CheckPermission">
+  <edge id="dummy.ToAll" weight="1"/>
+</node>
+
+<node id="ct.StopTabletServer">
+  <edge id="dummy.ToAll" weight="1"/>
+</node>
+
+<node id="ct.StartAll">
+  <edge id="dummy.ToAll" weight="1"/>
+</node>
+
+<node id="ct.Config">
+  <edge id="dummy.ToAll" weight="1"/>
+</node>
+
+<node id="ct.Shutdown">
+  <edge id="ct.StartAll" weight="1"/>
+</node>
+
+<node id="ct.Apocalypse">
+  <edge id="ct.StartAll" weight="1"/>
+</node>
+
+<node id="ct.CreateNamespace">
+  <edge id="ct.StartAll" weight="1"/>
+</node>
+
+<node id="ct.DeleteNamespace">
+  <edge id="ct.StartAll" weight="1"/>
+</node>
+
+<node id="ct.RenameNamespace">
+  <edge id="ct.StartAll" weight="1"/>
+</node>
+
+</module>

http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/ac5b271c/core/src/main/resources/randomwalk/modules/Conditional.xml
----------------------------------------------------------------------
diff --git a/core/src/main/resources/randomwalk/modules/Conditional.xml b/core/src/main/resources/randomwalk/modules/Conditional.xml
new file mode 100644
index 0000000..54ff7ab
--- /dev/null
+++ b/core/src/main/resources/randomwalk/modules/Conditional.xml
@@ -0,0 +1,74 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  Licensed to the Apache Software Foundation (ASF) under one or more
+  contributor license agreements.  See the NOTICE file distributed with
+  this work for additional information regarding copyright ownership.
+  The ASF licenses this file to You under the Apache License, Version 2.0
+  (the "License"); you may not use this file except in compliance with
+  the License.  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+-->
+<module>
+
+<package prefix="ct" value="org.apache.accumulo.test.randomwalk.conditional"/>
+
+
+<init id="ct.Setup"/>
+
+<node id="dummy.ToAll">
+  <edge id="ct.Compact" weight="1"/>
+  <edge id="ct.Flush" weight="1"/>
+  <edge id="ct.Merge" weight="1"/>
+  <edge id="ct.Split" weight="1"/>
+  <edge id="ct.Transfer" weight="100000"/>
+  <edge id="ct.Verify" weight="2"/>
+</node>
+
+<node id="ct.Setup">
+  <property key="numAccts" value="10000"/>
+  <property key="numBanks" value="1000"/>
+  <edge id="ct.Init" weight="1"/>
+</node>
+
+<node id="ct.Init">
+  <edge id="dummy.ToAll" weight="1"/>
+</node>
+
+<node id="ct.Compact">
+  <edge id="dummy.ToAll" weight="1"/>
+</node>
+
+<node id="ct.Flush">
+  <edge id="dummy.ToAll" weight="1"/>
+</node>
+
+<node id="ct.Merge">
+  <edge id="dummy.ToAll" weight="1"/>
+</node>
+
+<node id="ct.Split">
+  <edge id="dummy.ToAll" weight="1"/>
+</node>
+ 
+<node id="ct.Transfer">
+  <edge id="dummy.ToAll" weight="1"/>
+</node>
+
+<node id="ct.Verify">
+  <edge id="dummy.ToAll" weight="1000"/>
+  <edge id="ct.TearDown" weight="1"/>
+</node>
+
+
+<node id="ct.TearDown">
+  <edge id="END" weight="1"/>
+</node>
+
+</module>

http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/ac5b271c/core/src/main/resources/randomwalk/modules/Image.xml
----------------------------------------------------------------------
diff --git a/core/src/main/resources/randomwalk/modules/Image.xml b/core/src/main/resources/randomwalk/modules/Image.xml
new file mode 100644
index 0000000..7561895
--- /dev/null
+++ b/core/src/main/resources/randomwalk/modules/Image.xml
@@ -0,0 +1,70 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  Licensed to the Apache Software Foundation (ASF) under one or more
+  contributor license agreements.  See the NOTICE file distributed with
+  this work for additional information regarding copyright ownership.
+  The ASF licenses this file to You under the Apache License, Version 2.0
+  (the "License"); you may not use this file except in compliance with
+  the License.  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+-->
+<module>
+
+<package prefix="image" value="org.apache.accumulo.test.randomwalk.image"/>
+
+<fixture id="image.ImageFixture"/>
+
+<init id="BigWrite"/>
+
+<node id="BigWrite" src="image.Write">
+  <property key="minSize" value="400000"/>
+  <property key="maxSize" value="1000000"/>
+  <edge id="BigWrite" weight="167"/>
+  <edge id="SmallWrite" weight="833"/>
+  <edge id="image.Commit" weight="1"/>
+</node>
+
+<node id="SmallWrite" src="image.Write">
+  <property key="minSize" value="4000"/>
+  <property key="maxSize" value="10000"/>
+  <edge id="BigWrite" weight="167"/>
+  <edge id="SmallWrite" weight="833"/>
+  <edge id="image.Commit" weight="1"/>
+</node>
+
+<node id="image.Commit">
+  <edge id="dummy.ToAll" weight="1"/>
+</node>
+
+<node id="dummy.ToAll">
+  <edge id="SmallWrite" weight="300"/>
+  <edge id="BigWrite" weight="300"/>
+  <edge id="image.Verify" weight="150"/>
+  <edge id="image.ScanMeta" weight="150"/>
+  <edge id="image.TableOp" weight="30"/>
+  <edge id="END" weight="1"/>
+</node>
+
+<node id="image.Verify">
+  <property key="maxVerify" value="500"/>
+  <edge id="dummy.ToAll" weight="1"/>
+</node>
+
+<node id="image.ScanMeta">
+  <property key="minScan" value="100"/>
+  <property key="maxScan" value="500"/>
+  <edge id="dummy.ToAll" weight="1"/>
+</node>
+
+<node id="image.TableOp">
+  <edge id="dummy.ToAll" weight="1"/>
+</node>
+
+</module>

http://git-wip-us.apache.org/repos/asf/accumulo-testing/blob/ac5b271c/core/src/main/resources/randomwalk/modules/LongClean.xml
----------------------------------------------------------------------
diff --git a/core/src/main/resources/randomwalk/modules/LongClean.xml b/core/src/main/resources/randomwalk/modules/LongClean.xml
new file mode 100644
index 0000000..217afb0
--- /dev/null
+++ b/core/src/main/resources/randomwalk/modules/LongClean.xml
@@ -0,0 +1,60 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  Licensed to the Apache Software Foundation (ASF) under one or more
+  contributor license agreements.  See the NOTICE file distributed with
+  this work for additional information regarding copyright ownership.
+  The ASF licenses this file to You under the Apache License, Version 2.0
+  (the "License"); you may not use this file except in compliance with
+  the License.  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+-->
+<module>
+
+<init id="dummy.ToAll"/>
+
+<node id="Image.xml" maxSec="3600" teardown="true">
+  <edge id="dummy.ToAll" weight="1"/>
+</node>
+
+<node id="dummy.ToAll">
+  <edge id="Image.xml" weight="1"/>
+  <edge id="Sequential.xml" weight="1"/>
+  <edge id="MultiTable.xml" weight="1"/>
+  <edge id="Shard.xml" weight="1"/>
+  <edge id="Concurrent.xml" weight="1"/>
+  <edge id="Security.xml" weight="1"/>
+  <edge id="Bulk.xml" weight="1"/>
+</node>
+
+<node id="Sequential.xml" maxSec="3600" teardown="true">
+  <edge id="dummy.ToAll" weight="1"/>
+</node>
+
+<node id="MultiTable.xml" maxSec="3600" teardown="true"> 
+  <edge id="dummy.ToAll" weight="1"/>
+</node>
+
+<node id="Shard.xml" maxSec="3600" teardown="true">
+  <edge id="dummy.ToAll" weight="1"/>
+</node>
+
+<node id="Concurrent.xml" maxSec="3600" teardown="true">
+  <edge id="dummy.ToAll" weight="1"/>
+</node>
+
+<node id="Security.xml" maxSec="3600" teardown="true">
+  <edge id="dummy.ToAll" weight="1"/>
+</node>
+
+<node id="Bulk.xml" maxSec="3600" teardown="true">
+  <edge id="dummy.ToAll" weight="1"/>
+</node>
+
+</module>


Mime
View raw message