accumulo-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ktur...@apache.org
Subject [accumulo-testing] branch master updated: Created performance test framework (#21)
Date Mon, 23 Jul 2018 21:34:50 GMT
This is an automated email from the ASF dual-hosted git repository.

kturner pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/accumulo-testing.git


The following commit(s) were added to refs/heads/master by this push:
     new 930ca61  Created performance test framework (#21)
930ca61 is described below

commit 930ca61facf40481edd5ebd5e5e997b374061435
Author: Keith Turner <keith@deenlo.com>
AuthorDate: Mon Jul 23 17:34:48 2018 -0400

    Created performance test framework (#21)
---
 README.md                                          |  74 ++++++++-
 bin/performance-test                               |  97 +++++++++++
 conf/.gitignore                                    |   1 +
 conf/cluster-control.sh.uno                        |  86 ++++++++++
 core/pom.xml                                       |   4 +
 .../testing/core/performance/Environment.java      |  24 +++
 .../testing/core/performance/Parameter.java        |  31 ++++
 .../testing/core/performance/PerformanceTest.java  |  25 +++
 .../accumulo/testing/core/performance/Report.java  | 109 +++++++++++++
 .../accumulo/testing/core/performance/Result.java  |  54 +++++++
 .../accumulo/testing/core/performance/Stats.java   |  34 ++++
 .../core/performance/SystemConfiguration.java      |  37 +++++
 .../testing/core/performance/impl/Compare.java     | 117 ++++++++++++++
 .../core/performance/impl/ContextualReport.java    |  39 +++++
 .../testing/core/performance/impl/Csv.java         | 129 +++++++++++++++
 .../testing/core/performance/impl/ListTests.java   |  37 +++++
 .../core/performance/impl/MergeSiteConfig.java     |  52 ++++++
 .../core/performance/impl/PerfTestRunner.java      |  70 ++++++++
 .../core/performance/tests/ScanExecutorPT.java     | 177 +++++++++++++++++++++
 .../core/performance/tests/ScanFewFamiliesPT.java  | 114 +++++++++++++
 .../testing/core/performance/util/TestData.java    |  62 ++++++++
 .../core/performance/util/TestExecutor.java        |  59 +++++++
 22 files changed, 1431 insertions(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 8826cc3..c1e8d91 100644
--- a/README.md
+++ b/README.md
@@ -53,7 +53,7 @@ The YARN application can be killed at any time using the YARN resource manager o
 ## Continuous Ingest & Query
 
 The Continuous Ingest test runs many ingest clients that continually create linked lists of data
-in Accumulo. During ingest, query applications can be run to continously walk and verify the the
+in Accumulo. During ingest, query applications can be run to continuously walk and verify the
 linked lists and put a query load on Accumulo. At some point, the ingest clients are stopped and
 a MapReduce job is run to ensure that there are no holes in any linked list.
 
@@ -135,6 +135,78 @@ Run the command below stop the agitator:
 
             ./bin/accumulo-testing agitator stop
 
+## Performance Test
+
+To run performance test a `cluster-control.sh` script is needed to assist with starting, stopping,
+wiping, and confguring an Accumulo instance.  This script should define the following functions.
+
+```bash
+function get_version {
+  case $1 in
+    ACCUMULO)
+      # TODO echo accumulo version
+      ;;
+    HADOOP)
+      # TODO echo hadoop version
+      ;;
+    ZOOKEEPER)
+      # TODO echo zookeeper version
+      ;;
+    *)
+      return 1
+  esac
+}
+
+function start_cluster {
+  # TODO start Hadoop and Zookeeper if needed
+}
+
+function setup_accumulo {
+  # TODO kill any running Accumulo instance
+  # TODO setup a fresh install of Accumulo w/o starting it
+}
+
+function get_config_file {
+  local file_to_get=$1
+  local dest_dir=$2
+  # TODO copy $file_to_get from Accumulo conf dir to $dest_dir
+}
+
+function put_config_file {
+  local config_file=$1
+  # TODO copy $config_file to Accumulo conf dir
+}
+
+function put_server_code {
+  local jar_file=$1
+  # TODO add $jar_file to Accumulo's server side classpath.  Could put it in $ACCUMULO_HOME/lib/ext
+}
+
+function start_accumulo {
+  # TODO start accumulo
+}
+
+function stop_cluster {
+  # TODO kill Accumulo, Hadoop, and Zookeeper
+}
+```
+
+
+
+An example script for [Uno] is provided.  To use this doe the following and set
+`UNO_HOME` after copying. 
+
+    cp conf/cluster-control.sh.uno conf/cluster-control.sh
+
+After the cluster control script is setup, the following will run performance
+test and produce json result files.
+
+    ./bin/performance-test run <output dir>
+
+There are some utilities for working with the json result files, run the performance-test script
+with no options to see them.
+
+[Uno]: https://github.com/apache/fluo-uno
 [modules]: core/src/main/resources/randomwalk/modules
 [image]: core/src/main/resources/randomwalk/modules/Image.xml
 [ti]: https://travis-ci.org/apache/accumulo-testing.svg?branch=master
diff --git a/bin/performance-test b/bin/performance-test
new file mode 100755
index 0000000..93014e5
--- /dev/null
+++ b/bin/performance-test
@@ -0,0 +1,97 @@
+#! /usr/bin/env bash
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+bin_dir=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )
+at_home=$( cd "$( dirname "$bin_dir" )" && pwd )
+at_version=2.0.0-SNAPSHOT
+
+function print_usage() {
+  cat <<EOF
+
+Usage: performance-test <command> (<argument>)
+
+Possible commands:
+  run <output dir>                 Runs performance tests.
+  compare <result 1> <result 2>    Compares results of two test.
+  csv {files}                      Converts results to CSV
+EOF
+}
+
+
+function build_shade_jar() {
+  at_shaded_jar="$at_home/core/target/accumulo-testing-core-$at_version-shaded.jar"
+  if [ ! -f "$at_shaded_jar" ]; then
+    echo "Building $at_shaded_jar"
+    cd "$at_home" || exit 1
+    mvn clean package -P create-shade-jar -D skipTests -D accumulo.version=$(get_version "ACCUMULO") -D hadoop.version=$(get_version "HADOOP") -D zookeeper.version=$(get_version "ZOOKEEPER")
+  fi
+}
+
+log4j_config="$at_home/conf/log4j.properties"
+if [ ! -f "$log4j_config" ]; then
+  log4j_config="$at_home/conf/log4j.properties.example"
+  if [ ! -f "$log4j_config" ]; then
+    echo "Could not find logj4.properties or log4j.properties.example in $at_home/conf"
+    exit 1
+  fi
+fi
+
+
+if [ ! -f "$at_home/conf/cluster-control.sh" ]; then
+  echo "Could not find cluster-control.sh"
+  exit 1
+fi
+
+. $at_home/conf/cluster-control.sh
+build_shade_jar
+CP="$at_home/core/target/accumulo-testing-core-$at_version-shaded.jar"
+perf_pkg="org.apache.accumulo.testing.core.performance.impl"
+case "$1" in
+  run)
+    if [ -z "$2" ]; then
+      echo "ERROR: <output dir> needs to be set"
+      print_usage
+      exit 1
+    fi
+    mkdir -p "$2"
+    start_cluster
+    CLASSPATH="$CP" java -Dlog4j.configuration="file:$log4j_config" ${perf_pkg}.ListTests | while read test_class; do
+      echo "Running test $test_class"
+      pt_tmp=$(mktemp -d -t accumulo_pt_XXXXXXX)
+      setup_accumulo
+      get_config_file accumulo-site.xml "$pt_tmp"
+      CLASSPATH="$CP" java -Dlog4j.configuration="file:$log4j_config"  ${perf_pkg}.MergeSiteConfig "$test_class" "$pt_tmp"
+      put_config_file "$pt_tmp/accumulo-site.xml"
+      put_server_code "$at_home/core/target/accumulo-testing-core-$at_version.jar"
+      start_accumulo
+      get_config_file accumulo-client.properties "$pt_tmp"
+      CLASSPATH="$CP" java -Dlog4j.configuration="file:$log4j_config"  ${perf_pkg}.PerfTestRunner "$pt_tmp/accumulo-client.properties" "$test_class" "$(get_version 'ACCUMULO')" "$2"
+    done
+    stop_cluster
+    ;;
+  compare)
+    CLASSPATH="$CP" java -Dlog4j.configuration="file:$log4j_config"  ${perf_pkg}.Compare "$2" "$3"
+    ;;
+  csv)
+    CLASSPATH="$CP" java -Dlog4j.configuration="file:$log4j_config"  ${perf_pkg}.Csv "${@:2}"
+    ;;
+  *)
+    echo "Unknown command : $1"
+    print_usage
+    exit 1
+esac
+
diff --git a/conf/.gitignore b/conf/.gitignore
index 7c4fb33..ca4a321 100644
--- a/conf/.gitignore
+++ b/conf/.gitignore
@@ -1,3 +1,4 @@
 /accumulo-testing.properties
 /accumulo-testing-env.sh
 /log4j.properties
+/cluster-control.sh
diff --git a/conf/cluster-control.sh.uno b/conf/cluster-control.sh.uno
new file mode 100644
index 0000000..2b9abdb
--- /dev/null
+++ b/conf/cluster-control.sh.uno
@@ -0,0 +1,86 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# internal functions just for uno cluster control
+
+UNO_HOME=/home/ubuntu/git/uno
+UNO=$UNO_HOME/bin/uno
+
+function get_ah {
+  echo "$($UNO env | grep ACCUMULO_HOME | sed 's/export ACCUMULO_HOME=//' | sed 's/"//g')"
+}
+
+
+# functions required for accumulo testing cluster control
+
+function get_version {
+  case $1 in
+    ACCUMULO)
+      (
+        # run following in sub shell so it does not pollute
+        . $UNO_HOME/conf/uno.conf
+        echo $ACCUMULO_VERSION
+      )
+      ;;
+    HADOOP)
+      (
+        # run following in sub shell so it does not pollute
+        . $UNO_HOME/conf/uno.conf
+        echo $HADOOP_VERSION
+      )
+      ;;
+    ZOOKEEPER)
+      (
+        # run following in sub shell so it does not pollute
+        . $UNO_HOME/conf/uno.conf
+        echo $ZOOKEEPER_VERSION
+      )
+      ;;
+    *)
+      return 1
+  esac
+}
+
+function start_cluster {
+  $UNO setup accumulo
+}
+
+function setup_accumulo {
+  $UNO setup accumulo --no-deps
+}
+
+function get_config_file {
+  local ah=$(get_ah)
+  cp "$ah/conf/$1" "$2"
+}
+
+function put_config_file {
+  local ah=$(get_ah)
+  cp "$1" "$ah/conf"
+}
+
+function put_server_code {
+  local ah=$(get_ah)
+  cp "$1" "$ah/lib/ext"
+}
+
+function start_accumulo {
+  $UNO stop accumulo --no-deps &> /dev/null
+  $UNO start accumulo --no-deps &> /dev/null
+}
+
+function stop_cluster {
+  $UNO kill
+}
diff --git a/core/pom.xml b/core/pom.xml
index 4655949..0d03f95 100644
--- a/core/pom.xml
+++ b/core/pom.xml
@@ -119,6 +119,10 @@
                       </excludes>
                     </filter>
                   </filters>
+                  <transformers>
+                    <!-- Hadoop uses service loader to find filesystem impls, without this may not find them -->
+                    <transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer"/>
+                  </transformers>
                 </configuration>
               </execution>
             </executions>
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/performance/Environment.java b/core/src/main/java/org/apache/accumulo/testing/core/performance/Environment.java
new file mode 100644
index 0000000..941fb8c
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/testing/core/performance/Environment.java
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.accumulo.testing.core.performance;
+
+import org.apache.accumulo.core.client.Connector;
+
+public interface Environment {
+  Connector getConnector();
+}
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/performance/Parameter.java b/core/src/main/java/org/apache/accumulo/testing/core/performance/Parameter.java
new file mode 100644
index 0000000..41eb3d4
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/testing/core/performance/Parameter.java
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.accumulo.testing.core.performance;
+
+public class Parameter {
+
+  public final String id;
+  public final String data;
+  public final String description;
+
+  public Parameter(String id, String data, String description) {
+    this.id = id;
+    this.data = data;
+    this.description = description;
+  }
+}
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/performance/PerformanceTest.java b/core/src/main/java/org/apache/accumulo/testing/core/performance/PerformanceTest.java
new file mode 100644
index 0000000..04ef98f
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/testing/core/performance/PerformanceTest.java
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.accumulo.testing.core.performance;
+
+public interface PerformanceTest {
+
+  SystemConfiguration getConfiguration();
+
+  Report runTest(Environment env) throws Exception;
+}
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/performance/Report.java b/core/src/main/java/org/apache/accumulo/testing/core/performance/Report.java
new file mode 100644
index 0000000..28924e3
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/testing/core/performance/Report.java
@@ -0,0 +1,109 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.accumulo.testing.core.performance;
+
+import java.util.List;
+import java.util.LongSummaryStatistics;
+
+import org.apache.accumulo.testing.core.performance.Result.Purpose;
+
+import com.google.common.collect.ImmutableList;
+
+public class Report {
+  public final String id;
+  public final String description;
+  public final List<Result> results;
+  public final List<Parameter> parameters;
+
+  public Report(String id, String description, List<Result> results, List<Parameter> parameters) {
+    this.id = id;
+    this.description = description;
+    this.results = ImmutableList.copyOf(results);
+    this.parameters = ImmutableList.copyOf(parameters);
+  }
+
+  public static class Builder {
+    private String id;
+    private String description = "";
+    private final ImmutableList.Builder<Result> results = new ImmutableList.Builder<>();
+    private final ImmutableList.Builder<Parameter> parameters = new ImmutableList.Builder<>();
+
+    private Builder() {}
+
+    public Builder id(String id) {
+      this.id = id;
+      return this;
+    }
+
+    public Builder description(String desc) {
+      this.description = desc;
+      return this;
+    }
+
+    public Builder result(String id, LongSummaryStatistics stats, String description) {
+      results.add(new Result(id, new Stats(stats.getMin(), stats.getMax(), stats.getSum(), stats.getAverage(), stats.getCount()), description,
+          Purpose.COMPARISON));
+      return this;
+    }
+
+    public Builder result(String id, Number data, String description) {
+      results.add(new Result(id, data, description, Purpose.COMPARISON));
+      return this;
+    }
+
+    public Builder result(String id, long amount, long time, String description) {
+      results.add(new Result(id, amount / (time / 1000.0), description, Purpose.COMPARISON));
+      return this;
+    }
+
+    public Builder info(String id, LongSummaryStatistics stats, String description) {
+      results.add(new Result(id, new Stats(stats.getMin(), stats.getMax(), stats.getSum(), stats.getAverage(), stats.getCount()), description,
+          Purpose.INFORMATIONAL));
+      return this;
+    }
+
+    public Builder info(String id, long amount, long time, String description) {
+      results.add(new Result(id, amount / (time / 1000.0), description, Purpose.INFORMATIONAL));
+      return this;
+    }
+
+    public Builder info(String id, Number data, String description) {
+      results.add(new Result(id, data, description, Purpose.INFORMATIONAL));
+      return this;
+    }
+
+    public Builder parameter(String id, Number data, String description) {
+      parameters.add(new Parameter(id, data.toString(), description));
+      return this;
+    }
+
+    public Builder parameter(String id, String data, String description) {
+      parameters.add(new Parameter(id, data, description));
+      return this;
+    }
+
+    public Report build() {
+      return new Report(id, description, results.build(), parameters.build());
+    }
+  }
+
+  public static Builder builder() {
+    return new Builder();
+  }
+
+}
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/performance/Result.java b/core/src/main/java/org/apache/accumulo/testing/core/performance/Result.java
new file mode 100644
index 0000000..78d07fa
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/testing/core/performance/Result.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.accumulo.testing.core.performance;
+
+public class Result {
+
+  public final String id;
+  public final Number data;
+  public final Stats stats;
+  public final String description;
+  public final Purpose purpose;
+
+  public enum Purpose {
+    /**
+     * Use for results that are unrelated to the main objective or are not useful for comparison.
+     */
+    INFORMATIONAL,
+    /**
+     * Use for results that related to the main objective and are useful for comparison.
+     */
+    COMPARISON
+  }
+
+  public Result(String id, Number data, String description, Purpose purpose) {
+    this.id = id;
+    this.data = data;
+    this.stats = null;
+    this.description = description;
+    this.purpose = purpose;
+  }
+
+  public Result(String id, Stats stats, String description, Purpose purpose) {
+    this.id = id;
+    this.stats = stats;
+    this.data = null;
+    this.description = description;
+    this.purpose = purpose;
+  }
+}
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/performance/Stats.java b/core/src/main/java/org/apache/accumulo/testing/core/performance/Stats.java
new file mode 100644
index 0000000..cd9c859
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/testing/core/performance/Stats.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.accumulo.testing.core.performance;
+
+public class Stats {
+  public final long min;
+  public final long max;
+  public final long sum;
+  public final double average;
+  public final long count;
+
+  public Stats(long min, long max, long sum, double average, long count) {
+    this.min = min;
+    this.max = max;
+    this.sum = sum;
+    this.average = average;
+    this.count = count;
+  }
+}
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/performance/SystemConfiguration.java b/core/src/main/java/org/apache/accumulo/testing/core/performance/SystemConfiguration.java
new file mode 100644
index 0000000..60f02e5
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/testing/core/performance/SystemConfiguration.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.accumulo.testing.core.performance;
+
+import java.util.Collections;
+import java.util.Map;
+
+import com.google.common.collect.ImmutableMap;
+
+public class SystemConfiguration {
+
+  private Map<String,String> accumuloSite = Collections.emptyMap();
+
+  public SystemConfiguration setAccumuloConfig(Map<String,String> props) {
+    accumuloSite = ImmutableMap.copyOf(props);
+    return this;
+  }
+
+  public Map<String,String> getAccumuloSite() {
+    return accumuloSite;
+  }
+}
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/performance/impl/Compare.java b/core/src/main/java/org/apache/accumulo/testing/core/performance/impl/Compare.java
new file mode 100644
index 0000000..e14f47d
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/testing/core/performance/impl/Compare.java
@@ -0,0 +1,117 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.accumulo.testing.core.performance.impl;
+
+import java.io.BufferedReader;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+
+import org.apache.accumulo.testing.core.performance.Result;
+import org.apache.accumulo.testing.core.performance.Result.Purpose;
+
+import com.google.common.collect.Sets;
+import com.google.gson.Gson;
+import com.google.gson.GsonBuilder;
+import com.google.gson.JsonElement;
+import com.google.gson.JsonStreamParser;
+
+public class Compare {
+
+  private static class TestId {
+
+    final String testClass;
+    final String id;
+
+    public TestId(String testClass, String id) {
+      this.testClass = testClass;
+      this.id = id;
+    }
+
+    @Override
+    public int hashCode() {
+      return Objects.hash(testClass, id);
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+      if (this == obj)
+        return true;
+
+      if (obj instanceof TestId) {
+        TestId other = (TestId) obj;
+
+        return id.equals(other.id) && testClass.equals(other.testClass);
+      }
+
+      return false;
+    }
+  }
+
+  public static void main(String[] args) throws Exception {
+    Map<TestId,Double> oldResults = flatten(readReports(args[0]));
+    Map<TestId,Double> newResults = flatten(readReports(args[1]));
+
+    for (TestId testId : Sets.union(oldResults.keySet(), newResults.keySet())) {
+      Double oldResult = oldResults.get(testId);
+      Double newResult = newResults.get(testId);
+
+      if (oldResult == null || newResult == null) {
+        System.out.printf("%s %s %.2f %.2f\n", testId.testClass, testId.id, oldResult, newResult);
+      } else {
+        double change = (newResult - oldResult) / oldResult;
+        System.out.printf("%s %s %.2f %.2f %.2f%s\n", testId.testClass, testId.id, oldResult, newResult, change * 100, "%");
+      }
+    }
+  }
+
+  static Collection<ContextualReport> readReports(String file) throws Exception {
+    try (BufferedReader reader = Files.newBufferedReader(Paths.get(file))) {
+      Gson gson = new GsonBuilder().create();
+      JsonStreamParser p = new JsonStreamParser(reader);
+      List<ContextualReport> rl = new ArrayList<>();
+
+      while (p.hasNext()) {
+        JsonElement e = p.next();
+        ContextualReport results = gson.fromJson(e, ContextualReport.class);
+        rl.add(results);
+      }
+
+      return rl;
+    }
+  }
+
+  private static Map<TestId,Double> flatten(Collection<ContextualReport> results) {
+    HashMap<TestId,Double> flattened = new HashMap<>();
+
+    for (ContextualReport cr : results) {
+      for (Result r : cr.results) {
+        if (r.purpose == Purpose.COMPARISON) {
+          flattened.put(new TestId(cr.testClass, r.id), r.data.doubleValue());
+        }
+      }
+    }
+
+    return flattened;
+  }
+}
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/performance/impl/ContextualReport.java b/core/src/main/java/org/apache/accumulo/testing/core/performance/impl/ContextualReport.java
new file mode 100644
index 0000000..5104c31
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/testing/core/performance/impl/ContextualReport.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.accumulo.testing.core.performance.impl;
+
+import java.time.Instant;
+
+import org.apache.accumulo.testing.core.performance.Report;
+
+public class ContextualReport extends Report {
+
+  public final String testClass;
+  public final String accumuloVersion;
+  public final String startTime;
+  public final String finishTime;
+
+  public ContextualReport(String testClass, String accumuloVersion, Instant startTime, Instant finishTime, Report r) {
+    super(r.id, r.description, r.results, r.parameters);
+    this.testClass = testClass;
+    this.accumuloVersion = accumuloVersion;
+    this.startTime = startTime.toString();
+    this.finishTime = finishTime.toString();
+
+  }
+}
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/performance/impl/Csv.java b/core/src/main/java/org/apache/accumulo/testing/core/performance/impl/Csv.java
new file mode 100644
index 0000000..6433c3c
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/testing/core/performance/impl/Csv.java
@@ -0,0 +1,129 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.accumulo.testing.core.performance.impl;
+
+import static java.util.stream.Collectors.joining;
+import static java.util.stream.Collectors.toList;
+import static java.util.stream.Collectors.toSet;
+
+import java.time.Instant;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.StringJoiner;
+import java.util.TreeMap;
+import java.util.stream.Stream;
+
+import org.apache.accumulo.testing.core.performance.Result;
+import org.apache.accumulo.testing.core.performance.Result.Purpose;
+
+import com.google.common.collect.Iterables;
+
+public class Csv {
+
+  private static class RowId implements Comparable<RowId> {
+
+    final Instant startTime;
+    final String accumuloVersion;
+
+    public RowId(Instant startTime, String accumuloVersion) {
+      this.startTime = startTime;
+      this.accumuloVersion = accumuloVersion;
+    }
+
+    @Override
+    public int compareTo(RowId o) {
+      int cmp = startTime.compareTo(o.startTime);
+      if (cmp == 0) {
+        cmp = accumuloVersion.compareTo(o.accumuloVersion);
+      }
+      return cmp;
+    }
+
+    @Override
+    public int hashCode() {
+      return Objects.hash(startTime, accumuloVersion);
+    }
+
+    @Override
+    public boolean equals(Object o) {
+      if (o instanceof RowId) {
+        RowId orid = (RowId) o;
+        return startTime.equals(orid.startTime) && accumuloVersion.equals(orid.accumuloVersion);
+      }
+
+      return false;
+    }
+
+    @Override
+    public String toString() {
+      return startTime + " " + accumuloVersion;
+    }
+
+  }
+
+  public static void main(String[] args) throws Exception {
+    Map<RowId, Map<String, Double>> rows = new TreeMap<>();
+
+    for (String file : args) {
+      Collection<ContextualReport> reports = Compare.readReports(file);
+
+      Instant minStart = reports.stream().map(cr -> cr.startTime).map(Instant::parse).min(Instant::compareTo).get();
+
+      String version = Iterables.getOnlyElement(reports.stream().map(cr -> cr.accumuloVersion).collect(toSet()));
+
+      Map<String, Double> row = new HashMap<>();
+
+      for (ContextualReport report : reports) {
+
+        String id = report.id != null ? report.id :  report.testClass.substring(report.testClass.lastIndexOf('.')+1);
+
+        for (Result result : report.results) {
+          if(result.purpose == Purpose.COMPARISON) {
+            row.put(id+"."+result.id, result.data.doubleValue());
+          }
+        }
+      }
+
+      rows.put(new RowId(minStart, version), row);
+    }
+
+    List<String> allCols = rows.values().stream().flatMap(row -> row.keySet().stream()).distinct().sorted().collect(toList());
+
+    // print header
+    print(Stream.concat(Stream.of("Start Time","Version"), allCols.stream()).collect(joining(",")));
+
+    rows.forEach((id, row)->{
+      StringJoiner joiner = new StringJoiner(",");
+      joiner.add(id.startTime.toString());
+      joiner.add(id.accumuloVersion);
+      for (String col : allCols) {
+        joiner.add(row.getOrDefault(col, Double.valueOf(0)).toString());
+      }
+
+      print(joiner.toString());
+    });
+
+  }
+
+  private static void print(String s) {
+    System.out.println(s);
+  }
+}
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/performance/impl/ListTests.java b/core/src/main/java/org/apache/accumulo/testing/core/performance/impl/ListTests.java
new file mode 100644
index 0000000..f966765
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/testing/core/performance/impl/ListTests.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.accumulo.testing.core.performance.impl;
+
+import org.apache.accumulo.testing.core.performance.PerformanceTest;
+
+import com.google.common.collect.ImmutableSet;
+import com.google.common.reflect.ClassPath;
+import com.google.common.reflect.ClassPath.ClassInfo;
+
+public class ListTests {
+  public static void main(String[] args) throws Exception {
+
+    ImmutableSet<ClassInfo> classes = ClassPath.from(ListTests.class.getClassLoader()).getTopLevelClasses();
+
+    for (ClassInfo classInfo : classes) {
+      if (classInfo.getName().endsWith("PT") && PerformanceTest.class.isAssignableFrom(classInfo.load())) {
+        System.out.println(classInfo.getName());
+      }
+    }
+  }
+}
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/performance/impl/MergeSiteConfig.java b/core/src/main/java/org/apache/accumulo/testing/core/performance/impl/MergeSiteConfig.java
new file mode 100644
index 0000000..2f79a88
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/testing/core/performance/impl/MergeSiteConfig.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.accumulo.testing.core.performance.impl;
+
+import java.io.BufferedInputStream;
+import java.io.ByteArrayOutputStream;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+
+import org.apache.accumulo.testing.core.performance.PerformanceTest;
+import org.apache.hadoop.conf.Configuration;
+
+public class MergeSiteConfig {
+  public static void main(String[] args) throws Exception {
+    String className = args[0];
+    Path confFile = Paths.get(args[1], "accumulo-site.xml");
+
+    PerformanceTest perfTest = Class.forName(className).asSubclass(PerformanceTest.class).newInstance();
+
+    Configuration conf = new Configuration(false);
+    byte[] newConf;
+
+
+    try(BufferedInputStream in = new BufferedInputStream(Files.newInputStream(confFile))){
+      conf.addResource(in);
+      perfTest.getConfiguration().getAccumuloSite().forEach((k,v) -> conf.set(k, v));
+      ByteArrayOutputStream baos = new ByteArrayOutputStream();
+      conf.writeXml(baos);
+      baos.close();
+      newConf = baos.toByteArray();
+    }
+
+
+    Files.write(confFile, newConf);
+  }
+}
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/performance/impl/PerfTestRunner.java b/core/src/main/java/org/apache/accumulo/testing/core/performance/impl/PerfTestRunner.java
new file mode 100644
index 0000000..4ce6b1a
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/testing/core/performance/impl/PerfTestRunner.java
@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.accumulo.testing.core.performance.impl;
+
+import java.io.Writer;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.time.Instant;
+import java.time.ZoneId;
+import java.time.format.DateTimeFormatter;
+
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.testing.core.performance.Environment;
+import org.apache.accumulo.testing.core.performance.PerformanceTest;
+import org.apache.accumulo.testing.core.performance.Report;
+
+import com.google.gson.Gson;
+import com.google.gson.GsonBuilder;
+
+public class PerfTestRunner {
+  public static void main(String[] args) throws Exception {
+    String clientProps = args[0];
+    String className = args[1];
+    String accumuloVersion = args[2];
+    String outputDir = args[3];
+
+    PerformanceTest perfTest = Class.forName(className).asSubclass(PerformanceTest.class).newInstance();
+
+    Connector conn = Connector.builder().usingProperties(clientProps).build();
+
+    Instant start = Instant.now();
+
+    Report result = perfTest.runTest(new Environment() {
+      @Override
+      public Connector getConnector() {
+        return conn;
+      }
+    });
+
+    Instant stop = Instant.now();
+
+    Gson gson = new GsonBuilder().setPrettyPrinting().create();
+
+    ContextualReport report = new ContextualReport(className, accumuloVersion, start, stop, result);
+
+    DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyyMMddHHmmss");
+    String time = Instant.now().atZone(ZoneId.systemDefault()).format(formatter);
+    Path outputFile = Paths.get(outputDir, perfTest.getClass().getSimpleName() + "_" + time + ".json");
+
+    try (Writer writer = Files.newBufferedWriter(outputFile)) {
+      gson.toJson(report, writer);
+    }
+  }
+}
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/performance/tests/ScanExecutorPT.java b/core/src/main/java/org/apache/accumulo/testing/core/performance/tests/ScanExecutorPT.java
new file mode 100644
index 0000000..ecfa182
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/testing/core/performance/tests/ScanExecutorPT.java
@@ -0,0 +1,177 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.accumulo.testing.core.performance.tests;
+
+import java.util.HashMap;
+import java.util.LongSummaryStatistics;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Random;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.client.TableNotFoundException;
+import org.apache.accumulo.core.client.admin.NewTableConfiguration;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.core.spi.scan.IdleRatioScanPrioritizer;
+import org.apache.accumulo.testing.core.performance.Environment;
+import org.apache.accumulo.testing.core.performance.PerformanceTest;
+import org.apache.accumulo.testing.core.performance.Report;
+import org.apache.accumulo.testing.core.performance.SystemConfiguration;
+import org.apache.accumulo.testing.core.performance.util.TestData;
+import org.apache.accumulo.testing.core.performance.util.TestExecutor;
+import org.apache.hadoop.io.Text;
+
+import com.google.common.collect.Iterables;
+
+public class ScanExecutorPT implements PerformanceTest {
+
+  private static final int NUM_SHORT_SCANS_THREADS = 5;
+  private static final int NUM_LONG_SCANS = 50;
+
+  private static final int NUM_ROWS = 10000;
+  private static final int NUM_FAMS = 10;
+  private static final int NUM_QUALS = 10;
+
+  private static final String SCAN_EXECUTOR_THREADS = "2";
+  private static final String SCAN_PRIORITIZER = IdleRatioScanPrioritizer.class.getName();
+
+  private static final String TEST_DESC = "Scan Executor Test.  Test running lots of short scans while long scans are running in the background.  Each short scan reads a random row and family. A scan prioritizer that favors short scans is configured.  If the scan prioritizer is not working properly, then the short "
+      + "scans will be orders of magnitude slower.";
+
+  @Override
+  public SystemConfiguration getConfiguration() {
+    Map<String,String> siteCfg = new HashMap<>();
+
+    siteCfg.put(Property.TSERV_SCAN_MAX_OPENFILES.getKey(), "200");
+    siteCfg.put(Property.TSERV_MINTHREADS.getKey(), "200");
+    siteCfg.put(Property.TSERV_SCAN_EXECUTORS_PREFIX.getKey() + "se1.threads", SCAN_EXECUTOR_THREADS);
+    siteCfg.put(Property.TSERV_SCAN_EXECUTORS_PREFIX.getKey() + "se1.prioritizer", SCAN_PRIORITIZER);
+
+    return new SystemConfiguration().setAccumuloConfig(siteCfg);
+  }
+
+  @Override
+  public Report runTest(Environment env) throws Exception {
+
+    String tableName = "scept";
+
+    Map<String,String> props = new HashMap<>();
+    props.put(Property.TABLE_SCAN_DISPATCHER_OPTS.getKey() + "executor", "se1");
+
+    env.getConnector().tableOperations().create(tableName, new NewTableConfiguration().setProperties(props));
+
+    long t1 = System.currentTimeMillis();
+    TestData.generate(env.getConnector(), tableName, NUM_ROWS, NUM_FAMS, NUM_QUALS);
+    long t2 = System.currentTimeMillis();
+    env.getConnector().tableOperations().compact(tableName, null, null, true, true);
+    long t3 = System.currentTimeMillis();
+
+    AtomicBoolean stop = new AtomicBoolean(false);
+
+    TestExecutor<Long> longScans = startLongScans(env, tableName, stop);
+
+    LongSummaryStatistics shortStats1 = runShortScans(env, tableName, 50000);
+    LongSummaryStatistics shortStats2 = runShortScans(env, tableName, 500000);
+
+    stop.set(true);
+    long t4 = System.currentTimeMillis();
+
+    LongSummaryStatistics longStats = longScans.stream().mapToLong(l -> l).summaryStatistics();
+
+    longScans.close();
+
+    Report.Builder builder = Report.builder();
+
+    builder.id("sexec").description(TEST_DESC);
+    builder.info("write", NUM_ROWS * NUM_FAMS * NUM_QUALS, t2 - t1, "Data write rate entries/sec ");
+    builder.info("compact", NUM_ROWS * NUM_FAMS * NUM_QUALS, t3 - t2, "Compact rate entries/sec ");
+    builder.info("short_times1", shortStats1, "Times in ms for each short scan.  First run.");
+    builder.info("short_times2", shortStats2, "Times in ms for each short scan. Second run.");
+    builder.result("short", shortStats2.getAverage(), "Average times in ms for short scans from 2nd run.");
+    builder.info("long_counts", longStats, "Entries read by each long scan threads");
+    builder.info("long", longStats.getSum(), (t4-t3), "Combined rate in entries/second of all long scans");
+    builder.parameter("short_threads", NUM_SHORT_SCANS_THREADS, "Threads used to run short scans.");
+    builder.parameter("long_threads", NUM_LONG_SCANS, "Threads running long scans.  Each thread repeatedly scans entire table for duration of test.");
+    builder.parameter("rows", NUM_ROWS, "Rows in test table");
+    builder.parameter("familes", NUM_FAMS, "Families per row in test table");
+    builder.parameter("qualifiers", NUM_QUALS, "Qualifiers per family in test table");
+    builder.parameter("server_scan_threads", SCAN_EXECUTOR_THREADS, "Server side scan handler threads");
+    builder.parameter("prioritizer", SCAN_PRIORITIZER, "Server side scan prioritizer");
+
+    return builder.build();
+  }
+
+  private static long scan(String tableName, Connector c, byte[] row, byte[] fam) throws TableNotFoundException {
+    long t1 = System.currentTimeMillis();
+    int count = 0;
+    try (Scanner scanner = c.createScanner(tableName, Authorizations.EMPTY)) {
+      scanner.setRange(Range.exact(new Text(row), new Text(fam)));
+      if (Iterables.size(scanner) != NUM_QUALS) {
+        throw new RuntimeException("bad count " + count);
+      }
+    }
+
+    return System.currentTimeMillis() - t1;
+  }
+
+  private long scan(String tableName, Connector c, AtomicBoolean stop) throws TableNotFoundException {
+    long count = 0;
+    while (!stop.get()) {
+      try (Scanner scanner = c.createScanner(tableName, Authorizations.EMPTY)) {
+        for (Entry<Key,Value> entry : scanner) {
+          count++;
+          if (stop.get()) {
+            return count;
+          }
+        }
+      }
+    }
+    return count;
+  }
+
+  private LongSummaryStatistics runShortScans(Environment env, String tableName, int numScans) throws InterruptedException, ExecutionException {
+
+    try(TestExecutor<Long> executor = new TestExecutor<>(NUM_SHORT_SCANS_THREADS)) {
+      Random rand = new Random();
+
+      for (int i = 0; i < numScans; i++) {
+        byte[] row = TestData.row(rand.nextInt(NUM_ROWS));
+        byte[] fam = TestData.fam(rand.nextInt(NUM_FAMS));
+        executor.submit(() -> scan(tableName, env.getConnector(), row, fam));
+      }
+
+      return executor.stream().mapToLong(l -> l).summaryStatistics();
+    }
+  }
+
+  private TestExecutor<Long> startLongScans(Environment env, String tableName, AtomicBoolean stop) {
+    TestExecutor<Long> longScans = new TestExecutor<>(NUM_LONG_SCANS);
+
+    for (int i = 0; i < NUM_LONG_SCANS; i++) {
+      longScans.submit(() -> scan(tableName, env.getConnector(), stop));
+    }
+    return longScans;
+  }
+}
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/performance/tests/ScanFewFamiliesPT.java b/core/src/main/java/org/apache/accumulo/testing/core/performance/tests/ScanFewFamiliesPT.java
new file mode 100644
index 0000000..3dcf261
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/testing/core/performance/tests/ScanFewFamiliesPT.java
@@ -0,0 +1,114 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.accumulo.testing.core.performance.tests;
+
+import java.util.HashSet;
+import java.util.LongSummaryStatistics;
+import java.util.Random;
+import java.util.Set;
+
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.client.TableNotFoundException;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.testing.core.performance.Environment;
+import org.apache.accumulo.testing.core.performance.PerformanceTest;
+import org.apache.accumulo.testing.core.performance.Report;
+import org.apache.accumulo.testing.core.performance.SystemConfiguration;
+import org.apache.accumulo.testing.core.performance.util.TestData;
+import org.apache.hadoop.io.Text;
+
+import com.google.common.base.Strings;
+import com.google.common.collect.Iterables;
+
+public class ScanFewFamiliesPT implements PerformanceTest {
+
+  private static final String DESC = "This test times fetching a few column famlies when rows have many column families.";
+
+  private static final int NUM_ROWS = 500;
+  private static final int NUM_FAMS = 10000;
+  private static final int NUM_QUALS = 1;
+
+  @Override
+  public SystemConfiguration getConfiguration() {
+    return new SystemConfiguration();
+  }
+
+  @Override
+  public Report runTest(Environment env) throws Exception {
+
+    String tableName = "bigFamily";
+
+    env.getConnector().tableOperations().create(tableName);
+
+    long t1 = System.currentTimeMillis();
+    TestData.generate(env.getConnector(), tableName, NUM_ROWS, NUM_FAMS, NUM_QUALS);
+    long t2 = System.currentTimeMillis();
+    env.getConnector().tableOperations().compact(tableName, null, null, true, true);
+    long t3 = System.currentTimeMillis();
+    // warm up run
+    runScans(env, tableName, 1);
+
+    Report.Builder builder = Report.builder();
+
+    for (int numFams : new int[] {1, 2, 4, 8, 16}) {
+      LongSummaryStatistics stats = runScans(env, tableName, numFams);
+      String fams = Strings.padStart(numFams + "", 2, '0');
+      builder.info("f" + fams + "_stats", stats, "Times in ms to fetch " + numFams + " families from all rows");
+      builder.result("f" + fams, stats.getAverage(), "Average time in ms to fetch " + numFams + " families from all rows");
+    }
+
+    builder.id("sfewfam");
+    builder.description(DESC);
+    builder.info("write", NUM_ROWS * NUM_FAMS * NUM_QUALS, t2 - t1, "Data write rate entries/sec ");
+    builder.info("compact", NUM_ROWS * NUM_FAMS * NUM_QUALS, t3 - t2, "Compact rate entries/sec ");
+    builder.parameter("rows", NUM_ROWS, "Rows in test table");
+    builder.parameter("familes", NUM_FAMS, "Families per row in test table");
+    builder.parameter("qualifiers", NUM_QUALS, "Qualifiers per family in test table");
+
+    return builder.build();
+  }
+
+  private LongSummaryStatistics runScans(Environment env, String tableName, int numFamilies) throws TableNotFoundException {
+    Random rand = new Random();
+    LongSummaryStatistics stats = new LongSummaryStatistics();
+    for (int i = 0; i < 50; i++) {
+      stats.accept(scan(tableName, env.getConnector(), rand, numFamilies));
+    }
+    return stats;
+  }
+
+  private static long scan(String tableName, Connector c, Random rand, int numFamilies) throws TableNotFoundException {
+
+    Set<Text> families = new HashSet<>(numFamilies);
+    while(families.size() < numFamilies) {
+      families.add(new Text(TestData.fam(rand.nextInt(NUM_FAMS))));
+    }
+
+    long t1 = System.currentTimeMillis();
+    int count = 0;
+    try (Scanner scanner = c.createScanner(tableName, Authorizations.EMPTY)) {
+      families.forEach(scanner::fetchColumnFamily);
+      if (Iterables.size(scanner) != NUM_ROWS * NUM_QUALS * numFamilies) {
+        throw new RuntimeException("bad count " + count);
+      }
+    }
+
+    return System.currentTimeMillis() - t1;
+  }
+}
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/performance/util/TestData.java b/core/src/main/java/org/apache/accumulo/testing/core/performance/util/TestData.java
new file mode 100644
index 0000000..a626a25
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/testing/core/performance/util/TestData.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.accumulo.testing.core.performance.util;
+
+import org.apache.accumulo.core.client.BatchWriter;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.util.FastFormat;
+
+public class TestData {
+
+  private static final byte[] EMPTY = new byte[0];
+
+  public static byte[] row(long r) {
+    return FastFormat.toZeroPaddedString(r, 16, 16, EMPTY);
+  }
+
+  public static byte[] fam(int f) {
+    return FastFormat.toZeroPaddedString(f, 8, 16, EMPTY);
+  }
+
+  public static byte[] qual(int q) {
+    return FastFormat.toZeroPaddedString(q, 8, 16, EMPTY);
+  }
+
+  public static byte[] val(long v) {
+    return FastFormat.toZeroPaddedString(v, 9, 16, EMPTY);
+  }
+
+  public static void generate(Connector conn, String tableName, int rows, int fams, int quals) throws Exception {
+    try (BatchWriter writer = conn.createBatchWriter(tableName)) {
+      int v = 0;
+      for (int r = 0; r < rows; r++) {
+        Mutation m = new Mutation(row(r));
+        for (int f = 0; f < fams; f++) {
+          byte[] fam = fam(f);
+          for (int q = 0; q < quals; q++) {
+            byte[] qual = qual(q);
+            byte[] val = val(v++);
+            m.put(fam, qual, val);
+          }
+        }
+        writer.addMutation(m);
+      }
+    }
+  }
+}
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/performance/util/TestExecutor.java b/core/src/main/java/org/apache/accumulo/testing/core/performance/util/TestExecutor.java
new file mode 100644
index 0000000..c4530a4
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/testing/core/performance/util/TestExecutor.java
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.accumulo.testing.core.performance.util;
+
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.stream.Stream;
+
+public class TestExecutor<T> implements Iterable<T>, AutoCloseable {
+  private ExecutorService es;
+  private List<Future<T>> futures = new ArrayList<>();
+
+  public TestExecutor(int numThreads) {
+    es = Executors.newFixedThreadPool(numThreads);
+  }
+
+  public void submit(Callable<T> task) {
+    futures.add(es.submit(task));
+  }
+
+  @Override
+  public void close() {
+    es.shutdownNow();
+  }
+
+  public Stream<T> stream() {
+    return futures.stream().map(f -> {try {
+      return f.get();
+    } catch (InterruptedException | ExecutionException e) {
+      throw new RuntimeException(e);
+    }});
+  }
+
+  @Override
+  public Iterator<T> iterator() {
+    return stream().iterator();
+  }
+}


Mime
View raw message