accumulo-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ktur...@apache.org
Subject [accumulo-testing] branch master updated: fixed peformance test (#39)
Date Sat, 13 Oct 2018 17:33:04 GMT
This is an automated email from the ASF dual-hosted git repository.

kturner pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/accumulo-testing.git


The following commit(s) were added to refs/heads/master by this push:
     new 5355941  fixed peformance test (#39)
5355941 is described below

commit 53559410596e8d8cc79c5148232ee44e79fecfa4
Author: Keith Turner <kturner@apache.org>
AuthorDate: Sat Oct 13 13:33:00 2018 -0400

    fixed peformance test (#39)
---
 bin/performance-test                               |  4 ++--
 conf/cluster-control.sh.uno                        | 18 ++++++++++++---
 .../testing/core/continuous/ContinuousIngest.java  |  7 +++---
 .../testing/core/continuous/UndefinedAnalyzer.java |  2 +-
 .../testing/core/ingest/BulkImportDirectory.java   |  2 +-
 .../testing/core/performance/PerformanceTest.java  |  2 +-
 .../core/performance/SystemConfiguration.java      |  2 +-
 .../core/performance/impl/MergeSiteConfig.java     | 26 +++++++++-------------
 .../performance/tests/RandomCachedLookupsPT.java   |  2 +-
 .../core/performance/tests/ScanExecutorPT.java     |  5 +++--
 .../core/performance/tests/ScanFewFamiliesPT.java  |  2 +-
 11 files changed, 41 insertions(+), 31 deletions(-)

diff --git a/bin/performance-test b/bin/performance-test
index b606ad3..7fa1e68 100755
--- a/bin/performance-test
+++ b/bin/performance-test
@@ -74,9 +74,9 @@ case "$1" in
         echo "Running test $test_class"
         pt_tmp=$(mktemp -d -t accumulo_pt_XXXXXXX)
         setup_accumulo
-        get_config_file accumulo-site.xml "$pt_tmp"
+        get_config_file accumulo.properties "$pt_tmp"
         CLASSPATH="$CP" java -Dlog4j.configuration="file:$log4j_config"  ${perf_pkg}.MergeSiteConfig
"$test_class" "$pt_tmp"
-        put_config_file "$pt_tmp/accumulo-site.xml"
+        put_config_file "$pt_tmp/accumulo.properties"
         put_server_code "$at_home/core/target/accumulo-testing-core-$at_version.jar"
         start_accumulo
         get_config_file accumulo-client.properties "$pt_tmp"
diff --git a/conf/cluster-control.sh.uno b/conf/cluster-control.sh.uno
index 2b9abdb..678ee35 100644
--- a/conf/cluster-control.sh.uno
+++ b/conf/cluster-control.sh.uno
@@ -30,21 +30,33 @@ function get_version {
     ACCUMULO)
       (
         # run following in sub shell so it does not pollute
-        . $UNO_HOME/conf/uno.conf
+        if [ -f $UNO_HOME/conf/uno-local.conf ]; then
+          . $UNO_HOME/conf/uno-local.conf
+        else
+          . $UNO_HOME/conf/uno.conf
+        fi
         echo $ACCUMULO_VERSION
       )
       ;;
     HADOOP)
       (
         # run following in sub shell so it does not pollute
-        . $UNO_HOME/conf/uno.conf
+        if [ -f $UNO_HOME/conf/uno-local.conf ]; then
+          . $UNO_HOME/conf/uno-local.conf
+        else
+          . $UNO_HOME/conf/uno.conf
+        fi
         echo $HADOOP_VERSION
       )
       ;;
     ZOOKEEPER)
       (
         # run following in sub shell so it does not pollute
-        . $UNO_HOME/conf/uno.conf
+        if [ -f $UNO_HOME/conf/uno-local.conf ]; then
+          . $UNO_HOME/conf/uno-local.conf
+        else
+          . $UNO_HOME/conf/uno.conf
+        fi
         echo $ZOOKEEPER_VERSION
       )
       ;;
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/continuous/ContinuousIngest.java
b/core/src/main/java/org/apache/accumulo/testing/core/continuous/ContinuousIngest.java
index 499be09..375c619 100644
--- a/core/src/main/java/org/apache/accumulo/testing/core/continuous/ContinuousIngest.java
+++ b/core/src/main/java/org/apache/accumulo/testing/core/continuous/ContinuousIngest.java
@@ -28,7 +28,6 @@ import java.util.concurrent.TimeUnit;
 import java.util.zip.CRC32;
 import java.util.zip.Checksum;
 
-import com.google.common.base.Preconditions;
 import org.apache.accumulo.core.client.AccumuloClient;
 import org.apache.accumulo.core.client.BatchWriter;
 import org.apache.accumulo.core.client.MutationsRejectedException;
@@ -36,14 +35,16 @@ import org.apache.accumulo.core.client.TableNotFoundException;
 import org.apache.accumulo.core.data.Mutation;
 import org.apache.accumulo.core.data.Value;
 import org.apache.accumulo.core.security.ColumnVisibility;
-import org.apache.accumulo.core.trace.CountSampler;
 import org.apache.accumulo.core.trace.Trace;
+import org.apache.accumulo.core.trace.TraceSamplers;
 import org.apache.accumulo.core.util.FastFormat;
 import org.apache.accumulo.testing.core.TestProps;
 import org.apache.hadoop.io.Text;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import com.google.common.base.Preconditions;
+
 public class ContinuousIngest {
 
   private static final Logger log = LoggerFactory.getLogger(ContinuousIngest.class);
@@ -129,7 +130,7 @@ public class ContinuousIngest {
     }
 
     BatchWriter bw = client.createBatchWriter(tableName);
-    bw = Trace.wrapAll(bw, new CountSampler(1024));
+    bw = Trace.wrapAll(bw, TraceSamplers.countSampler(1024));
 
     Random r = new Random();
 
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/continuous/UndefinedAnalyzer.java
b/core/src/main/java/org/apache/accumulo/testing/core/continuous/UndefinedAnalyzer.java
index 8c4c14d..56b7aa0 100644
--- a/core/src/main/java/org/apache/accumulo/testing/core/continuous/UndefinedAnalyzer.java
+++ b/core/src/main/java/org/apache/accumulo/testing/core/continuous/UndefinedAnalyzer.java
@@ -36,12 +36,12 @@ import java.util.TreeMap;
 import java.util.concurrent.TimeUnit;
 
 import org.apache.accumulo.core.cli.BatchScannerOpts;
+import org.apache.accumulo.core.cli.ClientOnDefaultTable;
 import org.apache.accumulo.core.client.AccumuloClient;
 import org.apache.accumulo.core.client.BatchScanner;
 import org.apache.accumulo.core.data.Key;
 import org.apache.accumulo.core.data.Range;
 import org.apache.accumulo.core.data.Value;
-import org.apache.accumulo.server.cli.ClientOnDefaultTable;
 import org.apache.hadoop.io.Text;
 
 import com.beust.jcommander.Parameter;
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/ingest/BulkImportDirectory.java
b/core/src/main/java/org/apache/accumulo/testing/core/ingest/BulkImportDirectory.java
index e5617c4..bc1b305 100644
--- a/core/src/main/java/org/apache/accumulo/testing/core/ingest/BulkImportDirectory.java
+++ b/core/src/main/java/org/apache/accumulo/testing/core/ingest/BulkImportDirectory.java
@@ -18,10 +18,10 @@ package org.apache.accumulo.testing.core.ingest;
 
 import java.io.IOException;
 
+import org.apache.accumulo.core.cli.ClientOnRequiredTable;
 import org.apache.accumulo.core.client.AccumuloException;
 import org.apache.accumulo.core.client.AccumuloSecurityException;
 import org.apache.accumulo.core.client.TableNotFoundException;
-import org.apache.accumulo.server.cli.ClientOnRequiredTable;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/performance/PerformanceTest.java
b/core/src/main/java/org/apache/accumulo/testing/core/performance/PerformanceTest.java
index 04ef98f..d957286 100644
--- a/core/src/main/java/org/apache/accumulo/testing/core/performance/PerformanceTest.java
+++ b/core/src/main/java/org/apache/accumulo/testing/core/performance/PerformanceTest.java
@@ -19,7 +19,7 @@ package org.apache.accumulo.testing.core.performance;
 
 public interface PerformanceTest {
 
-  SystemConfiguration getConfiguration();
+  SystemConfiguration getSystemConfig();
 
   Report runTest(Environment env) throws Exception;
 }
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/performance/SystemConfiguration.java
b/core/src/main/java/org/apache/accumulo/testing/core/performance/SystemConfiguration.java
index 60f02e5..c626a01 100644
--- a/core/src/main/java/org/apache/accumulo/testing/core/performance/SystemConfiguration.java
+++ b/core/src/main/java/org/apache/accumulo/testing/core/performance/SystemConfiguration.java
@@ -31,7 +31,7 @@ public class SystemConfiguration {
     return this;
   }
 
-  public Map<String,String> getAccumuloSite() {
+  public Map<String,String> getAccumuloConfig() {
     return accumuloSite;
   }
 }
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/performance/impl/MergeSiteConfig.java
b/core/src/main/java/org/apache/accumulo/testing/core/performance/impl/MergeSiteConfig.java
index 2f79a88..aeacf20 100644
--- a/core/src/main/java/org/apache/accumulo/testing/core/performance/impl/MergeSiteConfig.java
+++ b/core/src/main/java/org/apache/accumulo/testing/core/performance/impl/MergeSiteConfig.java
@@ -17,36 +17,32 @@
 
 package org.apache.accumulo.testing.core.performance.impl;
 
-import java.io.BufferedInputStream;
-import java.io.ByteArrayOutputStream;
+import java.io.Reader;
+import java.io.Writer;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.Paths;
+import java.util.Properties;
 
 import org.apache.accumulo.testing.core.performance.PerformanceTest;
-import org.apache.hadoop.conf.Configuration;
 
 public class MergeSiteConfig {
   public static void main(String[] args) throws Exception {
     String className = args[0];
-    Path confFile = Paths.get(args[1], "accumulo-site.xml");
+    Path confFile = Paths.get(args[1], "accumulo.properties");
 
     PerformanceTest perfTest = Class.forName(className).asSubclass(PerformanceTest.class).newInstance();
 
-    Configuration conf = new Configuration(false);
-    byte[] newConf;
+    Properties props = new Properties();
 
-
-    try(BufferedInputStream in = new BufferedInputStream(Files.newInputStream(confFile))){
-      conf.addResource(in);
-      perfTest.getConfiguration().getAccumuloSite().forEach((k,v) -> conf.set(k, v));
-      ByteArrayOutputStream baos = new ByteArrayOutputStream();
-      conf.writeXml(baos);
-      baos.close();
-      newConf = baos.toByteArray();
+    try(Reader in = Files.newBufferedReader(confFile)){
+      props.load(in);
     }
 
+    perfTest.getSystemConfig().getAccumuloConfig().forEach((k,v) -> props.setProperty(k,
v));
 
-    Files.write(confFile, newConf);
+    try(Writer out = Files.newBufferedWriter(confFile)){
+      props.store(out, "Modified by performance test");
+    }
   }
 }
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/performance/tests/RandomCachedLookupsPT.java
b/core/src/main/java/org/apache/accumulo/testing/core/performance/tests/RandomCachedLookupsPT.java
index 28d34bb..5655c3f 100644
--- a/core/src/main/java/org/apache/accumulo/testing/core/performance/tests/RandomCachedLookupsPT.java
+++ b/core/src/main/java/org/apache/accumulo/testing/core/performance/tests/RandomCachedLookupsPT.java
@@ -59,7 +59,7 @@ public class RandomCachedLookupsPT implements PerformanceTest {
   private static final int NUM_ROWS = 100000;
 
   @Override
-  public SystemConfiguration getConfiguration() {
+  public SystemConfiguration getSystemConfig() {
     Map<String,String> siteCfg = new HashMap<>();
 
     siteCfg.put(Property.TSERV_SCAN_MAX_OPENFILES.getKey(), "1000");
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/performance/tests/ScanExecutorPT.java
b/core/src/main/java/org/apache/accumulo/testing/core/performance/tests/ScanExecutorPT.java
index 994af06..8f1e2d0 100644
--- a/core/src/main/java/org/apache/accumulo/testing/core/performance/tests/ScanExecutorPT.java
+++ b/core/src/main/java/org/apache/accumulo/testing/core/performance/tests/ScanExecutorPT.java
@@ -64,7 +64,7 @@ public class ScanExecutorPT implements PerformanceTest {
       + "a dedicated executor.  If the scan prioritizer or dispatcher is not working properly,
" + "then the short scans will be orders of magnitude slower.";
 
   @Override
-  public SystemConfiguration getConfiguration() {
+  public SystemConfiguration getSystemConfig() {
     Map<String,String> siteCfg = new HashMap<>();
 
     siteCfg.put(Property.TSERV_SCAN_MAX_OPENFILES.getKey(), "200");
@@ -85,6 +85,7 @@ public class ScanExecutorPT implements PerformanceTest {
     Map<String,String> props = new HashMap<>();
     props.put(Property.TABLE_SCAN_DISPATCHER_OPTS.getKey() + "executor", "se1");
     props.put(Property.TABLE_SCAN_DISPATCHER_OPTS.getKey() + "heed_hints", "true");
+    props.put(Property.TABLE_BLOCKCACHE_ENABLED.getKey(), "true");
 
     env.getClient().tableOperations().create(tableName,
         new NewTableConfiguration().setProperties(props));
@@ -100,7 +101,7 @@ public class ScanExecutorPT implements PerformanceTest {
     TestExecutor<Long> longScans = startLongScans(env, tableName, stop);
 
     LongSummaryStatistics shortStats1 = runShortScans(env, tableName, 50000);
-    LongSummaryStatistics shortStats2 = runShortScans(env, tableName, 500000);
+    LongSummaryStatistics shortStats2 = runShortScans(env, tableName, 100000);
 
     stop.set(true);
     long t4 = System.currentTimeMillis();
diff --git a/core/src/main/java/org/apache/accumulo/testing/core/performance/tests/ScanFewFamiliesPT.java
b/core/src/main/java/org/apache/accumulo/testing/core/performance/tests/ScanFewFamiliesPT.java
index b098ba0..4f675f5 100644
--- a/core/src/main/java/org/apache/accumulo/testing/core/performance/tests/ScanFewFamiliesPT.java
+++ b/core/src/main/java/org/apache/accumulo/testing/core/performance/tests/ScanFewFamiliesPT.java
@@ -45,7 +45,7 @@ public class ScanFewFamiliesPT implements PerformanceTest {
   private static final int NUM_QUALS = 1;
 
   @Override
-  public SystemConfiguration getConfiguration() {
+  public SystemConfiguration getSystemConfig() {
     return new SystemConfiguration();
   }
 


Mime
View raw message