accumulo-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From els...@apache.org
Subject [1/3] accumulo git commit: ACCUMULO-3845 Run each write test multiple times per durability.
Date Fri, 22 May 2015 02:45:45 GMT
Repository: accumulo
Updated Branches:
  refs/heads/1.7 747da5a9a -> 2eeaebb69
  refs/heads/master 350b491d9 -> 19bacd292


ACCUMULO-3845 Run each write test multiple times per durability.

This test can be a little sensitive to performance
outliers. If we run each ingest cycle at each durability
multiple times and take the median, we should have a
more reliable test.


Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/2eeaebb6
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/2eeaebb6
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/2eeaebb6

Branch: refs/heads/1.7
Commit: 2eeaebb692cbb9a649a7e20da34dfd3f3cf891f3
Parents: 747da5a
Author: Josh Elser <elserj@apache.org>
Authored: Thu May 21 22:43:23 2015 -0400
Committer: Josh Elser <elserj@apache.org>
Committed: Thu May 21 22:45:22 2015 -0400

----------------------------------------------------------------------
 .../accumulo/test/functional/DurabilityIT.java  | 44 +++++++++++++-------
 1 file changed, 29 insertions(+), 15 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/accumulo/blob/2eeaebb6/test/src/test/java/org/apache/accumulo/test/functional/DurabilityIT.java
----------------------------------------------------------------------
diff --git a/test/src/test/java/org/apache/accumulo/test/functional/DurabilityIT.java b/test/src/test/java/org/apache/accumulo/test/functional/DurabilityIT.java
index f2ba949..aa280dc 100644
--- a/test/src/test/java/org/apache/accumulo/test/functional/DurabilityIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/functional/DurabilityIT.java
@@ -19,6 +19,7 @@ package org.apache.accumulo.test.functional;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertTrue;
 
+import java.util.Arrays;
 import java.util.HashMap;
 import java.util.Map;
 import java.util.Map.Entry;
@@ -36,10 +37,13 @@ import org.apache.accumulo.minicluster.impl.ProcessReference;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.RawLocalFileSystem;
 import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 import com.google.common.collect.Iterators;
 
 public class DurabilityIT extends ConfigurableMacIT {
+  private static final Logger log = LoggerFactory.getLogger(DurabilityIT.class);
 
   @Override
   public void configure(MiniAccumuloConfigImpl cfg, Configuration hadoopCoreSite) {
@@ -82,17 +86,21 @@ public class DurabilityIT extends ConfigurableMacIT {
     TableOperations tableOps = getConnector().tableOperations();
     String tableNames[] = init();
     // write some gunk, delete the table to keep that table from messing with the performance
numbers of successive calls
+    // sync
     long t0 = writeSome(tableNames[0], N);
     tableOps.delete(tableNames[0]);
+    // flush
     long t1 = writeSome(tableNames[1], N);
     tableOps.delete(tableNames[1]);
+    // log
     long t2 = writeSome(tableNames[2], N);
     tableOps.delete(tableNames[2]);
+    // none
     long t3 = writeSome(tableNames[3], N);
     tableOps.delete(tableNames[3]);
     System.out.println(String.format("sync %d flush %d log %d none %d", t0, t1, t2, t3));
-    assertTrue("flush-only should be faster than sync", t0 > t1);
-    assertTrue("sync should be faster than log", t1 > t2);
+    assertTrue("flush should be faster than sync", t0 > t1);
+    assertTrue("log should be faster than flush", t1 > t2);
     assertTrue("no durability should be faster than log", t2 > t3);
   }
 
@@ -185,21 +193,27 @@ public class DurabilityIT extends ConfigurableMacIT {
   }
 
   private long writeSome(String table, long count) throws Exception {
-    long now = System.currentTimeMillis();
-    Connector c = getConnector();
-    BatchWriter bw = c.createBatchWriter(table, null);
-    for (int i = 1; i < count + 1; i++) {
-      Mutation m = new Mutation("" + i);
-      m.put("", "", "");
-      bw.addMutation(m);
-      if (i % (Math.max(1, count / 100)) == 0) {
-        bw.flush();
+    int iterations = 5;
+    long[] attempts = new long[iterations];
+    for (int attempt = 0; attempt < iterations; attempt++) {
+      long now = System.currentTimeMillis();
+      Connector c = getConnector();
+      BatchWriter bw = c.createBatchWriter(table, null);
+      for (int i = 1; i < count + 1; i++) {
+        Mutation m = new Mutation("" + i);
+        m.put("", "", "");
+        bw.addMutation(m);
+        if (i % (Math.max(1, count / 100)) == 0) {
+          bw.flush();
+        }
       }
+      bw.close();
+      attempts[attempt] = System.currentTimeMillis() - now;
     }
-    bw.close();
-    long result = System.currentTimeMillis() - now;
-    // c.tableOperations().flush(table, null, null, true);
-    return result;
+    Arrays.sort(attempts);
+    log.info("Attempt durations: {}", Arrays.toString(attempts));
+    // Return the median duration
+    return attempts[2];
   }
 
 }


Mime
View raw message