accumulo-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From bhava...@apache.org
Subject [01/10] git commit: ACCUMULO-2488 Change criteria for unbalanced servers in concurrent randomwalk
Date Tue, 18 Mar 2014 20:32:53 GMT
Repository: accumulo
Updated Branches:
  refs/heads/1.4.5-SNAPSHOT 6df83da60 -> a4174248a
  refs/heads/1.5.2-SNAPSHOT 205f0dca8 -> 1f4cb4d5b
  refs/heads/1.6.0-SNAPSHOT 70d472662 -> 99b6e252d
  refs/heads/master 39807cb2a -> 7cd087516


ACCUMULO-2488 Change criteria for unbalanced servers in concurrent randomwalk

The Concurrent randomwalk test used to consider servers unbalanced if any server's
tablet count differed from the cluster average by more than a fifth of the average or
by one, whichever was larger. This would cause failures under typical balancings from
the default balancer.

This commit changes the criterion for an unbalanced server to be double the standard
deviation from the cluster average.


Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/a4174248
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/a4174248
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/a4174248

Branch: refs/heads/1.4.5-SNAPSHOT
Commit: a4174248a96cadcc79a9de4015c90c6618a96418
Parents: 6df83da
Author: Bill Havanki <bhavanki@cloudera.com>
Authored: Tue Mar 18 09:12:19 2014 -0400
Committer: Bill Havanki <bhavanki@cloudera.com>
Committed: Tue Mar 18 16:22:22 2014 -0400

----------------------------------------------------------------------
 .../test/randomwalk/concurrent/CheckBalance.java  | 18 +++++++++++++++---
 1 file changed, 15 insertions(+), 3 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/accumulo/blob/a4174248/src/server/src/main/java/org/apache/accumulo/server/test/randomwalk/concurrent/CheckBalance.java
----------------------------------------------------------------------
diff --git a/src/server/src/main/java/org/apache/accumulo/server/test/randomwalk/concurrent/CheckBalance.java
b/src/server/src/main/java/org/apache/accumulo/server/test/randomwalk/concurrent/CheckBalance.java
index d00e2b4..5d0b978 100644
--- a/src/server/src/main/java/org/apache/accumulo/server/test/randomwalk/concurrent/CheckBalance.java
+++ b/src/server/src/main/java/org/apache/accumulo/server/test/randomwalk/concurrent/CheckBalance.java
@@ -16,6 +16,7 @@
  */
 package org.apache.accumulo.server.test.randomwalk.concurrent;
 
+import java.util.Collection;
 import java.util.HashMap;
 import java.util.Map;
 import java.util.Map.Entry;
@@ -57,9 +58,11 @@ public class CheckBalance extends Test {
       total += count.longValue();
     }
     final double average = total / counts.size();
-    
-    // Check for even # of tablets on each node
-    double maxDifference = Math.max(1, average / 5);
+    final double sd = stddev(counts.values(), average);
+    log.debug("average " + average + ", standard deviation " + sd);
+
+    // Check for balanced # of tablets on each node
+    double maxDifference = 2.0 * sd;
     String unbalancedLocation = null;
     long lastCount = 0L;
     boolean balanced = true;
@@ -94,4 +97,13 @@ public class CheckBalance extends Test {
     }
   }
   
+  private static double stddev(Collection<Long> samples, double avg) {
+    int num = samples.size();
+    double sqrtotal = 0.0;
+    for (Long s : samples) {
+      double diff = s.doubleValue() - avg;
+      sqrtotal += diff * diff;
+    }
+    return Math.sqrt(sqrtotal / (double) num);
+  }
 }


Mime
View raw message