hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From te...@apache.org
Subject hbase git commit: HBASE-17565 StochasticLoadBalancer may incorrectly skip balancing due to skewed multiplier sum
Date Tue, 07 Feb 2017 14:26:33 GMT
Repository: hbase
Updated Branches:
  refs/heads/branch-1 0a0aef345 -> 5a0020e86


HBASE-17565 StochasticLoadBalancer may incorrectly skip balancing due to skewed multiplier
sum


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/5a0020e8
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/5a0020e8
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/5a0020e8

Branch: refs/heads/branch-1
Commit: 5a0020e8674f83667e422bbcab2b8641342e2c67
Parents: 0a0aef3
Author: tedyu <yuzhihong@gmail.com>
Authored: Tue Feb 7 06:26:12 2017 -0800
Committer: tedyu <yuzhihong@gmail.com>
Committed: Tue Feb 7 06:26:12 2017 -0800

----------------------------------------------------------------------
 .../hbase/master/balancer/StochasticLoadBalancer.java  | 13 ++++++++++++-
 .../org/apache/hadoop/hbase/TestRegionRebalancing.java |  1 -
 .../hadoop/hbase/master/balancer/BalancerTestBase.java |  4 +++-
 .../master/balancer/TestStochasticLoadBalancer.java    |  1 -
 .../master/balancer/TestStochasticLoadBalancer2.java   |  1 -
 5 files changed, 15 insertions(+), 5 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hbase/blob/5a0020e8/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.java
index 36d3709..8872da6 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.java
@@ -283,6 +283,10 @@ public class StochasticLoadBalancer extends BaseLoadBalancer {
       if (multiplier <= 0) {
         continue;
       }
+      if (!c.isNeeded()) {
+        LOG.debug(c.getClass().getName() + " indicated that its cost should not be considered");
+        continue;
+      }
       sumMultiplier += multiplier;
       total += c.cost() * multiplier;
     }
@@ -926,9 +930,11 @@ public class StochasticLoadBalancer extends BaseLoadBalancer {
     protected Cluster cluster;
 
     CostFunction(Configuration c) {
-
     }
 
+    boolean isNeeded() {
+      return true;
+    }
     float getMultiplier() {
       return multiplier;
     }
@@ -1434,6 +1440,11 @@ public class StochasticLoadBalancer extends BaseLoadBalancer {
     }
 
     @Override
+    boolean isNeeded() {
+      return cluster.hasRegionReplicas;
+    }
+
+    @Override
     double cost() {
       if (maxCost <= 0) {
         return 0;

http://git-wip-us.apache.org/repos/asf/hbase/blob/5a0020e8/hbase-server/src/test/java/org/apache/hadoop/hbase/TestRegionRebalancing.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/TestRegionRebalancing.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/TestRegionRebalancing.java
index e4890e6..90e74f9 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/TestRegionRebalancing.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/TestRegionRebalancing.java
@@ -84,7 +84,6 @@ public class TestRegionRebalancing {
   public void before() throws Exception {
     UTIL.getConfiguration().set("hbase.master.loadbalancer.class", this.balancerName);
     // set minCostNeedBalance to 0, make sure balancer run
-    UTIL.getConfiguration().setFloat("hbase.master.balancer.stochastic.minCostNeedBalance",
0.0f);
     UTIL.startMiniCluster(1);
     this.desc = new HTableDescriptor(TableName.valueOf("test"));
     this.desc.addFamily(new HColumnDescriptor(FAMILY_NAME));

http://git-wip-us.apache.org/repos/asf/hbase/blob/5a0020e8/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/BalancerTestBase.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/BalancerTestBase.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/BalancerTestBase.java
index 5746e3f..5bd6606 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/BalancerTestBase.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/BalancerTestBase.java
@@ -71,7 +71,6 @@ public class BalancerTestBase {
     conf.setFloat("hbase.master.balancer.stochastic.maxMovePercent", 0.75f);
     conf.setFloat("hbase.regions.slop", 0.0f);
     conf.setFloat("hbase.master.balancer.stochastic.localityCost", 0);
-    conf.setFloat("hbase.master.balancer.stochastic.minCostNeedBalance", 0.0f);
     loadBalancer = new StochasticLoadBalancer();
     loadBalancer.setConf(conf);
   }
@@ -129,6 +128,9 @@ public class BalancerTestBase {
       new int[]{0, 0, 0, 7},
       // 5 node
       new int[]{1, 1, 1, 1, 4},
+      // 6 nodes
+      new int[]{1500, 500, 500, 500, 10, 0},
+      new int[]{1500, 500, 500, 500, 500, 0},
       // more nodes
       new int[]{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15},
       new int[]{0, 0, 0, 0, 0, 0, 0, 0, 0, 10},

http://git-wip-us.apache.org/repos/asf/hbase/blob/5a0020e8/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestStochasticLoadBalancer.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestStochasticLoadBalancer.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestStochasticLoadBalancer.java
index 3d975b8..8e3ec7c 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestStochasticLoadBalancer.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestStochasticLoadBalancer.java
@@ -104,7 +104,6 @@ public class TestStochasticLoadBalancer extends BalancerTestBase {
       assertNull(plans);
     }
     // reset config
-    conf.setFloat("hbase.master.balancer.stochastic.minCostNeedBalance", 0.0f);
     loadBalancer.setConf(conf);
   }
 

http://git-wip-us.apache.org/repos/asf/hbase/blob/5a0020e8/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestStochasticLoadBalancer2.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestStochasticLoadBalancer2.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestStochasticLoadBalancer2.java
index 62ad8a3..aafefca 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestStochasticLoadBalancer2.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestStochasticLoadBalancer2.java
@@ -42,7 +42,6 @@ public class TestStochasticLoadBalancer2 extends BalancerTestBase {
   @After
   public void after() {
     // reset config to make sure balancer run
-    conf.setFloat("hbase.master.balancer.stochastic.minCostNeedBalance", 0.0f);
     loadBalancer.setConf(conf);
   }
 


Mime
View raw message