Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id 65450200C14 for ; Tue, 7 Feb 2017 15:23:09 +0100 (CET) Received: by cust-asf.ponee.io (Postfix) id 63B35160B4B; Tue, 7 Feb 2017 14:23:09 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id 2C89C160B3E for ; Tue, 7 Feb 2017 15:23:08 +0100 (CET) Received: (qmail 22365 invoked by uid 500); 7 Feb 2017 14:23:07 -0000 Mailing-List: contact commits-help@hbase.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@hbase.apache.org Delivered-To: mailing list commits@hbase.apache.org Received: (qmail 22356 invoked by uid 99); 7 Feb 2017 14:23:07 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 07 Feb 2017 14:23:07 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 3B36ADFBE6; Tue, 7 Feb 2017 14:23:07 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: tedyu@apache.org To: commits@hbase.apache.org Message-Id: X-Mailer: ASF-Git Admin Mailer Subject: hbase git commit: HBASE-17565 StochasticLoadBalancer may incorrectly skip balancing due to skewed multiplier sum Date: Tue, 7 Feb 2017 14:23:07 +0000 (UTC) archived-at: Tue, 07 Feb 2017 14:23:09 -0000 Repository: hbase Updated Branches: refs/heads/master 5c77a7dcd -> 9d8de85fa HBASE-17565 StochasticLoadBalancer may incorrectly skip balancing due to skewed multiplier sum Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/9d8de85f Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/9d8de85f Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/9d8de85f Branch: refs/heads/master Commit: 9d8de85fa513dde34f2382af9221f93164248dc8 Parents: 5c77a7d Author: tedyu Authored: Tue Feb 7 06:22:47 2017 -0800 Committer: tedyu Committed: Tue Feb 7 06:22:47 2017 -0800 ---------------------------------------------------------------------- .../hbase/master/balancer/StochasticLoadBalancer.java | 13 ++++++++++++- .../org/apache/hadoop/hbase/TestRegionRebalancing.java | 1 - .../hadoop/hbase/master/balancer/BalancerTestBase.java | 4 +++- .../master/balancer/TestStochasticLoadBalancer.java | 1 - .../master/balancer/TestStochasticLoadBalancer2.java | 1 - 5 files changed, 15 insertions(+), 5 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hbase/blob/9d8de85f/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.java index 226dee7..8825637 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.java @@ -283,6 +283,10 @@ public class StochasticLoadBalancer extends BaseLoadBalancer { if (multiplier <= 0) { continue; } + if (!c.isNeeded()) { + LOG.debug(c.getClass().getName() + " indicated that its cost should not be considered"); + continue; + } sumMultiplier += multiplier; total += c.cost() * multiplier; } @@ -925,9 +929,11 @@ public class StochasticLoadBalancer extends BaseLoadBalancer { protected Cluster cluster; CostFunction(Configuration c) { - } + boolean isNeeded() { + return true; + } float getMultiplier() { return multiplier; } @@ -1430,6 +1436,11 @@ public class StochasticLoadBalancer extends BaseLoadBalancer { } @Override + boolean isNeeded() { + return cluster.hasRegionReplicas; + } + + @Override double cost() { if (maxCost <= 0) { return 0; http://git-wip-us.apache.org/repos/asf/hbase/blob/9d8de85f/hbase-server/src/test/java/org/apache/hadoop/hbase/TestRegionRebalancing.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/TestRegionRebalancing.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/TestRegionRebalancing.java index 6f2c5c4..03c5524 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/TestRegionRebalancing.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/TestRegionRebalancing.java @@ -85,7 +85,6 @@ public class TestRegionRebalancing { public void before() throws Exception { UTIL.getConfiguration().set("hbase.master.loadbalancer.class", this.balancerName); // set minCostNeedBalance to 0, make sure balancer run - UTIL.getConfiguration().setFloat("hbase.master.balancer.stochastic.minCostNeedBalance", 0.0f); UTIL.startMiniCluster(1); this.desc = new HTableDescriptor(TableName.valueOf("test")); this.desc.addFamily(new HColumnDescriptor(FAMILY_NAME)); http://git-wip-us.apache.org/repos/asf/hbase/blob/9d8de85f/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/BalancerTestBase.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/BalancerTestBase.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/BalancerTestBase.java index 622dc4b..0f427ad 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/BalancerTestBase.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/BalancerTestBase.java @@ -92,7 +92,6 @@ public class BalancerTestBase { conf.setFloat("hbase.master.balancer.stochastic.maxMovePercent", 0.75f); conf.setFloat("hbase.regions.slop", 0.0f); conf.setFloat("hbase.master.balancer.stochastic.localityCost", 0); - conf.setFloat("hbase.master.balancer.stochastic.minCostNeedBalance", 0.0f); loadBalancer = new StochasticLoadBalancer(); loadBalancer.setConf(conf); } @@ -150,6 +149,9 @@ public class BalancerTestBase { new int[]{0, 0, 0, 7}, // 5 node new int[]{1, 1, 1, 1, 4}, + // 6 nodes + new int[]{1500, 500, 500, 500, 10, 0}, + new int[]{1500, 500, 500, 500, 500, 0}, // more nodes new int[]{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15}, new int[]{0, 0, 0, 0, 0, 0, 0, 0, 0, 10}, http://git-wip-us.apache.org/repos/asf/hbase/blob/9d8de85f/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestStochasticLoadBalancer.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestStochasticLoadBalancer.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestStochasticLoadBalancer.java index ba030c8..e18334b 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestStochasticLoadBalancer.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestStochasticLoadBalancer.java @@ -104,7 +104,6 @@ public class TestStochasticLoadBalancer extends BalancerTestBase { assertNull(plans); } // reset config - conf.setFloat("hbase.master.balancer.stochastic.minCostNeedBalance", 0.0f); loadBalancer.setConf(conf); } http://git-wip-us.apache.org/repos/asf/hbase/blob/9d8de85f/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestStochasticLoadBalancer2.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestStochasticLoadBalancer2.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestStochasticLoadBalancer2.java index 44c806d..2f315de 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestStochasticLoadBalancer2.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestStochasticLoadBalancer2.java @@ -43,7 +43,6 @@ public class TestStochasticLoadBalancer2 extends BalancerTestBase { @After public void after() { // reset config to make sure balancer run - conf.setFloat("hbase.master.balancer.stochastic.minCostNeedBalance", 0.0f); loadBalancer.setConf(conf); }