hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From te...@apache.org
Subject hbase git commit: HBASE-17462 Use sliding window for read/write request costs in StochasticLoadBalancer (Tim Brown)
Date Mon, 23 Jan 2017 22:55:46 GMT
Repository: hbase
Updated Branches:
  refs/heads/branch-1 50ecbf1c8 -> 19f9a1a64


HBASE-17462 Use sliding window for read/write request costs in StochasticLoadBalancer (Tim
Brown)


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/19f9a1a6
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/19f9a1a6
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/19f9a1a6

Branch: refs/heads/branch-1
Commit: 19f9a1a64329be14f048128394df8b11131c49d4
Parents: 50ecbf1
Author: tedyu <yuzhihong@gmail.com>
Authored: Mon Jan 23 14:55:40 2017 -0800
Committer: tedyu <yuzhihong@gmail.com>
Committed: Mon Jan 23 14:55:40 2017 -0800

----------------------------------------------------------------------
 .../master/balancer/StochasticLoadBalancer.java | 45 +++++++++++++++-----
 .../balancer/TestStochasticLoadBalancer.java    | 24 +++++++++++
 2 files changed, 58 insertions(+), 11 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hbase/blob/19f9a1a6/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.java
index 7d7dc8e..36d3709 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.java
@@ -1301,19 +1301,42 @@ public class StochasticLoadBalancer extends BaseLoadBalancer {
       double cost = 0;
 
       for (RegionLoad rl : regionLoadList) {
-        double toAdd = getCostFromRl(rl);
+        cost += getCostFromRl(rl);
+      }
+      return cost / regionLoadList.size();
+    }
+
+
+    protected abstract double getCostFromRl(RegionLoad rl);
+  }
+
+  /**
+   * Class to be used for the subset of RegionLoad costs that should be treated as rates.
+   * We do not compare about the actual rate in requests per second but rather the rate relative
+   * to the rest of the regions.
+   */
+  abstract static class CostFromRegionLoadAsRateFunction extends CostFromRegionLoadFunction
{
+
+    CostFromRegionLoadAsRateFunction(Configuration conf) {
+      super(conf);
+    }
 
-        if (cost == 0) {
-          cost = toAdd;
+    @Override
+    protected double getRegionLoadCost(Collection<RegionLoad> regionLoadList) {
+      double cost = 0;
+      double previous = 0;
+      boolean isFirst = true;
+      for (RegionLoad rl : regionLoadList) {
+        double current = getCostFromRl(rl);
+        if (isFirst) {
+          isFirst = false;
         } else {
-          cost = (.5 * cost) + (.5 * toAdd);
+          cost += current - previous;
         }
+        previous = current;
       }
-
-      return cost;
+      return Math.max(0, cost / (regionLoadList.size() - 1));
     }
-
-    protected abstract double getCostFromRl(RegionLoad rl);
   }
 
   /**
@@ -1321,7 +1344,7 @@ public class StochasticLoadBalancer extends BaseLoadBalancer {
    * computed cost will be.  This uses a rolling average of regionload.
    */
 
-  static class ReadRequestCostFunction extends CostFromRegionLoadFunction {
+  static class ReadRequestCostFunction extends CostFromRegionLoadAsRateFunction {
 
     private static final String READ_REQUEST_COST_KEY =
         "hbase.master.balancer.stochastic.readRequestCost";
@@ -1343,7 +1366,7 @@ public class StochasticLoadBalancer extends BaseLoadBalancer {
    * Compute the cost of total number of write requests.  The more unbalanced the higher
the
    * computed cost will be.  This uses a rolling average of regionload.
    */
-  static class WriteRequestCostFunction extends CostFromRegionLoadFunction {
+  static class WriteRequestCostFunction extends CostFromRegionLoadAsRateFunction {
 
     private static final String WRITE_REQUEST_COST_KEY =
         "hbase.master.balancer.stochastic.writeRequestCost";
@@ -1521,7 +1544,7 @@ public class StochasticLoadBalancer extends BaseLoadBalancer {
    * Compute the cost of total memstore size.  The more unbalanced the higher the
    * computed cost will be.  This uses a rolling average of regionload.
    */
-  static class MemstoreSizeCostFunction extends CostFromRegionLoadFunction {
+  static class MemstoreSizeCostFunction extends CostFromRegionLoadAsRateFunction {
 
     private static final String MEMSTORE_SIZE_COST_KEY =
         "hbase.master.balancer.stochastic.memstoreSizeCost";

http://git-wip-us.apache.org/repos/asf/hbase/blob/19f9a1a6/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestStochasticLoadBalancer.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestStochasticLoadBalancer.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestStochasticLoadBalancer.java
index 094687b..3d975b8 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestStochasticLoadBalancer.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestStochasticLoadBalancer.java
@@ -231,6 +231,30 @@ public class TestStochasticLoadBalancer extends BalancerTestBase {
   }
 
   @Test
+  public void testRegionLoadCost() {
+    List<RegionLoad> regionLoads = new ArrayList<>();
+    for (int i = 1; i < 5; i++) {
+      RegionLoad regionLoad = mock(RegionLoad.class);
+      when(regionLoad.getReadRequestsCount()).thenReturn(new Long(i));
+      when(regionLoad.getStorefileSizeMB()).thenReturn(i);
+      regionLoads.add(regionLoad);
+    }
+
+    Configuration conf = HBaseConfiguration.create();
+    StochasticLoadBalancer.ReadRequestCostFunction readCostFunction =
+        new StochasticLoadBalancer.ReadRequestCostFunction(conf);
+    double rateResult = readCostFunction.getRegionLoadCost(regionLoads);
+    // read requests are treated as a rate so the average rate here is simply 1
+    assertEquals(1, rateResult, 0.01);
+
+    StochasticLoadBalancer.StoreFileCostFunction storeFileCostFunction =
+        new StochasticLoadBalancer.StoreFileCostFunction(conf);
+    double result = storeFileCostFunction.getRegionLoadCost(regionLoads);
+    // storefile size cost is simply an average of it's value over time
+    assertEquals(2.5, result, 0.01);
+  }
+
+  @Test
   public void testCostFromArray() {
     Configuration conf = HBaseConfiguration.create();
     StochasticLoadBalancer.CostFromRegionLoadFunction


Mime
View raw message