hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From els...@apache.org
Subject [37/50] [abbrv] hbase git commit: HBASE-17462 Use sliding window for read/write request costs in StochasticLoadBalancer (Tim Brown)
Date Mon, 23 Jan 2017 23:02:10 GMT
HBASE-17462 Use sliding window for read/write request costs in StochasticLoadBalancer (Tim
Brown)


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/7754a962
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/7754a962
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/7754a962

Branch: refs/heads/HBASE-16961
Commit: 7754a9620eff44d1d570fda534f9159a756310cd
Parents: f254e27
Author: tedyu <yuzhihong@gmail.com>
Authored: Sun Jan 22 18:35:38 2017 -0800
Committer: tedyu <yuzhihong@gmail.com>
Committed: Sun Jan 22 18:35:38 2017 -0800

----------------------------------------------------------------------
 .../master/balancer/StochasticLoadBalancer.java | 47 ++++++++++++++------
 .../balancer/TestStochasticLoadBalancer.java    | 24 ++++++++++
 2 files changed, 57 insertions(+), 14 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hbase/blob/7754a962/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.java
index b02aac1..4fbae6e 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.java
@@ -42,7 +42,6 @@ import org.apache.hadoop.hbase.ServerName;
 import org.apache.hadoop.hbase.TableName;
 import org.apache.hadoop.hbase.master.MasterServices;
 import org.apache.hadoop.hbase.master.RegionPlan;
-import org.apache.hadoop.hbase.master.balancer.BaseLoadBalancer.Cluster;
 import org.apache.hadoop.hbase.master.balancer.BaseLoadBalancer.Cluster.Action;
 import org.apache.hadoop.hbase.master.balancer.BaseLoadBalancer.Cluster.Action.Type;
 import org.apache.hadoop.hbase.master.balancer.BaseLoadBalancer.Cluster.AssignRegionAction;
@@ -1300,21 +1299,42 @@ public class StochasticLoadBalancer extends BaseLoadBalancer {
 
     protected double getRegionLoadCost(Collection<RegionLoad> regionLoadList) {
       double cost = 0;
-
       for (RegionLoad rl : regionLoadList) {
-        double toAdd = getCostFromRl(rl);
+        cost += getCostFromRl(rl);
+      }
+      return cost / regionLoadList.size();
+    }
+
+    protected abstract double getCostFromRl(RegionLoad rl);
+  }
 
-        if (cost == 0) {
-          cost = toAdd;
+  /**
+   * Class to be used for the subset of RegionLoad costs that should be treated as rates.
+   * We do not compare about the actual rate in requests per second but rather the rate relative
+   * to the rest of the regions.
+   */
+  abstract static class CostFromRegionLoadAsRateFunction extends CostFromRegionLoadFunction
{
+
+    CostFromRegionLoadAsRateFunction(Configuration conf) {
+      super(conf);
+    }
+
+    @Override
+    protected double getRegionLoadCost(Collection<RegionLoad> regionLoadList) {
+      double cost = 0;
+      double previous = 0;
+      boolean isFirst = true;
+      for (RegionLoad rl : regionLoadList) {
+        double current = getCostFromRl(rl);
+        if (isFirst) {
+          isFirst = false;
         } else {
-          cost = (.5 * cost) + (.5 * toAdd);
+          cost += current - previous;
         }
+        previous = current;
       }
-
-      return cost;
+      return Math.max(0, cost / (regionLoadList.size() - 1));
     }
-
-    protected abstract double getCostFromRl(RegionLoad rl);
   }
 
   /**
@@ -1322,7 +1342,7 @@ public class StochasticLoadBalancer extends BaseLoadBalancer {
    * computed cost will be.  This uses a rolling average of regionload.
    */
 
-  static class ReadRequestCostFunction extends CostFromRegionLoadFunction {
+  static class ReadRequestCostFunction extends CostFromRegionLoadAsRateFunction {
 
     private static final String READ_REQUEST_COST_KEY =
         "hbase.master.balancer.stochastic.readRequestCost";
@@ -1333,7 +1353,6 @@ public class StochasticLoadBalancer extends BaseLoadBalancer {
       this.setMultiplier(conf.getFloat(READ_REQUEST_COST_KEY, DEFAULT_READ_REQUEST_COST));
     }
 
-
     @Override
     protected double getCostFromRl(RegionLoad rl) {
       return rl.getReadRequestsCount();
@@ -1344,7 +1363,7 @@ public class StochasticLoadBalancer extends BaseLoadBalancer {
    * Compute the cost of total number of write requests.  The more unbalanced the higher
the
    * computed cost will be.  This uses a rolling average of regionload.
    */
-  static class WriteRequestCostFunction extends CostFromRegionLoadFunction {
+  static class WriteRequestCostFunction extends CostFromRegionLoadAsRateFunction {
 
     private static final String WRITE_REQUEST_COST_KEY =
         "hbase.master.balancer.stochastic.writeRequestCost";
@@ -1522,7 +1541,7 @@ public class StochasticLoadBalancer extends BaseLoadBalancer {
    * Compute the cost of total memstore size.  The more unbalanced the higher the
    * computed cost will be.  This uses a rolling average of regionload.
    */
-  static class MemstoreSizeCostFunction extends CostFromRegionLoadFunction {
+  static class MemstoreSizeCostFunction extends CostFromRegionLoadAsRateFunction {
 
     private static final String MEMSTORE_SIZE_COST_KEY =
         "hbase.master.balancer.stochastic.memstoreSizeCost";

http://git-wip-us.apache.org/repos/asf/hbase/blob/7754a962/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestStochasticLoadBalancer.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestStochasticLoadBalancer.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestStochasticLoadBalancer.java
index 094687b..3d975b8 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestStochasticLoadBalancer.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestStochasticLoadBalancer.java
@@ -231,6 +231,30 @@ public class TestStochasticLoadBalancer extends BalancerTestBase {
   }
 
   @Test
+  public void testRegionLoadCost() {
+    List<RegionLoad> regionLoads = new ArrayList<>();
+    for (int i = 1; i < 5; i++) {
+      RegionLoad regionLoad = mock(RegionLoad.class);
+      when(regionLoad.getReadRequestsCount()).thenReturn(new Long(i));
+      when(regionLoad.getStorefileSizeMB()).thenReturn(i);
+      regionLoads.add(regionLoad);
+    }
+
+    Configuration conf = HBaseConfiguration.create();
+    StochasticLoadBalancer.ReadRequestCostFunction readCostFunction =
+        new StochasticLoadBalancer.ReadRequestCostFunction(conf);
+    double rateResult = readCostFunction.getRegionLoadCost(regionLoads);
+    // read requests are treated as a rate so the average rate here is simply 1
+    assertEquals(1, rateResult, 0.01);
+
+    StochasticLoadBalancer.StoreFileCostFunction storeFileCostFunction =
+        new StochasticLoadBalancer.StoreFileCostFunction(conf);
+    double result = storeFileCostFunction.getRegionLoadCost(regionLoads);
+    // storefile size cost is simply an average of it's value over time
+    assertEquals(2.5, result, 0.01);
+  }
+
+  @Test
   public void testCostFromArray() {
     Configuration conf = HBaseConfiguration.create();
     StochasticLoadBalancer.CostFromRegionLoadFunction


Mime
View raw message