hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From te...@apache.org
Subject hbase git commit: HBASE-15529 Revert due to TestRegionRebalancing failing
Date Fri, 20 May 2016 19:56:46 GMT
Repository: hbase
Updated Branches:
  refs/heads/branch-1 b91a6199e -> 108a130f3


HBASE-15529 Revert due to TestRegionRebalancing failing


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/108a130f
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/108a130f
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/108a130f

Branch: refs/heads/branch-1
Commit: 108a130f38b30276ab9f1b767dc49d80d6f91369
Parents: b91a619
Author: tedyu <yuzhihong@gmail.com>
Authored: Fri May 20 12:56:48 2016 -0700
Committer: tedyu <yuzhihong@gmail.com>
Committed: Fri May 20 12:56:48 2016 -0700

----------------------------------------------------------------------
 .../hbase/master/balancer/BaseLoadBalancer.java |  2 +-
 .../master/balancer/StochasticLoadBalancer.java | 63 ++------------------
 .../hbase/master/balancer/BalancerTestBase.java |  1 -
 .../balancer/TestStochasticLoadBalancer.java    | 14 -----
 .../balancer/TestStochasticLoadBalancer2.java   |  6 +-
 5 files changed, 6 insertions(+), 80 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hbase/blob/108a130f/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.java
index 15c0fc2..b446172 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.java
@@ -66,7 +66,7 @@ import com.google.common.collect.Sets;
  *
  */
 public abstract class BaseLoadBalancer implements LoadBalancer {
-  protected static final int MIN_SERVER_BALANCE = 2;
+  private static final int MIN_SERVER_BALANCE = 2;
   private volatile boolean stopped = false;
 
   private static final List<HRegionInfo> EMPTY_REGION_LIST = new ArrayList<HRegionInfo>(0);

http://git-wip-us.apache.org/repos/asf/hbase/blob/108a130f/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.java
index d7895c4..ab075db 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.java
@@ -42,7 +42,6 @@ import org.apache.hadoop.hbase.ServerName;
 import org.apache.hadoop.hbase.TableName;
 import org.apache.hadoop.hbase.master.MasterServices;
 import org.apache.hadoop.hbase.master.RegionPlan;
-import org.apache.hadoop.hbase.master.balancer.BaseLoadBalancer.Cluster;
 import org.apache.hadoop.hbase.master.balancer.BaseLoadBalancer.Cluster.Action;
 import org.apache.hadoop.hbase.master.balancer.BaseLoadBalancer.Cluster.Action.Type;
 import org.apache.hadoop.hbase.master.balancer.BaseLoadBalancer.Cluster.AssignRegionAction;
@@ -108,8 +107,6 @@ public class StochasticLoadBalancer extends BaseLoadBalancer {
   protected static final String KEEP_REGION_LOADS =
       "hbase.master.balancer.stochastic.numRegionLoadsToRemember";
   private static final String TABLE_FUNCTION_SEP = "_";
-  protected static final String MIN_COST_NEED_BALANCE_KEY =
-      "hbase.master.balancer.stochastic.minCostNeedBalance";
 
   private static final Random RANDOM = new Random(System.currentTimeMillis());
   private static final Log LOG = LogFactory.getLog(StochasticLoadBalancer.class);
@@ -121,7 +118,6 @@ public class StochasticLoadBalancer extends BaseLoadBalancer {
   private int stepsPerRegion = 800;
   private long maxRunningTime = 30 * 1000 * 1; // 30 seconds.
   private int numRegionLoadsToRemember = 15;
-  private float minCostNeedBalance = 0.05f;
 
   private CandidateGenerator[] candidateGenerators;
   private CostFromRegionLoadFunction[] regionLoadFunctions;
@@ -167,8 +163,6 @@ public class StochasticLoadBalancer extends BaseLoadBalancer {
     numRegionLoadsToRemember = conf.getInt(KEEP_REGION_LOADS, numRegionLoadsToRemember);
     isByTable = conf.getBoolean(HConstants.HBASE_MASTER_LOADBALANCE_BYTABLE, isByTable);
 
-    minCostNeedBalance = conf.getFloat(MIN_COST_NEED_BALANCE_KEY, minCostNeedBalance);
-
     if (localityCandidateGenerator == null) {
       localityCandidateGenerator = new LocalityBasedCandidateGenerator(services);
     }
@@ -264,41 +258,6 @@ public class StochasticLoadBalancer extends BaseLoadBalancer {
   }
 
   @Override
-  protected boolean needsBalance(Cluster cluster) {
-    ClusterLoadState cs = new ClusterLoadState(cluster.clusterState);
-    if (cs.getNumServers() < MIN_SERVER_BALANCE) {
-      if (LOG.isDebugEnabled()) {
-        LOG.debug("Not running balancer because only " + cs.getNumServers()
-            + " active regionserver(s)");
-      }
-      return false;
-    }
-    if (areSomeRegionReplicasColocated(cluster)) {
-      return true;
-    }
-
-    double total = 0.0;
-    float sumMultiplier = 0.0f;
-    for (CostFunction c : costFunctions) {
-      float multiplier = c.getMultiplier();
-      if (multiplier <= 0) {
-        continue;
-      }
-      sumMultiplier += multiplier;
-      total += c.cost() * multiplier;
-    }
-
-    if (total <= 0 || sumMultiplier <= 0
-        || (sumMultiplier > 0 && (total / sumMultiplier) < minCostNeedBalance))
{
-      LOG.info("Skipping load balancing because balanced cluster; " + "total cost is " +
total
-          + ", sum multiplier is " + sumMultiplier + " min cost which need balance is "
-          + minCostNeedBalance);
-      return false;
-    }
-    return true;
-  }
-
-  @Override
   public synchronized List<RegionPlan> balanceCluster(TableName tableName, Map<ServerName,
     List<HRegionInfo>> clusterState) {
     this.tableName = tableName;
@@ -339,21 +298,19 @@ public class StochasticLoadBalancer extends BaseLoadBalancer {
     // Keep track of servers to iterate through them.
     Cluster cluster = new Cluster(clusterState, loads, finder, rackManager);
 
-    long startTime = EnvironmentEdgeManager.currentTime();
-
-    initCosts(cluster);
-
     if (!needsBalance(cluster)) {
       return null;
     }
 
+    long startTime = EnvironmentEdgeManager.currentTime();
+
+    initCosts(cluster);
+
     double currentCost = computeCost(cluster, Double.MAX_VALUE);
     curOverallCost = currentCost;
     for (int i = 0; i < this.curFunctionCosts.length; i++) {
       curFunctionCosts[i] = tempFunctionCosts[i];
     }
-    LOG.info("start StochasticLoadBalancer.balaner, initCost=" + currentCost + ", functionCost="
-        + functionCost());
 
     double initCost = currentCost;
     double newCost = currentCost;
@@ -450,18 +407,6 @@ public class StochasticLoadBalancer extends BaseLoadBalancer {
     }
   }
 
-  private String functionCost() {
-    StringBuilder builder = new StringBuilder();
-    for (CostFunction c:costFunctions) {
-      builder.append(c.getClass().getSimpleName());
-      builder.append(" : (");
-      builder.append(c.getMultiplier());
-      builder.append(", ");
-      builder.append(c.cost());
-      builder.append("); ");
-    }
-    return builder.toString();
-  }
 
   /**
    * Create all of the RegionPlan's needed to move from the initial cluster state to the
desired

http://git-wip-us.apache.org/repos/asf/hbase/blob/108a130f/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/BalancerTestBase.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/BalancerTestBase.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/BalancerTestBase.java
index 7ae0133..903f614 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/BalancerTestBase.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/BalancerTestBase.java
@@ -70,7 +70,6 @@ public class BalancerTestBase {
     conf.setFloat("hbase.master.balancer.stochastic.maxMovePercent", 0.75f);
     conf.setFloat("hbase.regions.slop", 0.0f);
     conf.setFloat("hbase.master.balancer.stochastic.localityCost", 0);
-    conf.setFloat("hbase.master.balancer.stochastic.minCostNeedBalance", 0.0f);
     loadBalancer = new StochasticLoadBalancer();
     loadBalancer.setConf(conf);
   }

http://git-wip-us.apache.org/repos/asf/hbase/blob/108a130f/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestStochasticLoadBalancer.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestStochasticLoadBalancer.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestStochasticLoadBalancer.java
index 9caf264..8300f32 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestStochasticLoadBalancer.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestStochasticLoadBalancer.java
@@ -93,20 +93,6 @@ public class TestStochasticLoadBalancer extends BalancerTestBase {
     }
   }
 
-  @Test
-  public void testNeedBalance() {
-    conf.setFloat("hbase.master.balancer.stochastic.minCostNeedBalance", 1.0f);
-    loadBalancer.setConf(conf);
-    for (int[] mockCluster : clusterStateMocks) {
-      Map<ServerName, List<HRegionInfo>> servers = mockClusterServers(mockCluster);
-      List<RegionPlan> plans = loadBalancer.balanceCluster(servers);
-      assertNull(plans);
-    }
-    // reset config
-    conf.setFloat("hbase.master.balancer.stochastic.minCostNeedBalance", 0.0f);
-    loadBalancer.setConf(conf);
-  }
-
   /**
    * Test the load balancing algorithm.
    *

http://git-wip-us.apache.org/repos/asf/hbase/blob/108a130f/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestStochasticLoadBalancer2.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestStochasticLoadBalancer2.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestStochasticLoadBalancer2.java
index da83114..723d368 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestStochasticLoadBalancer2.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestStochasticLoadBalancer2.java
@@ -48,7 +48,6 @@ public class TestStochasticLoadBalancer2 extends BalancerTestBase {
     conf.setLong(StochasticLoadBalancer.MAX_STEPS_KEY, 2000000L);
     conf.setLong("hbase.master.balancer.stochastic.maxRunningTime", 90 * 1000); // 90 sec
     conf.setFloat("hbase.master.balancer.stochastic.localityCost", 0);
-    conf.setFloat("hbase.master.balancer.stochastic.minCostNeedBalance", 0.05f);
     loadBalancer.setConf(conf);
     int numNodes = 1000;
     int numRegions = 20 * numNodes; // 20 * replication regions per RS
@@ -56,9 +55,6 @@ public class TestStochasticLoadBalancer2 extends BalancerTestBase {
     int numTables = 100;
     int replication = 3;
     testWithCluster(numNodes, numRegions, numRegionsPerServer, replication, numTables, true,
true);
-    // reset config
-    conf.setFloat("hbase.master.balancer.stochastic.minCostNeedBalance", 0.0f);
-    loadBalancer.setConf(conf);
   }
 
   @Test (timeout = 800000)
@@ -90,4 +86,4 @@ public class TestStochasticLoadBalancer2 extends BalancerTestBase {
     int numTables = 10;
     testWithCluster(numNodes, numRegions, numRegionsPerServer, replication, numTables, true,
false);
   }
-}
+}
\ No newline at end of file


Mime
View raw message