hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From te...@apache.org
Subject hbase git commit: HBASE-15515 Improve LocalityBasedCandidateGenerator in Balancer (Guanghao Zhang)
Date Thu, 24 Mar 2016 18:22:46 GMT
Repository: hbase
Updated Branches:
  refs/heads/branch-1 1361a25e6 -> ff261efbf


HBASE-15515 Improve LocalityBasedCandidateGenerator in Balancer (Guanghao Zhang)


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/ff261efb
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/ff261efb
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/ff261efb

Branch: refs/heads/branch-1
Commit: ff261efbf7ff8f4947dcb9387720b611acc86e71
Parents: 1361a25
Author: tedyu <yuzhihong@gmail.com>
Authored: Thu Mar 24 11:22:34 2016 -0700
Committer: tedyu <yuzhihong@gmail.com>
Committed: Thu Mar 24 11:22:34 2016 -0700

----------------------------------------------------------------------
 .../hbase/master/balancer/BaseLoadBalancer.java | 30 ++++++++++++++++----
 .../master/balancer/RegionLocationFinder.java   | 16 +++++++++++
 .../master/balancer/StochasticLoadBalancer.java |  6 ++--
 3 files changed, 42 insertions(+), 10 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hbase/blob/ff261efb/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.java
index 12beb1c..b446172 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.java
@@ -826,7 +826,7 @@ public abstract class BaseLoadBalancer implements LoadBalancer {
     int getLowestLocalityRegionOnServer(int serverIndex) {
       if (regionFinder != null) {
         float lowestLocality = 1.0f;
-        int lowestLocalityRegionIndex = 0;
+        int lowestLocalityRegionIndex = -1;
         if (regionsPerServer[serverIndex].length == 0) {
           // No regions on that region server
           return -1;
@@ -836,15 +836,24 @@ public abstract class BaseLoadBalancer implements LoadBalancer {
           HDFSBlocksDistribution distribution = regionFinder
               .getBlockDistribution(regions[regionIndex]);
           float locality = distribution.getBlockLocalityIndex(servers[serverIndex].getHostname());
+          // skip empty region
+          if (distribution.getUniqueBlocksTotalWeight() == 0) {
+            continue;
+          }
           if (locality < lowestLocality) {
             lowestLocality = locality;
             lowestLocalityRegionIndex = j;
           }
         }
+        if (lowestLocalityRegionIndex == -1) {
+          return -1;
+        }
         if (LOG.isTraceEnabled()) {
-          LOG.trace(" Lowest locality region index is " + lowestLocalityRegionIndex
-            + " and its region server contains " + regionsPerServer[serverIndex].length
-            + " regions");
+          LOG.trace("Lowest locality region is "
+              + regions[regionsPerServer[serverIndex][lowestLocalityRegionIndex]]
+                  .getRegionNameAsString() + " with locality " + lowestLocality
+              + " and its region server contains " + regionsPerServer[serverIndex].length
+              + " regions");
         }
         return regionsPerServer[serverIndex][lowestLocalityRegionIndex];
       } else {
@@ -861,9 +870,14 @@ public abstract class BaseLoadBalancer implements LoadBalancer {
       }
     }
 
-    int getLeastLoadedTopServerForRegion(int region) {
+    /**
+     * Returns a least loaded server which has better locality for this region
+     * than the current server.
+     */
+    int getLeastLoadedTopServerForRegion(int region, int currentServer) {
       if (regionFinder != null) {
-        List<ServerName> topLocalServers = regionFinder.getTopBlockLocations(regions[region]);
+        List<ServerName> topLocalServers = regionFinder.getTopBlockLocations(regions[region],
+          servers[currentServer].getHostname());
         int leastLoadedServerIndex = -1;
         int load = Integer.MAX_VALUE;
         for (ServerName sn : topLocalServers) {
@@ -880,6 +894,10 @@ public abstract class BaseLoadBalancer implements LoadBalancer {
             load = tempLoad;
           }
         }
+        if (leastLoadedServerIndex != -1) {
+          LOG.debug("Pick the least loaded server " + servers[leastLoadedServerIndex].getHostname()
+            + " with better locality for region " + regions[region]);
+        }
         return leastLoadedServerIndex;
       } else {
         return -1;

http://git-wip-us.apache.org/repos/asf/hbase/blob/ff261efb/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/RegionLocationFinder.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/RegionLocationFinder.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/RegionLocationFinder.java
index 9657e53..a6724ee 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/RegionLocationFinder.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/RegionLocationFinder.java
@@ -171,6 +171,22 @@ class RegionLocationFinder {
   }
 
   /**
+   * Returns an ordered list of hosts which have better locality for this region
+   * than the current host.
+   */
+  protected List<ServerName> getTopBlockLocations(HRegionInfo region, String currentHost)
{
+    HDFSBlocksDistribution blocksDistribution = getBlockDistribution(region);
+    List<String> topHosts = new ArrayList<String>();
+    for (String host : blocksDistribution.getTopHosts()) {
+      if (host.equals(currentHost)) {
+        break;
+      }
+      topHosts.add(host);
+    }
+    return mapHostNameToServerName(topHosts);
+  }
+
+  /**
    * Returns an ordered list of hosts that are hosting the blocks for this
    * region. The weight of each host is the sum of the block lengths of all
    * files on that host, so the first host in the list is the server which holds

http://git-wip-us.apache.org/repos/asf/hbase/blob/ff261efb/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.java
b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.java
index c686e48..ce9bf7c 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.java
@@ -705,9 +705,7 @@ public class StochasticLoadBalancer extends BaseLoadBalancer {
         return pickRandomRegions(cluster, thisServer, otherServer);
       }
 
-      cluster.calculateRegionServerLocalities();
-      // Pick server with lowest locality
-      int thisServer = pickLowestLocalityServer(cluster);
+      int thisServer = pickRandomServer(cluster);
       int thisRegion;
       if (thisServer == -1) {
         LOG.warn("Could not pick lowest locality region server");
@@ -722,7 +720,7 @@ public class StochasticLoadBalancer extends BaseLoadBalancer {
       }
 
       // Pick the least loaded server with good locality for the region
-      int otherServer = cluster.getLeastLoadedTopServerForRegion(thisRegion);
+      int otherServer = cluster.getLeastLoadedTopServerForRegion(thisRegion, thisServer);
 
       if (otherServer == -1) {
         return Cluster.NullAction;


Mime
View raw message