hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From e...@apache.org
Subject svn commit: r1573743 - in /hbase/branches/hbase-10070: ./ hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/ hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/
Date Mon, 03 Mar 2014 20:44:40 GMT
Author: enis
Date: Mon Mar  3 20:44:39 2014
New Revision: 1573743

URL: http://svn.apache.org/r1573743
Log:
HBASE-10632 Region lost in limbo after ArrayIndexOutOfBoundsException during assignment

Modified:
    hbase/branches/hbase-10070/   (props changed)
    hbase/branches/hbase-10070/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.java
    hbase/branches/hbase-10070/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.java
    hbase/branches/hbase-10070/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestBaseLoadBalancer.java

Propchange: hbase/branches/hbase-10070/
------------------------------------------------------------------------------
  Merged /hbase/trunk:r1573723

Modified: hbase/branches/hbase-10070/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.java
URL: http://svn.apache.org/viewvc/hbase/branches/hbase-10070/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.java?rev=1573743&r1=1573742&r2=1573743&view=diff
==============================================================================
--- hbase/branches/hbase-10070/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.java
(original)
+++ hbase/branches/hbase-10070/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.java
Mon Mar  3 20:44:39 2014
@@ -228,7 +228,13 @@ public abstract class BaseLoadBalancer i
           servers[serverIndex] = entry.getKey();
         }
 
-        regionsPerServer[serverIndex] = new int[entry.getValue().size()];
+        if (regionsPerServer[serverIndex] != null) {
+          // there is another server with the same hostAndPort in ClusterState.
+          // allocate the array for the total size
+          regionsPerServer[serverIndex] = new int[entry.getValue().size() + regionsPerServer[serverIndex].length];
+        } else {
+          regionsPerServer[serverIndex] = new int[entry.getValue().size()];
+        }
         primariesOfRegionsPerServer[serverIndex] = new int[entry.getValue().size()];
         serverIndicesSortedByRegionCount[serverIndex] = serverIndex;
       }
@@ -417,7 +423,7 @@ public abstract class BaseLoadBalancer i
         for (int i=0; i < loc.size(); i++) {
           regionLocations[regionIndex][i] =
               loc.get(i) == null ? -1 :
-                (serversToIndex.get(loc.get(i)) == null ? -1 : serversToIndex.get(loc.get(i)));
+                (serversToIndex.get(loc.get(i).getHostAndPort()) == null ? -1 : serversToIndex.get(loc.get(i).getHostAndPort()));
         }
       }
     }

Modified: hbase/branches/hbase-10070/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.java
URL: http://svn.apache.org/viewvc/hbase/branches/hbase-10070/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.java?rev=1573743&r1=1573742&r2=1573743&view=diff
==============================================================================
--- hbase/branches/hbase-10070/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.java
(original)
+++ hbase/branches/hbase-10070/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.java
Mon Mar  3 20:44:39 2014
@@ -553,7 +553,9 @@ public class StochasticLoadBalancer exte
         idx++;
       }
 
-      return idx;
+      return idx < regionLocations.length
+          ? regionLocations[idx]
+          : pickOtherRandomServer(cluster, thisServer);
     }
 
     void setServices(MasterServices services) {

Modified: hbase/branches/hbase-10070/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestBaseLoadBalancer.java
URL: http://svn.apache.org/viewvc/hbase/branches/hbase-10070/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestBaseLoadBalancer.java?rev=1573743&r1=1573742&r2=1573743&view=diff
==============================================================================
--- hbase/branches/hbase-10070/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestBaseLoadBalancer.java
(original)
+++ hbase/branches/hbase-10070/hbase-server/src/test/java/org/apache/hadoop/hbase/master/balancer/TestBaseLoadBalancer.java
Mon Mar  3 20:44:39 2014
@@ -19,15 +19,19 @@ package org.apache.hadoop.hbase.master.b
 
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertTrue;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
 
 import java.util.ArrayList;
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.HashMap;
 import java.util.Set;
 import java.util.TreeMap;
 import java.util.TreeSet;
 
+import org.apache.commons.lang.ArrayUtils;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
@@ -46,6 +50,8 @@ import org.junit.Test;
 import org.junit.experimental.categories.Category;
 import org.mockito.Mockito;
 
+import com.google.common.collect.Lists;
+
 @Category(MediumTests.class)
 public class TestBaseLoadBalancer extends BalancerTestBase {
 
@@ -382,4 +388,97 @@ public class TestBaseLoadBalancer extend
       }
     }
   }
+  @Test
+  public void testClusterServersWithSameHostPort() {
+    // tests whether the BaseLoadBalancer.Cluster can be constructed with servers
+    // sharing same host and port
+    List<ServerName> servers = getListOfServerNames(randomServers(10, 10));
+    List<HRegionInfo> regions = randomRegions(101);
+    Map<ServerName, List<HRegionInfo>> clusterState = new HashMap<ServerName,
List<HRegionInfo>>();
+
+    assignRegions(regions, servers, clusterState);
+
+    // construct another list of servers, but sharing same hosts and ports
+    List<ServerName> oldServers = new ArrayList<ServerName>(servers.size());
+    for (ServerName sn : servers) {
+      // The old server would have had same host and port, but different start code!
+      oldServers.add(ServerName.valueOf(sn.getHostname(), sn.getPort(), sn.getStartcode()
- 10));
+    }
+
+    regions = randomRegions(9); // some more regions
+    assignRegions(regions, oldServers, clusterState);
+
+    // should not throw exception:
+    BaseLoadBalancer.Cluster cluster = new Cluster(clusterState, null, null, null);
+    assertEquals(101 + 9, cluster.numRegions);
+    assertEquals(10, cluster.numServers); // only 10 servers because they share the same
host + port
+  }
+
+  private void assignRegions(List<HRegionInfo> regions, List<ServerName> servers,
+      Map<ServerName, List<HRegionInfo>> clusterState) {
+    for (int i = 0; i < regions.size(); i++) {
+      ServerName sn = servers.get(i % servers.size());
+      List<HRegionInfo> regionsOfServer = clusterState.get(sn);
+      if (regionsOfServer == null) {
+        regionsOfServer = new ArrayList<HRegionInfo>(10);
+        clusterState.put(sn, regionsOfServer);
+      }
+
+      regionsOfServer.add(regions.get(i));
+    }
+  }
+
+  @Test
+  public void testClusterRegionLocations() {
+    // tests whether region locations are handled correctly in Cluster
+    List<ServerName> servers = getListOfServerNames(randomServers(10, 10));
+    List<HRegionInfo> regions = randomRegions(101);
+    Map<ServerName, List<HRegionInfo>> clusterState = new HashMap<ServerName,
List<HRegionInfo>>();
+
+    assignRegions(regions, servers, clusterState);
+
+    // mock block locality for some regions
+    RegionLocationFinder locationFinder = mock(RegionLocationFinder.class);
+    // block locality: region:0   => {server:0}
+    //                 region:1   => {server:0, server:1}
+    //                 region:42 => {server:4, server:9, server:5}
+    when(locationFinder.getTopBlockLocations(regions.get(0))).thenReturn(
+      Lists.newArrayList(servers.get(0)));
+    when(locationFinder.getTopBlockLocations(regions.get(1))).thenReturn(
+      Lists.newArrayList(servers.get(0), servers.get(1)));
+    when(locationFinder.getTopBlockLocations(regions.get(42))).thenReturn(
+      Lists.newArrayList(servers.get(4), servers.get(9), servers.get(5)));
+
+    BaseLoadBalancer.Cluster cluster = new Cluster(clusterState, null, locationFinder, null);
+
+    int r0 = ArrayUtils.indexOf(cluster.regions, regions.get(0)); // this is ok, it is just
a test
+    int r1 = ArrayUtils.indexOf(cluster.regions, regions.get(1));
+    int r10 = ArrayUtils.indexOf(cluster.regions, regions.get(10));
+    int r42 = ArrayUtils.indexOf(cluster.regions, regions.get(42));
+
+    int s0 = cluster.serversToIndex.get(servers.get(0).getHostAndPort());
+    int s1 = cluster.serversToIndex.get(servers.get(1).getHostAndPort());
+    int s4 = cluster.serversToIndex.get(servers.get(4).getHostAndPort());
+    int s5 = cluster.serversToIndex.get(servers.get(5).getHostAndPort());
+    int s9 = cluster.serversToIndex.get(servers.get(9).getHostAndPort());
+
+    // region 0 locations
+    assertEquals(1, cluster.regionLocations[r0].length);
+    assertEquals(s0, cluster.regionLocations[r0][0]);
+
+    // region 1 locations
+    assertEquals(2, cluster.regionLocations[r1].length);
+    assertEquals(s0, cluster.regionLocations[r1][0]);
+    assertEquals(s1, cluster.regionLocations[r1][1]);
+
+    // region 10 locations
+    assertEquals(0, cluster.regionLocations[r10].length);
+
+    // region 42 locations
+    assertEquals(3, cluster.regionLocations[r42].length);
+    assertEquals(s4, cluster.regionLocations[r42][0]);
+    assertEquals(s9, cluster.regionLocations[r42][1]);
+    assertEquals(s5, cluster.regionLocations[r42][2]);
+  }
+
 }



Mime
View raw message