hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From st...@apache.org
Subject git commit: HBASE-12149 TestRegionPlacement is failing undeterministically (Manukranth Kolloju)
Date Thu, 02 Oct 2014 18:30:14 GMT
Repository: hbase
Updated Branches:
  refs/heads/master 6f104e3ac -> 630a66f1d


HBASE-12149 TestRegionPlacement is failing undeterministically (Manukranth Kolloju)


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/630a66f1
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/630a66f1
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/630a66f1

Branch: refs/heads/master
Commit: 630a66f1d39eb792d0362c24f000adad2427d472
Parents: 6f104e3
Author: stack <stack@apache.org>
Authored: Thu Oct 2 11:29:44 2014 -0700
Committer: stack <stack@apache.org>
Committed: Thu Oct 2 11:29:44 2014 -0700

----------------------------------------------------------------------
 .../hbase/master/TestRegionPlacement.java       | 136 +------------
 .../hbase/master/TestRegionPlacement2.java      | 196 +++++++++++++++++++
 2 files changed, 200 insertions(+), 132 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hbase/blob/630a66f1/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestRegionPlacement.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestRegionPlacement.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestRegionPlacement.java
index c34fb48..5da9d03 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestRegionPlacement.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestRegionPlacement.java
@@ -32,13 +32,11 @@ import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Random;
-import java.util.Set;
 import java.util.concurrent.atomic.AtomicInteger;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.HBaseIOException;
 import org.apache.hadoop.hbase.HBaseTestingUtility;
 import org.apache.hadoop.hbase.HColumnDescriptor;
 import org.apache.hadoop.hbase.HConstants;
@@ -58,7 +56,6 @@ import org.apache.hadoop.hbase.master.balancer.FavoredNodeAssignmentHelper;
 import org.apache.hadoop.hbase.master.balancer.FavoredNodeLoadBalancer;
 import org.apache.hadoop.hbase.master.balancer.FavoredNodesPlan;
 import org.apache.hadoop.hbase.master.balancer.FavoredNodesPlan.Position;
-import org.apache.hadoop.hbase.master.balancer.LoadBalancerFactory;
 import org.apache.hadoop.hbase.regionserver.HRegion;
 import org.apache.hadoop.hbase.regionserver.HRegionServer;
 import org.apache.hadoop.hbase.testclassification.MasterTests;
@@ -83,9 +80,6 @@ public class TestRegionPlacement {
   private int REGION_NUM = 10;
   private Map<HRegionInfo, ServerName[]> favoredNodesAssignmentPlan =
       new HashMap<HRegionInfo, ServerName[]>();
-  private final static int PRIMARY = Position.PRIMARY.ordinal();
-  private final static int SECONDARY = Position.SECONDARY.ordinal();
-  private final static int TERTIARY = Position.TERTIARY.ordinal();
 
   @BeforeClass
   public static void setupBeforeClass() throws Exception {
@@ -105,111 +99,6 @@ public class TestRegionPlacement {
   }
 
   @Test
-  public void testFavoredNodesPresentForRoundRobinAssignment() throws HBaseIOException {
-    LoadBalancer balancer = LoadBalancerFactory.getLoadBalancer(TEST_UTIL.getConfiguration());
-    balancer.setMasterServices(TEST_UTIL.getMiniHBaseCluster().getMaster());
-    List<ServerName> servers = new ArrayList<ServerName>();
-    for (int i = 0; i < SLAVES; i++) {
-      ServerName server = TEST_UTIL.getMiniHBaseCluster().getRegionServer(i).getServerName();
-      servers.add(server);
-    }
-    List<HRegionInfo> regions = new ArrayList<HRegionInfo>(1);
-    HRegionInfo region = new HRegionInfo(TableName.valueOf("foobar"));
-    regions.add(region);
-    Map<ServerName,List<HRegionInfo>> assignmentMap = balancer.roundRobinAssignment(regions,
-        servers);
-    Set<ServerName> serverBefore = assignmentMap.keySet();
-    List<ServerName> favoredNodesBefore =
-        ((FavoredNodeLoadBalancer)balancer).getFavoredNodes(region);
-    assertTrue(favoredNodesBefore.size() == 3);
-    // the primary RS should be the one that the balancer's assignment returns
-    assertTrue(ServerName.isSameHostnameAndPort(serverBefore.iterator().next(),
-        favoredNodesBefore.get(PRIMARY)));
-    // now remove the primary from the list of available servers
-    List<ServerName> removedServers = removeMatchingServers(serverBefore, servers);
-    // call roundRobinAssignment with the modified servers list
-    assignmentMap = balancer.roundRobinAssignment(regions, servers);
-    List<ServerName> favoredNodesAfter =
-        ((FavoredNodeLoadBalancer)balancer).getFavoredNodes(region);
-    assertTrue(favoredNodesAfter.size() == 3);
-    // We don't expect the favored nodes assignments to change in multiple calls
-    // to the roundRobinAssignment method in the balancer (relevant for AssignmentManager.assign
-    // failures)
-    assertTrue(favoredNodesAfter.containsAll(favoredNodesBefore));
-    Set<ServerName> serverAfter = assignmentMap.keySet();
-    // We expect the new RegionServer assignee to be one of the favored nodes
-    // chosen earlier.
-    assertTrue(ServerName.isSameHostnameAndPort(serverAfter.iterator().next(),
-                 favoredNodesBefore.get(SECONDARY)) ||
-               ServerName.isSameHostnameAndPort(serverAfter.iterator().next(),
-                 favoredNodesBefore.get(TERTIARY)));
-
-    // put back the primary in the list of available servers
-    servers.addAll(removedServers);
-    // now roundRobinAssignment with the modified servers list should return the primary
-    // as the regionserver assignee
-    assignmentMap = balancer.roundRobinAssignment(regions, servers);
-    Set<ServerName> serverWithPrimary = assignmentMap.keySet();
-    assertTrue(serverBefore.containsAll(serverWithPrimary));
-
-    // Make all the favored nodes unavailable for assignment
-    removeMatchingServers(favoredNodesAfter, servers);
-    // call roundRobinAssignment with the modified servers list
-    assignmentMap = balancer.roundRobinAssignment(regions, servers);
-    List<ServerName> favoredNodesNow =
-        ((FavoredNodeLoadBalancer)balancer).getFavoredNodes(region);
-    assertTrue(favoredNodesNow.size() == 3);
-    assertTrue(!favoredNodesNow.contains(favoredNodesAfter.get(PRIMARY)) &&
-        !favoredNodesNow.contains(favoredNodesAfter.get(SECONDARY)) &&
-        !favoredNodesNow.contains(favoredNodesAfter.get(TERTIARY)));
-  }
-
-  @Test
-  public void testFavoredNodesPresentForRandomAssignment() throws HBaseIOException {
-    LoadBalancer balancer = LoadBalancerFactory.getLoadBalancer(TEST_UTIL.getConfiguration());
-    balancer.setMasterServices(TEST_UTIL.getMiniHBaseCluster().getMaster());
-    List<ServerName> servers = new ArrayList<ServerName>();
-    for (int i = 0; i < SLAVES; i++) {
-      ServerName server = TEST_UTIL.getMiniHBaseCluster().getRegionServer(i).getServerName();
-      servers.add(server);
-    }
-    List<HRegionInfo> regions = new ArrayList<HRegionInfo>(1);
-    HRegionInfo region = new HRegionInfo(TableName.valueOf("foobar"));
-    regions.add(region);
-    ServerName serverBefore = balancer.randomAssignment(region, servers);
-    List<ServerName> favoredNodesBefore =
-        ((FavoredNodeLoadBalancer)balancer).getFavoredNodes(region);
-    assertTrue(favoredNodesBefore.size() == 3);
-    // the primary RS should be the one that the balancer's assignment returns
-    assertTrue(ServerName.isSameHostnameAndPort(serverBefore,favoredNodesBefore.get(PRIMARY)));
-    // now remove the primary from the list of servers
-    removeMatchingServers(serverBefore, servers);
-    // call randomAssignment with the modified servers list
-    ServerName serverAfter = balancer.randomAssignment(region, servers);
-    List<ServerName> favoredNodesAfter =
-        ((FavoredNodeLoadBalancer)balancer).getFavoredNodes(region);
-    assertTrue(favoredNodesAfter.size() == 3);
-    // We don't expect the favored nodes assignments to change in multiple calls
-    // to the randomAssignment method in the balancer (relevant for AssignmentManager.assign
-    // failures)
-    assertTrue(favoredNodesAfter.containsAll(favoredNodesBefore));
-    // We expect the new RegionServer assignee to be one of the favored nodes
-    // chosen earlier.
-    assertTrue(ServerName.isSameHostnameAndPort(serverAfter, favoredNodesBefore.get(SECONDARY))
||
-               ServerName.isSameHostnameAndPort(serverAfter, favoredNodesBefore.get(TERTIARY)));
-    // Make all the favored nodes unavailable for assignment
-    removeMatchingServers(favoredNodesAfter, servers);
-    // call randomAssignment with the modified servers list
-    balancer.randomAssignment(region, servers);
-    List<ServerName> favoredNodesNow =
-        ((FavoredNodeLoadBalancer)balancer).getFavoredNodes(region);
-    assertTrue(favoredNodesNow.size() == 3);
-    assertTrue(!favoredNodesNow.contains(favoredNodesAfter.get(PRIMARY)) &&
-        !favoredNodesNow.contains(favoredNodesAfter.get(SECONDARY)) &&
-        !favoredNodesNow.contains(favoredNodesAfter.get(TERTIARY)));
-  }
-
-  @Test
   public void testRegionPlacement() throws Exception {
     String tableStr = "testRegionAssignment";
     TableName table = TableName.valueOf(tableStr);
@@ -501,27 +390,6 @@ public class TestRegionPlacement {
           regionMovement, expected);
   }
 
-  private List<ServerName> removeMatchingServers(ServerName serverWithoutStartCode,
-      List<ServerName> servers) {
-    List<ServerName> serversToRemove = new ArrayList<ServerName>();
-    for (ServerName s : servers) {
-      if (ServerName.isSameHostnameAndPort(s, serverWithoutStartCode)) {
-        serversToRemove.add(s);
-      }
-    }
-    servers.removeAll(serversToRemove);
-    return serversToRemove;
-  }
-
-  private List<ServerName> removeMatchingServers(Collection<ServerName> serversWithoutStartCode,
-      List<ServerName> servers) {
-    List<ServerName> serversToRemove = new ArrayList<ServerName>();
-    for (ServerName s : serversWithoutStartCode) {
-      serversToRemove.addAll(removeMatchingServers(s, servers));
-    }
-    return serversToRemove;
-  }
-
   /**
    * Verify the number of user regions is assigned to the primary
    * region server based on the plan is expected
@@ -601,6 +469,7 @@ public class TestRegionPlacement {
     MetaScannerVisitor visitor = new MetaScannerVisitor() {
       public boolean processRow(Result result) throws IOException {
         try {
+          @SuppressWarnings("deprecation")
           HRegionInfo info = MetaScanner.getHRegionInfo(result);
           if(info.getTable().getNamespaceAsString()
               .equals(NamespaceDescriptor.SYSTEM_NAMESPACE_NAME_STR)) {
@@ -680,9 +549,12 @@ public class TestRegionPlacement {
     desc.addFamily(new HColumnDescriptor(HConstants.CATALOG_FAMILY));
     admin.createTable(desc, splitKeys);
 
+    @SuppressWarnings("deprecation")
     HTable ht = new HTable(TEST_UTIL.getConfiguration(), tableName);
+    @SuppressWarnings("deprecation")
     Map<HRegionInfo, ServerName> regions = ht.getRegionLocations();
     assertEquals("Tried to create " + expectedRegions + " regions "
         + "but only found " + regions.size(), expectedRegions, regions.size());
+    ht.close();
   }
 }

http://git-wip-us.apache.org/repos/asf/hbase/blob/630a66f1/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestRegionPlacement2.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestRegionPlacement2.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestRegionPlacement2.java
new file mode 100644
index 0000000..3f34bc4
--- /dev/null
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestRegionPlacement2.java
@@ -0,0 +1,196 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.master;
+
+import static org.junit.Assert.*;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HBaseIOException;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.ServerName;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.master.balancer.FavoredNodeLoadBalancer;
+import org.apache.hadoop.hbase.master.balancer.LoadBalancerFactory;
+import org.apache.hadoop.hbase.master.balancer.FavoredNodesPlan.Position;
+import org.apache.hadoop.hbase.testclassification.MasterTests;
+import org.apache.hadoop.hbase.testclassification.MediumTests;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+@Category({MasterTests.class, MediumTests.class})
+public class TestRegionPlacement2 {
+  final static Log LOG = LogFactory.getLog(TestRegionPlacement2.class);
+  private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
+  private final static int SLAVES = 7;
+  private final static int PRIMARY = Position.PRIMARY.ordinal();
+  private final static int SECONDARY = Position.SECONDARY.ordinal();
+  private final static int TERTIARY = Position.TERTIARY.ordinal();
+
+  @BeforeClass
+  public static void setupBeforeClass() throws Exception {
+    Configuration conf = TEST_UTIL.getConfiguration();
+    // Enable the favored nodes based load balancer
+    conf.setClass(HConstants.HBASE_MASTER_LOADBALANCER_CLASS,
+        FavoredNodeLoadBalancer.class, LoadBalancer.class);
+    conf.setBoolean("hbase.tests.use.shortcircuit.reads", false);
+    TEST_UTIL.startMiniCluster(SLAVES);
+  }
+
+  @AfterClass
+  public static void tearDownAfterClass() throws Exception {
+    TEST_UTIL.shutdownMiniCluster();
+  }
+
+  @Test
+  public void testFavoredNodesPresentForRoundRobinAssignment() throws HBaseIOException {
+    LoadBalancer balancer = LoadBalancerFactory.getLoadBalancer(TEST_UTIL.getConfiguration());
+    balancer.setMasterServices(TEST_UTIL.getMiniHBaseCluster().getMaster());
+    List<ServerName> servers = new ArrayList<ServerName>();
+    for (int i = 0; i < SLAVES; i++) {
+      ServerName server = TEST_UTIL.getMiniHBaseCluster().getRegionServer(i).getServerName();
+      servers.add(server);
+    }
+    List<HRegionInfo> regions = new ArrayList<HRegionInfo>(1);
+    HRegionInfo region = new HRegionInfo(TableName.valueOf("foobar"));
+    regions.add(region);
+    Map<ServerName,List<HRegionInfo>> assignmentMap = balancer.roundRobinAssignment(regions,
+        servers);
+    Set<ServerName> serverBefore = assignmentMap.keySet();
+    List<ServerName> favoredNodesBefore =
+        ((FavoredNodeLoadBalancer)balancer).getFavoredNodes(region);
+    assertTrue(favoredNodesBefore.size() == 3);
+    // the primary RS should be the one that the balancer's assignment returns
+    assertTrue(ServerName.isSameHostnameAndPort(serverBefore.iterator().next(),
+        favoredNodesBefore.get(PRIMARY)));
+    // now remove the primary from the list of available servers
+    List<ServerName> removedServers = removeMatchingServers(serverBefore, servers);
+    // call roundRobinAssignment with the modified servers list
+    assignmentMap = balancer.roundRobinAssignment(regions, servers);
+    List<ServerName> favoredNodesAfter =
+        ((FavoredNodeLoadBalancer)balancer).getFavoredNodes(region);
+    assertTrue(favoredNodesAfter.size() == 3);
+    // We don't expect the favored nodes assignments to change in multiple calls
+    // to the roundRobinAssignment method in the balancer (relevant for AssignmentManager.assign
+    // failures)
+    assertTrue(favoredNodesAfter.containsAll(favoredNodesBefore));
+    Set<ServerName> serverAfter = assignmentMap.keySet();
+    // We expect the new RegionServer assignee to be one of the favored nodes
+    // chosen earlier.
+    assertTrue(ServerName.isSameHostnameAndPort(serverAfter.iterator().next(),
+                 favoredNodesBefore.get(SECONDARY)) ||
+               ServerName.isSameHostnameAndPort(serverAfter.iterator().next(),
+                 favoredNodesBefore.get(TERTIARY)));
+
+    // put back the primary in the list of available servers
+    servers.addAll(removedServers);
+    // now roundRobinAssignment with the modified servers list should return the primary
+    // as the regionserver assignee
+    assignmentMap = balancer.roundRobinAssignment(regions, servers);
+    Set<ServerName> serverWithPrimary = assignmentMap.keySet();
+    assertTrue(serverBefore.containsAll(serverWithPrimary));
+
+    // Make all the favored nodes unavailable for assignment
+    removeMatchingServers(favoredNodesAfter, servers);
+    // call roundRobinAssignment with the modified servers list
+    assignmentMap = balancer.roundRobinAssignment(regions, servers);
+    List<ServerName> favoredNodesNow =
+        ((FavoredNodeLoadBalancer)balancer).getFavoredNodes(region);
+    assertTrue(favoredNodesNow.size() == 3);
+    assertTrue(!favoredNodesNow.contains(favoredNodesAfter.get(PRIMARY)) &&
+        !favoredNodesNow.contains(favoredNodesAfter.get(SECONDARY)) &&
+        !favoredNodesNow.contains(favoredNodesAfter.get(TERTIARY)));
+  }
+
+  @Test
+  public void testFavoredNodesPresentForRandomAssignment() throws HBaseIOException {
+    LoadBalancer balancer = LoadBalancerFactory.getLoadBalancer(TEST_UTIL.getConfiguration());
+    balancer.setMasterServices(TEST_UTIL.getMiniHBaseCluster().getMaster());
+    List<ServerName> servers = new ArrayList<ServerName>();
+    for (int i = 0; i < SLAVES; i++) {
+      ServerName server = TEST_UTIL.getMiniHBaseCluster().getRegionServer(i).getServerName();
+      servers.add(server);
+    }
+    List<HRegionInfo> regions = new ArrayList<HRegionInfo>(1);
+    HRegionInfo region = new HRegionInfo(TableName.valueOf("foobar"));
+    regions.add(region);
+    ServerName serverBefore = balancer.randomAssignment(region, servers);
+    List<ServerName> favoredNodesBefore =
+        ((FavoredNodeLoadBalancer)balancer).getFavoredNodes(region);
+    assertTrue(favoredNodesBefore.size() == 3);
+    // the primary RS should be the one that the balancer's assignment returns
+    assertTrue(ServerName.isSameHostnameAndPort(serverBefore,favoredNodesBefore.get(PRIMARY)));
+    // now remove the primary from the list of servers
+    removeMatchingServers(serverBefore, servers);
+    // call randomAssignment with the modified servers list
+    ServerName serverAfter = balancer.randomAssignment(region, servers);
+    List<ServerName> favoredNodesAfter =
+        ((FavoredNodeLoadBalancer)balancer).getFavoredNodes(region);
+    assertTrue(favoredNodesAfter.size() == 3);
+    // We don't expect the favored nodes assignments to change in multiple calls
+    // to the randomAssignment method in the balancer (relevant for AssignmentManager.assign
+    // failures)
+    assertTrue(favoredNodesAfter.containsAll(favoredNodesBefore));
+    // We expect the new RegionServer assignee to be one of the favored nodes
+    // chosen earlier.
+    assertTrue(ServerName.isSameHostnameAndPort(serverAfter, favoredNodesBefore.get(SECONDARY))
||
+               ServerName.isSameHostnameAndPort(serverAfter, favoredNodesBefore.get(TERTIARY)));
+    // Make all the favored nodes unavailable for assignment
+    removeMatchingServers(favoredNodesAfter, servers);
+    // call randomAssignment with the modified servers list
+    balancer.randomAssignment(region, servers);
+    List<ServerName> favoredNodesNow =
+        ((FavoredNodeLoadBalancer)balancer).getFavoredNodes(region);
+    assertTrue(favoredNodesNow.size() == 3);
+    assertTrue(!favoredNodesNow.contains(favoredNodesAfter.get(PRIMARY)) &&
+        !favoredNodesNow.contains(favoredNodesAfter.get(SECONDARY)) &&
+        !favoredNodesNow.contains(favoredNodesAfter.get(TERTIARY)));
+  }
+
+  private List<ServerName> removeMatchingServers(Collection<ServerName> serversWithoutStartCode,
+      List<ServerName> servers) {
+    List<ServerName> serversToRemove = new ArrayList<ServerName>();
+    for (ServerName s : serversWithoutStartCode) {
+      serversToRemove.addAll(removeMatchingServers(s, servers));
+    }
+    return serversToRemove;
+  }
+
+  private List<ServerName> removeMatchingServers(ServerName serverWithoutStartCode,
+      List<ServerName> servers) {
+    List<ServerName> serversToRemove = new ArrayList<ServerName>();
+    for (ServerName s : servers) {
+      if (ServerName.isSameHostnameAndPort(s, serverWithoutStartCode)) {
+        serversToRemove.add(s);
+      }
+    }
+    servers.removeAll(serversToRemove);
+    return serversToRemove;
+  }
+}


Mime
View raw message