hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jxi...@apache.org
Subject [1/5] HBASE-11059 ZK-less region assignment
Date Sat, 14 Jun 2014 15:40:36 GMT
Repository: hbase
Updated Branches:
  refs/heads/master 047463c27 -> 58549428a


http://git-wip-us.apache.org/repos/asf/hbase/blob/58549428/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java
index 548f4d5..10c2335 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitTransactionOnCluster.java
@@ -38,6 +38,7 @@ import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hbase.Abortable;
+import org.apache.hadoop.hbase.CoordinatedStateManager;
 import org.apache.hadoop.hbase.Coprocessor;
 import org.apache.hadoop.hbase.HBaseIOException;
 import org.apache.hadoop.hbase.HBaseTestingUtility;
@@ -65,7 +66,6 @@ import org.apache.hadoop.hbase.client.Put;
 import org.apache.hadoop.hbase.client.Result;
 import org.apache.hadoop.hbase.client.ResultScanner;
 import org.apache.hadoop.hbase.client.Scan;
-import org.apache.hadoop.hbase.CoordinatedStateManager;
 import org.apache.hadoop.hbase.coordination.ZKSplitTransactionCoordination;
 import org.apache.hadoop.hbase.coordination.ZkCloseRegionCoordination;
 import org.apache.hadoop.hbase.coordination.ZkCoordinatedStateManager;
@@ -120,15 +120,24 @@ public class TestSplitTransactionOnCluster {
   private static volatile boolean secondSplit = false;
   private static volatile boolean callRollBack = false;
   private static volatile boolean firstSplitCompleted = false;
+  private static boolean useZKForAssignment = true;
 
-  private static final HBaseTestingUtility TESTING_UTIL =
+  static final HBaseTestingUtility TESTING_UTIL =
     new HBaseTestingUtility();
 
-  @BeforeClass public static void before() throws Exception {
+  static void setupOnce() throws Exception {
     TESTING_UTIL.getConfiguration().setInt("hbase.balancer.period", 60000);
+    useZKForAssignment = TESTING_UTIL.getConfiguration().getBoolean(
+      "hbase.assignment.usezk", false);
     TESTING_UTIL.startMiniCluster(NB_SERVERS);
   }
 
+  @BeforeClass public static void before() throws Exception {
+    // Use ZK for region assignment
+    TESTING_UTIL.getConfiguration().setBoolean("hbase.assignment.usezk", true);
+    setupOnce();
+  }
+
   @AfterClass public static void after() throws Exception {
     TESTING_UTIL.shutdownMiniCluster();
   }
@@ -175,6 +184,12 @@ public class TestSplitTransactionOnCluster {
   public void testShouldFailSplitIfZNodeDoesNotExistDueToPrevRollBack() throws Exception
{
     final TableName tableName =
         TableName.valueOf("testShouldFailSplitIfZNodeDoesNotExistDueToPrevRollBack");
+
+    if (!useZKForAssignment) {
+      // This test doesn't apply if not using ZK for assignment
+      return;
+    }
+
     try {
       // Create table then get the single region for our new table.
       HTable t = createTableAndWait(tableName.getName(), Bytes.toBytes("cf"));
@@ -304,7 +319,7 @@ public class TestSplitTransactionOnCluster {
       TESTING_UTIL.deleteTable(tableName);
     }
   }
-  @Test
+  @Test(timeout = 60000)
   @SuppressWarnings("deprecation")
   public void testSplitFailedCompactionAndSplit() throws Exception {
     final byte[] tableName = Bytes.toBytes("testSplitFailedCompactionAndSplit");
@@ -403,42 +418,46 @@ public class TestSplitTransactionOnCluster {
       AssignmentManager.TEST_SKIP_SPLIT_HANDLING = true;
       // Now try splitting and it should work.
       split(hri, server, regionCount);
-      // Get daughters
-      List<HRegion> daughters = checkAndGetDaughters(tableName);
-      // Assert the ephemeral node is up in zk.
+        // Assert the ephemeral node is up in zk.
       String path = ZKAssign.getNodeName(TESTING_UTIL.getZooKeeperWatcher(),
         hri.getEncodedName());
       RegionTransition rt = null;
       Stat stats = null;
-      // Wait till the znode moved to SPLIT
-      for (int i=0; i<100; i++) {
-        stats = TESTING_UTIL.getZooKeeperWatcher().getRecoverableZooKeeper().exists(path,
false);
-        rt = RegionTransition.parseFrom(ZKAssign.getData(TESTING_UTIL.getZooKeeperWatcher(),
-          hri.getEncodedName()));
-        if (rt.getEventType().equals(EventType.RS_ZK_REGION_SPLIT)) break;
-        Thread.sleep(100);
+      List<HRegion> daughters = null;
+      if (useZKForAssignment) {
+        daughters = checkAndGetDaughters(tableName);
+
+        // Wait till the znode moved to SPLIT
+        for (int i=0; i<100; i++) {
+          stats = TESTING_UTIL.getZooKeeperWatcher().getRecoverableZooKeeper().exists(path,
false);
+          rt = RegionTransition.parseFrom(ZKAssign.getData(TESTING_UTIL.getZooKeeperWatcher(),
+            hri.getEncodedName()));
+          if (rt.getEventType().equals(EventType.RS_ZK_REGION_SPLIT)) break;
+          Thread.sleep(100);
+        }
+        LOG.info("EPHEMERAL NODE BEFORE SERVER ABORT, path=" + path + ", stats=" + stats);
+        assertTrue(rt != null && rt.getEventType().equals(EventType.RS_ZK_REGION_SPLIT));
+        // Now crash the server
+        cluster.abortRegionServer(tableRegionIndex);
       }
-      LOG.info("EPHEMERAL NODE BEFORE SERVER ABORT, path=" + path + ", stats=" + stats);
-      assertTrue(rt != null && rt.getEventType().equals(EventType.RS_ZK_REGION_SPLIT));
-      // Now crash the server
-      cluster.abortRegionServer(tableRegionIndex);
       waitUntilRegionServerDead();
-      awaitDaughters(tableName, daughters.size());
+      awaitDaughters(tableName, 2);
+      if (useZKForAssignment) {
+        regions = cluster.getRegions(tableName);
+        for (HRegion r: regions) {
+          assertTrue(daughters.contains(r));
+        }
 
-      // Assert daughters are online.
-      regions = cluster.getRegions(tableName);
-      for (HRegion r: regions) {
-        assertTrue(daughters.contains(r));
-      }
-      // Finally assert that the ephemeral SPLIT znode was cleaned up.
-      for (int i=0; i<100; i++) {
-        // wait a bit (10s max) for the node to disappear
-        stats = TESTING_UTIL.getZooKeeperWatcher().getRecoverableZooKeeper().exists(path,
false);
-        if (stats == null) break;
-        Thread.sleep(100);
+        // Finally assert that the ephemeral SPLIT znode was cleaned up.
+        for (int i=0; i<100; i++) {
+          // wait a bit (10s max) for the node to disappear
+          stats = TESTING_UTIL.getZooKeeperWatcher().getRecoverableZooKeeper().exists(path,
false);
+          if (stats == null) break;
+          Thread.sleep(100);
+        }
+        LOG.info("EPHEMERAL NODE AFTER SERVER ABORT, path=" + path + ", stats=" + stats);
+        assertTrue(stats == null);
       }
-      LOG.info("EPHEMERAL NODE AFTER SERVER ABORT, path=" + path + ", stats=" + stats);
-      assertTrue(stats == null);
     } finally {
       // Set this flag back.
       AssignmentManager.TEST_SKIP_SPLIT_HANDLING = false;
@@ -461,6 +480,8 @@ public class TestSplitTransactionOnCluster {
 
     int tableRegionIndex = ensureTableRegionNotOnSameServerAsMeta(admin, hri);
 
+    RegionStates regionStates = cluster.getMaster().getAssignmentManager().getRegionStates();
+
     // Turn off balancer so it doesn't cut in and mess up our placements.
     this.admin.setBalancerRunning(false, true);
     // Turn off the meta scanner so it don't remove parent on us.
@@ -475,8 +496,12 @@ public class TestSplitTransactionOnCluster {
       // Insert into zk a blocking znode, a znode of same name as region
       // so it gets in way of our splitting.
       ServerName fakedServer = ServerName.valueOf("any.old.server", 1234, -1);
-      ZKAssign.createNodeClosing(TESTING_UTIL.getZooKeeperWatcher(),
-        hri, fakedServer);
+      if (useZKForAssignment) {
+        ZKAssign.createNodeClosing(TESTING_UTIL.getZooKeeperWatcher(),
+          hri, fakedServer);
+      } else {
+        regionStates.updateRegionState(hri, RegionState.State.CLOSING);
+      }
       // Now try splitting.... should fail.  And each should successfully
       // rollback.
       this.admin.split(hri.getRegionNameAsString());
@@ -488,9 +513,13 @@ public class TestSplitTransactionOnCluster {
         assertEquals(regionCount, ProtobufUtil.getOnlineRegions(
           server.getRSRpcServices()).size());
       }
-      // Now clear the zknode
-      ZKAssign.deleteClosingNode(TESTING_UTIL.getZooKeeperWatcher(),
-        hri, fakedServer);
+      if (useZKForAssignment) {
+        // Now clear the zknode
+        ZKAssign.deleteClosingNode(TESTING_UTIL.getZooKeeperWatcher(),
+          hri, fakedServer);
+      } else {
+        regionStates.regionOnline(hri, server.getServerName());
+      }
       // Now try splitting and it should work.
       split(hri, server, regionCount);
       // Get daughters
@@ -682,6 +711,11 @@ public class TestSplitTransactionOnCluster {
       KeeperException, DeserializationException, ServiceException {
     final byte[] tableName = Bytes.toBytes("testMasterRestartWhenSplittingIsPartial");
 
+    if (!useZKForAssignment) {
+      // This test doesn't apply if not using ZK for assignment
+      return;
+    }
+
     // Create table then get the single region for our new table.
     HTable t = createTableAndWait(tableName, HConstants.CATALOG_FAMILY);
     List<HRegion> regions = cluster.getRegions(tableName);
@@ -841,7 +875,7 @@ public class TestSplitTransactionOnCluster {
    * @throws InterruptedException
    * @throws KeeperException
    */
-  @Test
+  @Test(timeout = 60000)
   public void testSplitBeforeSettingSplittingInZK() throws Exception,
       InterruptedException, KeeperException {
     testSplitBeforeSettingSplittingInZKInternals();
@@ -912,7 +946,7 @@ public class TestSplitTransactionOnCluster {
    * If a table has regions that have no store files in a region, they should split successfully
    * into two regions with no store files.
    */
-  @Test
+  @Test(timeout = 60000)
   public void testSplitRegionWithNoStoreFiles()
       throws Exception {
     final TableName tableName =
@@ -1196,14 +1230,22 @@ public class TestSplitTransactionOnCluster {
   private void split(final HRegionInfo hri, final HRegionServer server, final int regionCount)
       throws IOException, InterruptedException {
     this.admin.split(hri.getRegionNameAsString());
-    for (int i = 0; ProtobufUtil.getOnlineRegions(
-        server.getRSRpcServices()).size() <= regionCount && i < 300; i++) {
-      LOG.debug("Waiting on region to split");
-      Thread.sleep(100);
-    }
+    try {
+      for (int i = 0; ProtobufUtil.getOnlineRegions(
+          server.getRSRpcServices()).size() <= regionCount && i < 300; i++)
{
+        LOG.debug("Waiting on region to split");
+        Thread.sleep(100);
+      }
 
-    assertFalse("Waited too long for split",
-      ProtobufUtil.getOnlineRegions(server.getRSRpcServices()).size() <= regionCount);
+      assertFalse("Waited too long for split",
+        ProtobufUtil.getOnlineRegions(server.getRSRpcServices()).size() <= regionCount);
+    } catch (RegionServerStoppedException e) {
+      if (useZKForAssignment) {
+        // If not using ZK for assignment, the exception may be expected.
+        LOG.error(e);
+        throw e;
+      }
+    }
   }
 
   /**

http://git-wip-us.apache.org/repos/asf/hbase/blob/58549428/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestZKLessMergeOnCluster.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestZKLessMergeOnCluster.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestZKLessMergeOnCluster.java
new file mode 100644
index 0000000..d9e8480
--- /dev/null
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestZKLessMergeOnCluster.java
@@ -0,0 +1,45 @@
+/**
+ * Copyright The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hadoop.hbase.regionserver;
+
+import org.apache.hadoop.hbase.LargeTests;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.experimental.categories.Category;
+
+/**
+ * Like {@link TestRegionMergeTransaction} in that we're testing
+ * {@link RegionMergeTransaction} only the below tests are against a running
+ * cluster where {@link TestRegionMergeTransaction} is tests against bare
+ * {@link HRegion}.
+ */
+@Category(LargeTests.class)
+public class TestZKLessMergeOnCluster extends TestRegionMergeTransactionOnCluster {
+  @BeforeClass
+  public static void beforeAllTests() throws Exception {
+    // Don't use ZK for region assignment
+    TEST_UTIL.getConfiguration().setBoolean("hbase.assignment.usezk", false);
+    setupOnce();
+  }
+
+  @AfterClass
+  public static void afterAllTests() throws Exception {
+    TestRegionMergeTransactionOnCluster.afterAllTests();
+  }
+}

http://git-wip-us.apache.org/repos/asf/hbase/blob/58549428/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestZKLessSplitOnCluster.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestZKLessSplitOnCluster.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestZKLessSplitOnCluster.java
new file mode 100644
index 0000000..1e3de64
--- /dev/null
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestZKLessSplitOnCluster.java
@@ -0,0 +1,45 @@
+/**
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.regionserver;
+
+import org.apache.hadoop.hbase.LargeTests;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.experimental.categories.Category;
+
+/**
+ * Like {@link TestSplitTransaction} in that we're testing {@link SplitTransaction}
+ * only the below tests are against a running cluster where {@link TestSplitTransaction}
+ * is tests against a bare {@link HRegion}.
+ */
+@Category(LargeTests.class)
+public class TestZKLessSplitOnCluster extends TestSplitTransactionOnCluster {
+  @BeforeClass
+  public static void before() throws Exception {
+    // Don't use ZK for region assignment
+    TESTING_UTIL.getConfiguration().setBoolean("hbase.assignment.usezk", false);
+    setupOnce();
+  }
+
+  @AfterClass
+  public static void after() throws Exception {
+    TestSplitTransactionOnCluster.after();
+  }
+}
+

http://git-wip-us.apache.org/repos/asf/hbase/blob/58549428/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/handler/TestCloseRegionHandler.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/handler/TestCloseRegionHandler.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/handler/TestCloseRegionHandler.java
index 43d5875..2990014 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/handler/TestCloseRegionHandler.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/handler/TestCloseRegionHandler.java
@@ -65,6 +65,7 @@ public class TestCloseRegionHandler {
   private int testIndex = 0;
 
   @BeforeClass public static void before() throws Exception {
+    HTU.getConfiguration().setBoolean("hbase.assignment.usezk", true);
     HTU.startMiniZKCluster();
   }
 

http://git-wip-us.apache.org/repos/asf/hbase/blob/58549428/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/handler/TestOpenRegionHandler.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/handler/TestOpenRegionHandler.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/handler/TestOpenRegionHandler.java
index 869d727..d472d57 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/handler/TestOpenRegionHandler.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/handler/TestOpenRegionHandler.java
@@ -56,6 +56,7 @@ public class TestOpenRegionHandler {
   private int testIndex = 0;
 
   @BeforeClass public static void before() throws Exception {
+    HTU.getConfiguration().setBoolean("hbase.assignment.usezk", true);
     HTU.startMiniZKCluster();
     TEST_HTD = new HTableDescriptor(TableName.valueOf("TestOpenRegionHandler.java"));
   }

http://git-wip-us.apache.org/repos/asf/hbase/blob/58549428/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java
index 134a953..0a2e8fd 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java
@@ -1630,6 +1630,12 @@ public class TestHBaseFsck {
       assertFalse(fsck.shouldRerun());
       assertErrors(fsck, new ERROR_CODE[] {
         ERROR_CODE.NOT_IN_META, ERROR_CODE.HOLE_IN_REGION_CHAIN});
+
+      // fix the cluster so other tests won't be impacted
+      fsck = doFsck(conf, true);
+      assertTrue(fsck.shouldRerun());
+      fsck = doFsck(conf, true);
+      assertNoErrors(fsck);
     } finally {
       deleteTable(table);
     }

http://git-wip-us.apache.org/repos/asf/hbase/blob/58549428/hbase-server/src/test/java/org/apache/hadoop/hbase/zookeeper/TestRecoverableZooKeeper.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/zookeeper/TestRecoverableZooKeeper.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/zookeeper/TestRecoverableZooKeeper.java
index f805ca4..7ed8215 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/zookeeper/TestRecoverableZooKeeper.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/zookeeper/TestRecoverableZooKeeper.java
@@ -69,7 +69,7 @@ public class TestRecoverableZooKeeper {
 
   @Test
   public void testSetDataVersionMismatchInLoop() throws Exception {
-    String znode = "/hbase/region-in-transition/9af7cfc9b15910a0b3d714bf40a3248f";
+    String znode = "/hbase/splitWAL/9af7cfc9b15910a0b3d714bf40a3248f";
     Configuration conf = TEST_UTIL.getConfiguration();
     Properties properties = ZKConfig.makeZKProps(conf);
     ZooKeeperWatcher zkw = new ZooKeeperWatcher(conf, "testSetDataVersionMismatchInLoop",


Mime
View raw message