hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From st...@apache.org
Subject hbase git commit: HBASE-14290 Spin up less threads in tests
Date Tue, 25 Aug 2015 23:35:52 GMT
Repository: hbase
Updated Branches:
  refs/heads/branch-1.2 86145cf5e -> 08ca89658


HBASE-14290 Spin up less threads in tests

Conflicts:
	hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestDistributedLogSplitting.java


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/08ca8965
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/08ca8965
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/08ca8965

Branch: refs/heads/branch-1.2
Commit: 08ca896583bc30c8446df8396f13a18277582a8c
Parents: 86145cf
Author: stack <stack@apache.org>
Authored: Tue Aug 25 16:16:03 2015 -0700
Committer: stack <stack@apache.org>
Committed: Tue Aug 25 16:35:44 2015 -0700

----------------------------------------------------------------------
 .../org/apache/hadoop/hbase/ipc/AsyncCall.java  |    5 +-
 .../hadoop/hbase/ipc/AsyncRpcChannel.java       |    4 +-
 .../org/apache/hadoop/hbase/ipc/RpcServer.java  |    2 +-
 .../client/TestCloneSnapshotFromClient.java     |    1 -
 .../master/TestDistributedLogSplitting.java     | 1408 +++++++++---------
 .../hadoop/hbase/master/TestRollingRestart.java |    3 +-
 .../regionserver/TestRegionReplicaFailover.java |    1 -
 .../hbase/snapshot/TestExportSnapshot.java      |    1 -
 hbase-server/src/test/resources/hbase-site.xml  |    8 +
 9 files changed, 752 insertions(+), 681 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hbase/blob/08ca8965/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/AsyncCall.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/AsyncCall.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/AsyncCall.java
index ec1909a..431c669 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/AsyncCall.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/AsyncCall.java
@@ -86,8 +86,9 @@ public class AsyncCall extends DefaultPromise<Message> {
 
   @Override
   public String toString() {
-    return "callId: " + this.id + " methodName: " + this.method.getName() + " param {"
-        + (this.param != null ? ProtobufUtil.getShortTextFormat(this.param) : "") + "}";
+    return "callId=" + this.id + ", method=" + this.method.getName() +
+      ", rpcTimeout=" + this.rpcTimeout + ", param {" +
+      (this.param != null ? ProtobufUtil.getShortTextFormat(this.param) : "") + "}";
   }
 
   /**

http://git-wip-us.apache.org/repos/asf/hbase/blob/08ca8965/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/AsyncRpcChannel.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/AsyncRpcChannel.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/AsyncRpcChannel.java
index cfc8b1b..dbc39cf 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/AsyncRpcChannel.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/AsyncRpcChannel.java
@@ -146,10 +146,10 @@ public class AsyncRpcChannel {
 
     this.channel = connect(bootstrap).channel();
 
-    name = ("IPC Client (" + channel.hashCode() + ") connection to " +
+    name = ("IPC Client (" + channel.hashCode() + ") to " +
         address.toString() +
         ((ticket == null) ?
-            " from an unknown user" :
+            " from unknown user" :
             (" from " + ticket.getName())));
   }
 

http://git-wip-us.apache.org/repos/asf/hbase/blob/08ca8965/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/RpcServer.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/RpcServer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/RpcServer.java
index b1aa66c..0ee93eb 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/RpcServer.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/RpcServer.java
@@ -595,7 +595,7 @@ public class RpcServer implements RpcServerInterface {
         readers[i] = reader;
         readPool.execute(reader);
       }
-      LOG.info(getName() + ": started " + readThreads + " reader(s).");
+      LOG.info(getName() + ": started " + readThreads + " reader(s) listening on port=" + port);
 
       // Register accepts on the server socket with the selector.
       acceptChannel.register(selector, SelectionKey.OP_ACCEPT);

http://git-wip-us.apache.org/repos/asf/hbase/blob/08ca8965/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestCloneSnapshotFromClient.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestCloneSnapshotFromClient.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestCloneSnapshotFromClient.java
index 19e7410..d8a1e47 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestCloneSnapshotFromClient.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestCloneSnapshotFromClient.java
@@ -70,7 +70,6 @@ public class TestCloneSnapshotFromClient {
     TEST_UTIL.getConfiguration().setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 6);
     TEST_UTIL.getConfiguration().setBoolean(
         "hbase.master.enabletable.roundrobin", true);
-    TEST_UTIL.getConfiguration().setInt(HConstants.REGION_SERVER_HIGH_PRIORITY_HANDLER_COUNT, 40);
     TEST_UTIL.startMiniCluster(3);
   }
 

http://git-wip-us.apache.org/repos/asf/hbase/blob/08ca8965/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestDistributedLogSplitting.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestDistributedLogSplitting.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestDistributedLogSplitting.java
index 83dd123..2364861 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestDistributedLogSplitting.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestDistributedLogSplitting.java
@@ -71,7 +71,6 @@ import org.apache.hadoop.hbase.client.ClusterConnection;
 import org.apache.hadoop.hbase.client.ConnectionUtils;
 import org.apache.hadoop.hbase.client.Delete;
 import org.apache.hadoop.hbase.client.Get;
-import org.apache.hadoop.hbase.client.HTable;
 import org.apache.hadoop.hbase.client.Increment;
 import org.apache.hadoop.hbase.client.NonceGenerator;
 import org.apache.hadoop.hbase.client.PerClientRandomNonceGenerator;
@@ -84,6 +83,7 @@ import org.apache.hadoop.hbase.coordination.BaseCoordinatedStateManager;
 import org.apache.hadoop.hbase.coordination.ZKSplitLogManagerCoordination;
 import org.apache.hadoop.hbase.exceptions.OperationConflictException;
 import org.apache.hadoop.hbase.exceptions.RegionInRecoveryException;
+import org.apache.hadoop.hbase.ipc.ServerNotRunningYetException;
 import org.apache.hadoop.hbase.master.SplitLogManager.TaskBatch;
 import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
 import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.GetRegionInfoResponse.CompactionState;
@@ -104,11 +104,9 @@ import org.apache.hadoop.hbase.wal.WAL;
 import org.apache.hadoop.hbase.wal.WALFactory;
 import org.apache.hadoop.hbase.wal.WALSplitter;
 import org.apache.hadoop.hbase.zookeeper.MiniZooKeeperCluster;
-import org.apache.hadoop.hbase.zookeeper.ZKAssign;
 import org.apache.hadoop.hbase.zookeeper.ZKUtil;
 import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
 import org.apache.hadoop.hdfs.MiniDFSCluster;
-import org.apache.zookeeper.KeeperException;
 import org.junit.After;
 import org.junit.AfterClass;
 import org.junit.Assert;
@@ -117,7 +115,7 @@ import org.junit.BeforeClass;
 import org.junit.Test;
 import org.junit.experimental.categories.Category;
 
-@Category(LargeTests.class)
+@Category({LargeTests.class})
 @SuppressWarnings("deprecation")
 public class TestDistributedLogSplitting {
   private static final Log LOG = LogFactory.getLog(TestSplitLogManager.class);
@@ -168,7 +166,6 @@ public class TestDistributedLogSplitting {
     conf.setInt(HConstants.REGIONSERVER_INFO_PORT, -1);
     conf.setFloat(HConstants.LOAD_BALANCER_SLOP_KEY, (float) 100.0); // no load balancing
     conf.setInt("hbase.regionserver.wal.max.splitters", 3);
-    conf.setInt("hbase.master.maximum.ping.server.attempts", 3);
     conf.setInt(HConstants.REGION_SERVER_HIGH_PRIORITY_HANDLER_COUNT, 40);
     TEST_UTIL.shutdownMiniHBaseCluster();
     TEST_UTIL = new HBaseTestingUtility(conf);
@@ -180,7 +177,7 @@ public class TestDistributedLogSplitting {
     cluster.waitForActiveAndReadyMaster();
     master = cluster.getMaster();
     while (cluster.getLiveRegionServerThreads().size() < num_rs) {
-      Threads.sleep(1);
+      Threads.sleep(10);
     }
   }
 
@@ -223,69 +220,75 @@ public class TestDistributedLogSplitting {
 
     Path rootdir = FSUtils.getRootDir(conf);
 
-    installTable(new ZooKeeperWatcher(conf, "table-creation", null),
+    Table t = installTable(new ZooKeeperWatcher(conf, "table-creation", null),
         "table", "family", 40);
-    TableName table = TableName.valueOf("table");
-    List<HRegionInfo> regions = null;
-    HRegionServer hrs = null;
-    for (int i = 0; i < NUM_RS; i++) {
-      boolean foundRs = false;
-      hrs = rsts.get(i).getRegionServer();
-      regions = ProtobufUtil.getOnlineRegions(hrs.getRSRpcServices());
-      for (HRegionInfo region : regions) {
-        if (region.getTable().getNameAsString().equalsIgnoreCase("table")) {
-          foundRs = true;
-          break;
+    try {
+      TableName table = t.getName();
+      List<HRegionInfo> regions = null;
+      HRegionServer hrs = null;
+      for (int i = 0; i < NUM_RS; i++) {
+        boolean foundRs = false;
+        hrs = rsts.get(i).getRegionServer();
+        regions = ProtobufUtil.getOnlineRegions(hrs.getRSRpcServices());
+        for (HRegionInfo region : regions) {
+          if (region.getTable().getNameAsString().equalsIgnoreCase("table")) {
+            foundRs = true;
+            break;
+          }
         }
+        if (foundRs) break;
       }
-      if (foundRs) break;
-    }
-    final Path logDir = new Path(rootdir, DefaultWALProvider.getWALDirectoryName(hrs
-        .getServerName().toString()));
-
-    LOG.info("#regions = " + regions.size());
-    Iterator<HRegionInfo> it = regions.iterator();
-    while (it.hasNext()) {
-      HRegionInfo region = it.next();
-      if (region.getTable().getNamespaceAsString()
-          .equals(NamespaceDescriptor.SYSTEM_NAMESPACE_NAME_STR)) {
-        it.remove();
+      final Path logDir = new Path(rootdir, DefaultWALProvider.getWALDirectoryName(hrs
+          .getServerName().toString()));
+
+      LOG.info("#regions = " + regions.size());
+      Iterator<HRegionInfo> it = regions.iterator();
+      while (it.hasNext()) {
+        HRegionInfo region = it.next();
+        if (region.getTable().getNamespaceAsString()
+            .equals(NamespaceDescriptor.SYSTEM_NAMESPACE_NAME_STR)) {
+          it.remove();
+        }
       }
-    }
-    
-    makeWAL(hrs, regions, "table", "family", NUM_LOG_LINES, 100);
 
-    slm.splitLogDistributed(logDir);
+      makeWAL(hrs, regions, "table", "family", NUM_LOG_LINES, 100);
 
-    int count = 0;
-    for (HRegionInfo hri : regions) {
+      slm.splitLogDistributed(logDir);
 
-      Path tdir = FSUtils.getTableDir(rootdir, table);
-      Path editsdir =
-        WALSplitter.getRegionDirRecoveredEditsDir(HRegion.getRegionDir(tdir, hri.getEncodedName()));
-      LOG.debug("checking edits dir " + editsdir);
-      FileStatus[] files = fs.listStatus(editsdir, new PathFilter() {
-        @Override
-        public boolean accept(Path p) {
-          if (WALSplitter.isSequenceIdFile(p)) {
-            return false;
+      int count = 0;
+      for (HRegionInfo hri : regions) {
+
+        Path tdir = FSUtils.getTableDir(rootdir, table);
+        Path editsdir =
+          WALSplitter.getRegionDirRecoveredEditsDir(
+            HRegion.getRegionDir(tdir, hri.getEncodedName()));
+        LOG.debug("checking edits dir " + editsdir);
+        FileStatus[] files = fs.listStatus(editsdir, new PathFilter() {
+          @Override
+          public boolean accept(Path p) {
+            if (WALSplitter.isSequenceIdFile(p)) {
+              return false;
+            }
+            return true;
           }
-          return true;
+        });
+        assertTrue(
+          "edits dir should have more than a single file in it. instead has " + files.length,
+            files.length > 1);
+        for (int i = 0; i < files.length; i++) {
+          int c = countWAL(files[i].getPath(), fs, conf);
+          count += c;
         }
-      });
-      assertTrue("edits dir should have more than a single file in it. instead has " + files.length,
-          files.length > 1);
-      for (int i = 0; i < files.length; i++) {
-        int c = countWAL(files[i].getPath(), fs, conf);
-        count += c;
+        LOG.info(count + " edits in " + files.length + " recovered edits files.");
       }
-      LOG.info(count + " edits in " + files.length + " recovered edits files.");
-    }
 
-    // check that the log file is moved
-    assertFalse(fs.exists(logDir));
+      // check that the log file is moved
+      assertFalse(fs.exists(logDir));
 
-    assertEquals(NUM_LOG_LINES, count);
+      assertEquals(NUM_LOG_LINES, count);
+    } finally {
+      if (t != null) t.close();
+    }
   }
 
   @Test(timeout = 300000)
@@ -302,15 +305,17 @@ public class TestDistributedLogSplitting {
 
     final ZooKeeperWatcher zkw = new ZooKeeperWatcher(conf, "table-creation", null);
     Table ht = installTable(zkw, "table", "family", NUM_REGIONS_TO_CREATE);
+    try {
+      HRegionServer hrs = findRSToKill(false, "table");
+      List<HRegionInfo> regions = ProtobufUtil.getOnlineRegions(hrs.getRSRpcServices());
+      makeWAL(hrs, regions, "table", "family", NUM_LOG_LINES, 100);
 
-    HRegionServer hrs = findRSToKill(false, "table");
-    List<HRegionInfo> regions = ProtobufUtil.getOnlineRegions(hrs.getRSRpcServices());
-    makeWAL(hrs, regions, "table", "family", NUM_LOG_LINES, 100);
-
-    // wait for abort completes
-    this.abortRSAndVerifyRecovery(hrs, ht, zkw, NUM_REGIONS_TO_CREATE, NUM_LOG_LINES);
-    ht.close();
-    zkw.close();
+      // wait for abort completes
+      this.abortRSAndVerifyRecovery(hrs, ht, zkw, NUM_REGIONS_TO_CREATE, NUM_LOG_LINES);
+    } finally {
+      if (ht != null) ht.close();
+      if (zkw != null) zkw.close();
+    }
   }
 
   private static class NonceGeneratorWithDups extends PerClientRandomNonceGenerator {
@@ -344,10 +349,11 @@ public class TestDistributedLogSplitting {
     master.balanceSwitch(false);
 
     final ZooKeeperWatcher zkw = new ZooKeeperWatcher(conf, "table-creation", null);
-    HTable ht = installTable(zkw, TABLE_NAME, FAMILY_NAME, NUM_REGIONS_TO_CREATE);
+    Table ht = installTable(zkw, TABLE_NAME, FAMILY_NAME, NUM_REGIONS_TO_CREATE);
     NonceGeneratorWithDups ng = new NonceGeneratorWithDups();
     NonceGenerator oldNg =
-        ConnectionUtils.injectNonceGeneratorForTesting((ClusterConnection)ht.getConnection(), ng);
+        ConnectionUtils.injectNonceGeneratorForTesting(
+            (ClusterConnection)TEST_UTIL.getConnection(), ng);
 
     try {
       List<Increment> reqs = new ArrayList<Increment>();
@@ -381,9 +387,10 @@ public class TestDistributedLogSplitting {
         }
       }
     } finally {
-      ConnectionUtils.injectNonceGeneratorForTesting((ClusterConnection) ht.getConnection(), oldNg);
-      ht.close();
-      zkw.close();
+      ConnectionUtils.injectNonceGeneratorForTesting((ClusterConnection)
+              TEST_UTIL.getConnection(), oldNg);
+      if (ht != null) ht.close();
+      if (zkw != null) zkw.close();
     }
   }
 
@@ -400,14 +407,16 @@ public class TestDistributedLogSplitting {
 
     final ZooKeeperWatcher zkw = new ZooKeeperWatcher(conf, "table-creation", null);
     Table ht = installTable(zkw, "table", "family", NUM_REGIONS_TO_CREATE);
+    try {
+      HRegionServer hrs = findRSToKill(true, "table");
+      List<HRegionInfo> regions = ProtobufUtil.getOnlineRegions(hrs.getRSRpcServices());
+      makeWAL(hrs, regions, "table", "family", NUM_LOG_LINES, 100);
 
-    HRegionServer hrs = findRSToKill(true, "table");
-    List<HRegionInfo> regions = ProtobufUtil.getOnlineRegions(hrs.getRSRpcServices());
-    makeWAL(hrs, regions, "table", "family", NUM_LOG_LINES, 100);
-
-    this.abortRSAndVerifyRecovery(hrs, ht, zkw, NUM_REGIONS_TO_CREATE, NUM_LOG_LINES);
-    ht.close();
-    zkw.close();
+      this.abortRSAndVerifyRecovery(hrs, ht, zkw, NUM_REGIONS_TO_CREATE, NUM_LOG_LINES);
+    } finally {
+      if (ht != null) ht.close();
+      if (zkw != null) zkw.close();
+    }
   }
 
   private void abortRSAndVerifyRecovery(HRegionServer hrs, Table ht, final ZooKeeperWatcher zkw,
@@ -468,46 +477,47 @@ public class TestDistributedLogSplitting {
 
     final ZooKeeperWatcher zkw = new ZooKeeperWatcher(conf, "table-creation", null);
     Table ht = installTable(zkw, "table", "family", NUM_REGIONS_TO_CREATE);
+    try {
+      HRegionServer hrs = findRSToKill(false, "table");
+      List<HRegionInfo> regions = ProtobufUtil.getOnlineRegions(hrs.getRSRpcServices());
+      makeWAL(hrs, regions, "table", "family", NUM_LOG_LINES, 100);
 
-    HRegionServer hrs = findRSToKill(false, "table");
-    List<HRegionInfo> regions = ProtobufUtil.getOnlineRegions(hrs.getRSRpcServices());
-    makeWAL(hrs, regions, "table", "family", NUM_LOG_LINES, 100);
-
-    // abort master
-    abortMaster(cluster);
+      // abort master
+      abortMaster(cluster);
 
-    // abort RS
-    LOG.info("Aborting region server: " + hrs.getServerName());
-    hrs.abort("testing");
-
-    // wait for abort completes
-    TEST_UTIL.waitFor(120000, 200, new Waiter.Predicate<Exception>() {
-      @Override
-      public boolean evaluate() throws Exception {
-        return (cluster.getLiveRegionServerThreads().size() <= (NUM_RS - 1));
-      }
-    });
+      // abort RS
+      LOG.info("Aborting region server: " + hrs.getServerName());
+      hrs.abort("testing");
 
-    Thread.sleep(2000);
-    LOG.info("Current Open Regions:"
-        + HBaseTestingUtility.getAllOnlineRegions(cluster).size());
+      // wait for abort completes
+      TEST_UTIL.waitFor(120000, 200, new Waiter.Predicate<Exception>() {
+        @Override
+        public boolean evaluate() throws Exception {
+          return (cluster.getLiveRegionServerThreads().size() <= (NUM_RS - 1));
+        }
+      });
 
-    // wait for abort completes
-    TEST_UTIL.waitFor(120000, 200, new Waiter.Predicate<Exception>() {
-      @Override
-      public boolean evaluate() throws Exception {
-        return (HBaseTestingUtility.getAllOnlineRegions(cluster).size()
-          >= (NUM_REGIONS_TO_CREATE + 1));
-      }
-    });
+      Thread.sleep(2000);
+      LOG.info("Current Open Regions:"
+          + HBaseTestingUtility.getAllOnlineRegions(cluster).size());
 
-    LOG.info("Current Open Regions After Master Node Starts Up:"
-        + HBaseTestingUtility.getAllOnlineRegions(cluster).size());
+      // wait for abort completes
+      TEST_UTIL.waitFor(120000, 200, new Waiter.Predicate<Exception>() {
+        @Override
+        public boolean evaluate() throws Exception {
+          return (HBaseTestingUtility.getAllOnlineRegions(cluster).size()
+              >= (NUM_REGIONS_TO_CREATE + 1));
+        }
+      });
 
-    assertEquals(NUM_LOG_LINES, TEST_UTIL.countRows(ht));
+      LOG.info("Current Open Regions After Master Node Starts Up:"
+          + HBaseTestingUtility.getAllOnlineRegions(cluster).size());
 
-    ht.close();
-    zkw.close();
+      assertEquals(NUM_LOG_LINES, TEST_UTIL.countRows(ht));
+    } finally {
+      if (ht != null) ht.close();
+      if (zkw != null) zkw.close();
+    }
   }
 
   @Test(timeout = 300000)
@@ -525,50 +535,51 @@ public class TestDistributedLogSplitting {
 
     final ZooKeeperWatcher zkw = new ZooKeeperWatcher(conf, "table-creation", null);
     Table ht = installTable(zkw, "table", "family", NUM_REGIONS_TO_CREATE);
+    try {
+      HRegionServer hrs = findRSToKill(false, "table");
+      List<HRegionInfo> regions = ProtobufUtil.getOnlineRegions(hrs.getRSRpcServices());
+      makeWAL(hrs, regions, "table", "family", NUM_LOG_LINES, 100);
 
-    HRegionServer hrs = findRSToKill(false, "table");
-    List<HRegionInfo> regions = ProtobufUtil.getOnlineRegions(hrs.getRSRpcServices());
-    makeWAL(hrs, regions, "table", "family", NUM_LOG_LINES, 100);
-
-    // abort master
-    abortMaster(cluster);
+      // abort master
+      abortMaster(cluster);
 
-    // abort RS
-    LOG.info("Aborting region server: " + hrs.getServerName());
-    hrs.abort("testing");
+      // abort RS
+      LOG.info("Aborting region server: " + hrs.getServerName());
+      hrs.abort("testing");
 
-    // wait for the RS dies
-    TEST_UTIL.waitFor(120000, 200, new Waiter.Predicate<Exception>() {
-      @Override
-      public boolean evaluate() throws Exception {
-        return (cluster.getLiveRegionServerThreads().size() <= (NUM_RS - 1));
-      }
-    });
+      // wait for the RS dies
+      TEST_UTIL.waitFor(120000, 200, new Waiter.Predicate<Exception>() {
+        @Override
+        public boolean evaluate() throws Exception {
+          return (cluster.getLiveRegionServerThreads().size() <= (NUM_RS - 1));
+        }
+      });
 
-    Thread.sleep(2000);
-    LOG.info("Current Open Regions:" + HBaseTestingUtility.getAllOnlineRegions(cluster).size());
+      Thread.sleep(2000);
+      LOG.info("Current Open Regions:" + HBaseTestingUtility.getAllOnlineRegions(cluster).size());
 
-    // wait for all regions are fully recovered
-    TEST_UTIL.waitFor(180000, 200, new Waiter.Predicate<Exception>() {
-      @Override
-      public boolean evaluate() throws Exception {
-        List<String> recoveringRegions = zkw.getRecoverableZooKeeper().getChildren(
-          zkw.recoveringRegionsZNode, false);
-        boolean done = recoveringRegions != null && recoveringRegions.size() == 0;
-        if (!done) {
-          LOG.info("Recovering regions: " + recoveringRegions);
+      // wait for all regions are fully recovered
+      TEST_UTIL.waitFor(180000, 200, new Waiter.Predicate<Exception>() {
+        @Override
+        public boolean evaluate() throws Exception {
+          List<String> recoveringRegions = zkw.getRecoverableZooKeeper().getChildren(
+              zkw.recoveringRegionsZNode, false);
+          boolean done = recoveringRegions != null && recoveringRegions.size() == 0;
+          if (!done) {
+            LOG.info("Recovering regions: " + recoveringRegions);
+          }
+          return done;
         }
-        return done;
-      }
-    });
-
-    LOG.info("Current Open Regions After Master Node Starts Up:"
-        + HBaseTestingUtility.getAllOnlineRegions(cluster).size());
+      });
 
-    assertEquals(NUM_LOG_LINES, TEST_UTIL.countRows(ht));
+      LOG.info("Current Open Regions After Master Node Starts Up:"
+          + HBaseTestingUtility.getAllOnlineRegions(cluster).size());
 
-    ht.close();
-    zkw.close();
+      assertEquals(NUM_LOG_LINES, TEST_UTIL.countRows(ht));
+    } finally {
+     if (ht != null) ht.close();
+     if (zkw != null) zkw.close();
+    }
   }
 
 
@@ -586,72 +597,74 @@ public class TestDistributedLogSplitting {
     List<RegionServerThread> rsts = cluster.getLiveRegionServerThreads();
     final ZooKeeperWatcher zkw = new ZooKeeperWatcher(conf, "table-creation", null);
     Table ht = installTable(zkw, "table", "family", NUM_REGIONS_TO_CREATE);
+    try {
+      List<HRegionInfo> regions = null;
+      HRegionServer hrs1 = findRSToKill(false, "table");
+      regions = ProtobufUtil.getOnlineRegions(hrs1.getRSRpcServices());
 
-    List<HRegionInfo> regions = null;
-    HRegionServer hrs1 = findRSToKill(false, "table");
-    regions = ProtobufUtil.getOnlineRegions(hrs1.getRSRpcServices());
-
-    makeWAL(hrs1, regions, "table", "family", NUM_LOG_LINES, 100);
+      makeWAL(hrs1, regions, "table", "family", NUM_LOG_LINES, 100);
 
-    // abort RS1
-    LOG.info("Aborting region server: " + hrs1.getServerName());
-    hrs1.abort("testing");
+      // abort RS1
+      LOG.info("Aborting region server: " + hrs1.getServerName());
+      hrs1.abort("testing");
 
-    // wait for abort completes
-    TEST_UTIL.waitFor(120000, 200, new Waiter.Predicate<Exception>() {
-      @Override
-      public boolean evaluate() throws Exception {
-        return (cluster.getLiveRegionServerThreads().size() <= (NUM_RS - 1));
-      }
-    });
+      // wait for abort completes
+      TEST_UTIL.waitFor(120000, 200, new Waiter.Predicate<Exception>() {
+        @Override
+        public boolean evaluate() throws Exception {
+          return (cluster.getLiveRegionServerThreads().size() <= (NUM_RS - 1));
+        }
+      });
 
-    // wait for regions come online
-    TEST_UTIL.waitFor(180000, 200, new Waiter.Predicate<Exception>() {
-      @Override
-      public boolean evaluate() throws Exception {
-        return (HBaseTestingUtility.getAllOnlineRegions(cluster).size()
-            >= (NUM_REGIONS_TO_CREATE + 1));
-      }
-    });
+      // wait for regions come online
+      TEST_UTIL.waitFor(180000, 200, new Waiter.Predicate<Exception>() {
+        @Override
+        public boolean evaluate() throws Exception {
+          return (HBaseTestingUtility.getAllOnlineRegions(cluster).size()
+              >= (NUM_REGIONS_TO_CREATE + 1));
+        }
+      });
 
-    // sleep a little bit in order to interrupt recovering in the middle
-    Thread.sleep(300);
-    // abort second region server
-    rsts = cluster.getLiveRegionServerThreads();
-    HRegionServer hrs2 = rsts.get(0).getRegionServer();
-    LOG.info("Aborting one more region server: " + hrs2.getServerName());
-    hrs2.abort("testing");
+      // sleep a little bit in order to interrupt recovering in the middle
+      Thread.sleep(300);
+      // abort second region server
+      rsts = cluster.getLiveRegionServerThreads();
+      HRegionServer hrs2 = rsts.get(0).getRegionServer();
+      LOG.info("Aborting one more region server: " + hrs2.getServerName());
+      hrs2.abort("testing");
 
-    // wait for abort completes
-    TEST_UTIL.waitFor(120000, 200, new Waiter.Predicate<Exception>() {
-      @Override
-      public boolean evaluate() throws Exception {
-        return (cluster.getLiveRegionServerThreads().size() <= (NUM_RS - 2));
-      }
-    });
+      // wait for abort completes
+      TEST_UTIL.waitFor(120000, 200, new Waiter.Predicate<Exception>() {
+        @Override
+        public boolean evaluate() throws Exception {
+          return (cluster.getLiveRegionServerThreads().size() <= (NUM_RS - 2));
+        }
+      });
 
-    // wait for regions come online
-    TEST_UTIL.waitFor(180000, 200, new Waiter.Predicate<Exception>() {
-      @Override
-      public boolean evaluate() throws Exception {
-        return (HBaseTestingUtility.getAllOnlineRegions(cluster).size()
-            >= (NUM_REGIONS_TO_CREATE + 1));
-      }
-    });
+      // wait for regions come online
+      TEST_UTIL.waitFor(180000, 200, new Waiter.Predicate<Exception>() {
+        @Override
+        public boolean evaluate() throws Exception {
+          return (HBaseTestingUtility.getAllOnlineRegions(cluster).size()
+              >= (NUM_REGIONS_TO_CREATE + 1));
+        }
+      });
 
-    // wait for all regions are fully recovered
-    TEST_UTIL.waitFor(180000, 200, new Waiter.Predicate<Exception>() {
-      @Override
-      public boolean evaluate() throws Exception {
-        List<String> recoveringRegions = zkw.getRecoverableZooKeeper().getChildren(
-          zkw.recoveringRegionsZNode, false);
-        return (recoveringRegions != null && recoveringRegions.size() == 0);
-      }
-    });
+      // wait for all regions are fully recovered
+      TEST_UTIL.waitFor(180000, 200, new Waiter.Predicate<Exception>() {
+        @Override
+        public boolean evaluate() throws Exception {
+          List<String> recoveringRegions = zkw.getRecoverableZooKeeper().getChildren(
+              zkw.recoveringRegionsZNode, false);
+          return (recoveringRegions != null && recoveringRegions.size() == 0);
+        }
+      });
 
-    assertEquals(NUM_LOG_LINES, TEST_UTIL.countRows(ht));
-    ht.close();
-    zkw.close();
+      assertEquals(NUM_LOG_LINES, TEST_UTIL.countRows(ht));
+    } finally {
+      if (ht != null) ht.close();
+      if (zkw != null) zkw.close();
+    }
   }
 
   @Test(timeout = 300000)
@@ -663,41 +676,45 @@ public class TestDistributedLogSplitting {
     List<RegionServerThread> rsts = cluster.getLiveRegionServerThreads();
     final ZooKeeperWatcher zkw = master.getZooKeeper();
     Table ht = installTable(zkw, "table", "family", 40);
-    final SplitLogManager slm = master.getMasterFileSystem().splitLogManager;
-
-    Set<HRegionInfo> regionSet = new HashSet<HRegionInfo>();
-    HRegionInfo region = null;
-    HRegionServer hrs = null;
-    ServerName firstFailedServer = null;
-    ServerName secondFailedServer = null;
-    for (int i = 0; i < NUM_RS; i++) {
-      hrs = rsts.get(i).getRegionServer();
-      List<HRegionInfo> regions = ProtobufUtil.getOnlineRegions(hrs.getRSRpcServices());
-      if (regions.isEmpty()) continue;
-      region = regions.get(0);
-      regionSet.add(region);
-      firstFailedServer = hrs.getServerName();
-      secondFailedServer = rsts.get((i + 1) % NUM_RS).getRegionServer().getServerName();
-      break;
-    }
+    try {
+      final SplitLogManager slm = master.getMasterFileSystem().splitLogManager;
+
+      Set<HRegionInfo> regionSet = new HashSet<HRegionInfo>();
+      HRegionInfo region = null;
+      HRegionServer hrs = null;
+      ServerName firstFailedServer = null;
+      ServerName secondFailedServer = null;
+      for (int i = 0; i < NUM_RS; i++) {
+        hrs = rsts.get(i).getRegionServer();
+        List<HRegionInfo> regions = ProtobufUtil.getOnlineRegions(hrs.getRSRpcServices());
+        if (regions.isEmpty()) continue;
+        region = regions.get(0);
+        regionSet.add(region);
+        firstFailedServer = hrs.getServerName();
+        secondFailedServer = rsts.get((i + 1) % NUM_RS).getRegionServer().getServerName();
+        break;
+      }
 
-    slm.markRegionsRecovering(firstFailedServer, regionSet);
-    slm.markRegionsRecovering(secondFailedServer, regionSet);
+      slm.markRegionsRecovering(firstFailedServer, regionSet);
+      slm.markRegionsRecovering(secondFailedServer, regionSet);
 
-    List<String> recoveringRegions = ZKUtil.listChildrenNoWatch(zkw,
-      ZKUtil.joinZNode(zkw.recoveringRegionsZNode, region.getEncodedName()));
+      List<String> recoveringRegions = ZKUtil.listChildrenNoWatch(zkw,
+          ZKUtil.joinZNode(zkw.recoveringRegionsZNode, region.getEncodedName()));
 
-    assertEquals(recoveringRegions.size(), 2);
+      assertEquals(recoveringRegions.size(), 2);
 
-    // wait for splitLogWorker to mark them up because there is no WAL files recorded in ZK
-    final HRegionServer tmphrs = hrs;
-    TEST_UTIL.waitFor(60000, 1000, new Waiter.Predicate<Exception>() {
-      @Override
-      public boolean evaluate() throws Exception {
-        return (tmphrs.getRecoveringRegions().size() == 0);
-      }
-    });
-    ht.close();
+      // wait for splitLogWorker to mark them up because there is no WAL files recorded in ZK
+      final HRegionServer tmphrs = hrs;
+      TEST_UTIL.waitFor(60000, 1000, new Waiter.Predicate<Exception>() {
+        @Override
+        public boolean evaluate() throws Exception {
+          return (tmphrs.getRecoveringRegions().size() == 0);
+        }
+      });
+    } finally {
+      if (ht != null) ht.close();
+      if (zkw != null) zkw.close();
+    }
   }
 
   @Test(timeout = 300000)
@@ -712,37 +729,38 @@ public class TestDistributedLogSplitting {
 
     List<RegionServerThread> rsts = cluster.getLiveRegionServerThreads();
     final ZooKeeperWatcher zkw = new ZooKeeperWatcher(conf, "table-creation", null);
-    HTable ht = installTable(zkw, "table", "family", NUM_REGIONS_TO_CREATE);
-
-    List<HRegionInfo> regions = null;
-    HRegionServer hrs = null;
-    for (int i = 0; i < NUM_RS; i++) {
-      boolean isCarryingMeta = false;
-      hrs = rsts.get(i).getRegionServer();
-      regions = ProtobufUtil.getOnlineRegions(hrs.getRSRpcServices());
-      for (HRegionInfo region : regions) {
-        if (region.isMetaRegion()) {
-          isCarryingMeta = true;
-          break;
+    Table ht = installTable(zkw, "table", "family", NUM_REGIONS_TO_CREATE);
+    try {
+      List<HRegionInfo> regions = null;
+      HRegionServer hrs = null;
+      for (int i = 0; i < NUM_RS; i++) {
+        boolean isCarryingMeta = false;
+        hrs = rsts.get(i).getRegionServer();
+        regions = ProtobufUtil.getOnlineRegions(hrs.getRSRpcServices());
+        for (HRegionInfo region : regions) {
+          if (region.isMetaRegion()) {
+            isCarryingMeta = true;
+            break;
+          }
         }
+        if (isCarryingMeta) {
+          continue;
+        }
+        if (regions.size() > 0) break;
       }
-      if (isCarryingMeta) {
-        continue;
-      }
-      if (regions.size() > 0) break;
-    }
-
-    this.prepareData(ht, Bytes.toBytes("family"), Bytes.toBytes("c1"));
-    String originalCheckSum = TEST_UTIL.checksumRows(ht);
 
-    // abort RA and trigger replay
-    abortRSAndWaitForRecovery(hrs, zkw, NUM_REGIONS_TO_CREATE);
+      this.prepareData(ht, Bytes.toBytes("family"), Bytes.toBytes("c1"));
+      String originalCheckSum = TEST_UTIL.checksumRows(ht);
 
-    assertEquals("Data should remain after reopening of regions", originalCheckSum,
-      TEST_UTIL.checksumRows(ht));
+      // abort RA and trigger replay
+      abortRSAndWaitForRecovery(hrs, zkw, NUM_REGIONS_TO_CREATE);
 
-    ht.close();
-    zkw.close();
+      assertEquals("Data should remain after reopening of regions", originalCheckSum,
+          TEST_UTIL.checksumRows(ht));
+    } finally {
+     if (ht != null) ht.close();
+     if (zkw != null) zkw.close();
+    }
   }
 
   @Test(timeout = 300000)
@@ -757,134 +775,139 @@ public class TestDistributedLogSplitting {
     final ZooKeeperWatcher zkw = new ZooKeeperWatcher(conf, "table-creation", null);
     Table disablingHT = installTable(zkw, "disableTable", "family", NUM_REGIONS_TO_CREATE);
     Table ht = installTable(zkw, "table", "family", NUM_REGIONS_TO_CREATE, NUM_REGIONS_TO_CREATE);
-
-    // turn off load balancing to prevent regions from moving around otherwise
-    // they will consume recovered.edits
-    master.balanceSwitch(false);
-
-    List<HRegionInfo> regions = null;
-    HRegionServer hrs = null;
-    boolean hasRegionsForBothTables = false;
-    String tableName = null;
-    for (int i = 0; i < NUM_RS; i++) {
-      tableName = null;
-      hasRegionsForBothTables = false;
-      boolean isCarryingSystem = false;
-      hrs = rsts.get(i).getRegionServer();
-      regions = ProtobufUtil.getOnlineRegions(hrs.getRSRpcServices());
-      for (HRegionInfo region : regions) {
-        if (region.getTable().isSystemTable()) {
-          isCarryingSystem = true;
-          break;
+    try {
+      // turn off load balancing to prevent regions from moving around otherwise
+      // they will consume recovered.edits
+      master.balanceSwitch(false);
+
+      List<HRegionInfo> regions = null;
+      HRegionServer hrs = null;
+      boolean hasRegionsForBothTables = false;
+      String tableName = null;
+      for (int i = 0; i < NUM_RS; i++) {
+        tableName = null;
+        hasRegionsForBothTables = false;
+        boolean isCarryingSystem = false;
+        hrs = rsts.get(i).getRegionServer();
+        regions = ProtobufUtil.getOnlineRegions(hrs.getRSRpcServices());
+        for (HRegionInfo region : regions) {
+          if (region.getTable().isSystemTable()) {
+            isCarryingSystem = true;
+            break;
+          }
+          if (tableName != null &&
+              !tableName.equalsIgnoreCase(region.getTable().getNameAsString())) {
+            // make sure that we find a RS has online regions for both "table" and "disableTable"
+            hasRegionsForBothTables = true;
+            break;
+          } else if (tableName == null) {
+            tableName = region.getTable().getNameAsString();
+          }
         }
-        if (tableName != null &&
-            !tableName.equalsIgnoreCase(region.getTable().getNameAsString())) {
-          // make sure that we find a RS has online regions for both "table" and "disableTable"
-          hasRegionsForBothTables = true;
+        if (isCarryingSystem) {
+          continue;
+        }
+        if (hasRegionsForBothTables) {
           break;
-        } else if (tableName == null) {
-          tableName = region.getTable().getNameAsString();
         }
       }
-      if (isCarryingSystem) {
-        continue;
-      }
-      if (hasRegionsForBothTables) {
-        break;
-      }
-    }
 
-    // make sure we found a good RS
-    Assert.assertTrue(hasRegionsForBothTables);
+      // make sure we found a good RS
+      Assert.assertTrue(hasRegionsForBothTables);
 
-    LOG.info("#regions = " + regions.size());
-    Iterator<HRegionInfo> it = regions.iterator();
-    while (it.hasNext()) {
-      HRegionInfo region = it.next();
-      if (region.isMetaTable()) {
-        it.remove();
+      LOG.info("#regions = " + regions.size());
+      Iterator<HRegionInfo> it = regions.iterator();
+      while (it.hasNext()) {
+        HRegionInfo region = it.next();
+        if (region.isMetaTable()) {
+          it.remove();
+        }
       }
-    }
-    makeWAL(hrs, regions, "disableTable", "family", NUM_LOG_LINES, 100, false);
-    makeWAL(hrs, regions, "table", "family", NUM_LOG_LINES, 100);
+      makeWAL(hrs, regions, "disableTable", "family", NUM_LOG_LINES, 100, false);
+      makeWAL(hrs, regions, "table", "family", NUM_LOG_LINES, 100);
 
-    LOG.info("Disabling table\n");
-    TEST_UTIL.getHBaseAdmin().disableTable(TableName.valueOf("disableTable"));
+      LOG.info("Disabling table\n");
+      TEST_UTIL.getHBaseAdmin().disableTable(TableName.valueOf("disableTable"));
+      TEST_UTIL.waitTableDisabled(TableName.valueOf("disableTable").getName());
 
-    // abort RS
-    LOG.info("Aborting region server: " + hrs.getServerName());
-    hrs.abort("testing");
+      // abort RS
+      LOG.info("Aborting region server: " + hrs.getServerName());
+      hrs.abort("testing");
 
-    // wait for abort completes
-    TEST_UTIL.waitFor(120000, 200, new Waiter.Predicate<Exception>() {
-      @Override
-      public boolean evaluate() throws Exception {
-        return (cluster.getLiveRegionServerThreads().size() <= (NUM_RS - 1));
-      }
-    });
-
-    // wait for regions come online
-    TEST_UTIL.waitFor(180000, 200, new Waiter.Predicate<Exception>() {
-      @Override
-      public boolean evaluate() throws Exception {
-        return (HBaseTestingUtility.getAllOnlineRegions(cluster).size()
-            >= (NUM_REGIONS_TO_CREATE + 1));
-      }
-    });
+      // wait for abort completes
+      TEST_UTIL.waitFor(120000, 200, new Waiter.Predicate<Exception>() {
+        @Override
+        public boolean evaluate() throws Exception {
+          return (cluster.getLiveRegionServerThreads().size() <= (NUM_RS - 1));
+        }
+      });
 
-    // wait for all regions are fully recovered
-    TEST_UTIL.waitFor(180000, 200, new Waiter.Predicate<Exception>() {
-      @Override
-      public boolean evaluate() throws Exception {
-        List<String> recoveringRegions = zkw.getRecoverableZooKeeper().getChildren(
-          zkw.recoveringRegionsZNode, false);
-        ServerManager serverManager = master.getServerManager();
-        return (!serverManager.areDeadServersInProgress() &&
-            recoveringRegions != null && recoveringRegions.size() == 0);
-      }
-    });
+      // wait for regions come online
+      TEST_UTIL.waitFor(180000, 200, new Waiter.Predicate<Exception>() {
+        @Override
+        public boolean evaluate() throws Exception {
+          return (HBaseTestingUtility.getAllOnlineRegions(cluster).size()
+              >= (NUM_REGIONS_TO_CREATE + 1));
+        }
+      });
 
-    int count = 0;
-    FileSystem fs = master.getMasterFileSystem().getFileSystem();
-    Path rootdir = FSUtils.getRootDir(conf);
-    Path tdir = FSUtils.getTableDir(rootdir, TableName.valueOf("disableTable"));
-    for (HRegionInfo hri : regions) {
-      Path editsdir =
-        WALSplitter.getRegionDirRecoveredEditsDir(HRegion.getRegionDir(tdir, hri.getEncodedName()));
-      LOG.debug("checking edits dir " + editsdir);
-      if(!fs.exists(editsdir)) continue;
-      FileStatus[] files = fs.listStatus(editsdir, new PathFilter() {
+      // wait for all regions are fully recovered
+      TEST_UTIL.waitFor(180000, 200, new Waiter.Predicate<Exception>() {
         @Override
-        public boolean accept(Path p) {
-          if (WALSplitter.isSequenceIdFile(p)) {
-            return false;
-          }
-          return true;
+        public boolean evaluate() throws Exception {
+          List<String> recoveringRegions = zkw.getRecoverableZooKeeper().getChildren(
+              zkw.recoveringRegionsZNode, false);
+          ServerManager serverManager = master.getServerManager();
+          return (!serverManager.areDeadServersInProgress() &&
+              recoveringRegions != null && recoveringRegions.size() == 0);
         }
       });
-      if(files != null) {
-        for(FileStatus file : files) {
-          int c = countWAL(file.getPath(), fs, conf);
-          count += c;
-          LOG.info(c + " edits in " + file.getPath());
+
+      int count = 0;
+      FileSystem fs = master.getMasterFileSystem().getFileSystem();
+      Path rootdir = FSUtils.getRootDir(conf);
+      Path tdir = FSUtils.getTableDir(rootdir, TableName.valueOf("disableTable"));
+      for (HRegionInfo hri : regions) {
+        Path editsdir =
+          WALSplitter.getRegionDirRecoveredEditsDir(
+            HRegion.getRegionDir(tdir, hri.getEncodedName()));
+        LOG.debug("checking edits dir " + editsdir);
+        if(!fs.exists(editsdir)) continue;
+        FileStatus[] files = fs.listStatus(editsdir, new PathFilter() {
+          @Override
+          public boolean accept(Path p) {
+            if (WALSplitter.isSequenceIdFile(p)) {
+              return false;
+            }
+            return true;
+          }
+        });
+        if(files != null) {
+          for(FileStatus file : files) {
+            int c = countWAL(file.getPath(), fs, conf);
+            count += c;
+            LOG.info(c + " edits in " + file.getPath());
+          }
         }
       }
-    }
-
-    LOG.info("Verify edits in recovered.edits files");
-    assertEquals(NUM_LOG_LINES, count);
-    LOG.info("Verify replayed edits");
-    assertEquals(NUM_LOG_LINES, TEST_UTIL.countRows(ht));
 
-    // clean up
-    for (HRegionInfo hri : regions) {
-      Path editsdir =
-        WALSplitter.getRegionDirRecoveredEditsDir(HRegion.getRegionDir(tdir, hri.getEncodedName()));
-      fs.delete(editsdir, true);
+      LOG.info("Verify edits in recovered.edits files");
+      assertEquals(NUM_LOG_LINES, count);
+      LOG.info("Verify replayed edits");
+      assertEquals(NUM_LOG_LINES, TEST_UTIL.countRows(ht));
+
+      // clean up
+      for (HRegionInfo hri : regions) {
+        Path editsdir =
+          WALSplitter.getRegionDirRecoveredEditsDir(
+            HRegion.getRegionDir(tdir, hri.getEncodedName()));
+        fs.delete(editsdir, true);
+      }
+      disablingHT.close();
+    } finally {
+      if (ht != null) ht.close();
+      if (zkw != null) zkw.close();
     }
-    disablingHT.close();
-    ht.close();
-    zkw.close();
   }
 
   @Test(timeout = 300000)
@@ -902,65 +925,66 @@ public class TestDistributedLogSplitting {
     List<RegionServerThread> rsts = cluster.getLiveRegionServerThreads();
     final ZooKeeperWatcher zkw = new ZooKeeperWatcher(conf, "table-creation", null);
     Table ht = installTable(zkw, "table", "family", NUM_REGIONS_TO_CREATE);
-    final SplitLogManager slm = master.getMasterFileSystem().splitLogManager;
-
-    Set<HRegionInfo> regionSet = new HashSet<HRegionInfo>();
-    HRegionInfo region = null;
-    HRegionServer hrs = null;
-    HRegionServer dstRS = null;
-    for (int i = 0; i < NUM_RS; i++) {
-      hrs = rsts.get(i).getRegionServer();
-      List<HRegionInfo> regions = ProtobufUtil.getOnlineRegions(hrs.getRSRpcServices());
-      if (regions.isEmpty()) continue;
-      region = regions.get(0);
-      if (region.isMetaRegion()) continue;
-      regionSet.add(region);
-      dstRS = rsts.get((i+1) % NUM_RS).getRegionServer();
-      break;
-    }
-
-    slm.markRegionsRecovering(hrs.getServerName(), regionSet);
-    // move region in order for the region opened in recovering state
-    final HRegionInfo hri = region;
-    final HRegionServer tmpRS = dstRS;
-    TEST_UTIL.getHBaseAdmin().move(region.getEncodedNameAsBytes(),
-      Bytes.toBytes(dstRS.getServerName().getServerName()));
-    // wait for region move completes
-    final RegionStates regionStates =
-        TEST_UTIL.getHBaseCluster().getMaster().getAssignmentManager().getRegionStates();
-    TEST_UTIL.waitFor(45000, 200, new Waiter.Predicate<Exception>() {
-      @Override
-      public boolean evaluate() throws Exception {
-        ServerName sn = regionStates.getRegionServerOfRegion(hri);
-        return (sn != null && sn.equals(tmpRS.getServerName()));
-      }
-    });
-
     try {
-      byte[] key = region.getStartKey();
-      if (key == null || key.length == 0) {
-        key = new byte[] { 0, 0, 0, 0, 1 };
+      final SplitLogManager slm = master.getMasterFileSystem().splitLogManager;
+
+      Set<HRegionInfo> regionSet = new HashSet<HRegionInfo>();
+      HRegionInfo region = null;
+      HRegionServer hrs = null;
+      HRegionServer dstRS = null;
+      for (int i = 0; i < NUM_RS; i++) {
+        hrs = rsts.get(i).getRegionServer();
+        List<HRegionInfo> regions = ProtobufUtil.getOnlineRegions(hrs.getRSRpcServices());
+        if (regions.isEmpty()) continue;
+        region = regions.get(0);
+        regionSet.add(region);
+        dstRS = rsts.get((i+1) % NUM_RS).getRegionServer();
+        break;
       }
-      Put put = new Put(key);
-      put.add(Bytes.toBytes("family"), Bytes.toBytes("c1"), new byte[]{'b'});
-      ht.put(put);
-    } catch (IOException ioe) {
-      Assert.assertTrue(ioe instanceof RetriesExhaustedWithDetailsException);
-      RetriesExhaustedWithDetailsException re = (RetriesExhaustedWithDetailsException) ioe;
-      boolean foundRegionInRecoveryException = false;
-      for (Throwable t : re.getCauses()) {
-        if (t instanceof RegionInRecoveryException) {
-          foundRegionInRecoveryException = true;
-          break;
+
+      slm.markRegionsRecovering(hrs.getServerName(), regionSet);
+      // move region in order for the region opened in recovering state
+      final HRegionInfo hri = region;
+      final HRegionServer tmpRS = dstRS;
+      TEST_UTIL.getHBaseAdmin().move(region.getEncodedNameAsBytes(),
+          Bytes.toBytes(dstRS.getServerName().getServerName()));
+      // wait for region move completes
+      final RegionStates regionStates =
+          TEST_UTIL.getHBaseCluster().getMaster().getAssignmentManager().getRegionStates();
+      TEST_UTIL.waitFor(45000, 200, new Waiter.Predicate<Exception>() {
+        @Override
+        public boolean evaluate() throws Exception {
+          ServerName sn = regionStates.getRegionServerOfRegion(hri);
+          return (sn != null && sn.equals(tmpRS.getServerName()));
+        }
+      });
+
+      try {
+        byte[] key = region.getStartKey();
+        if (key == null || key.length == 0) {
+          key = new byte[] { 0, 0, 0, 0, 1 };
+        }
+        Put put = new Put(key);
+        put.add(Bytes.toBytes("family"), Bytes.toBytes("c1"), new byte[]{'b'});
+        ht.put(put);
+      } catch (IOException ioe) {
+        Assert.assertTrue(ioe instanceof RetriesExhaustedWithDetailsException);
+        RetriesExhaustedWithDetailsException re = (RetriesExhaustedWithDetailsException) ioe;
+        boolean foundRegionInRecoveryException = false;
+        for (Throwable t : re.getCauses()) {
+          if (t instanceof RegionInRecoveryException) {
+            foundRegionInRecoveryException = true;
+            break;
+          }
         }
+        Assert.assertTrue(
+            "No RegionInRecoveryException. Following exceptions returned=" + re.getCauses(),
+            foundRegionInRecoveryException);
       }
-      Assert.assertTrue(
-        "No RegionInRecoveryException. Following exceptions returned=" + re.getCauses(),
-        foundRegionInRecoveryException);
+    } finally {
+      if (ht != null) ht.close();
+      if (ht != null) zkw.close();
     }
-
-    ht.close();
-    zkw.close();
   }
 
   /**
@@ -986,48 +1010,51 @@ public class TestDistributedLogSplitting {
     final Path logDir = new Path(rootdir,
         DefaultWALProvider.getWALDirectoryName(hrs.getServerName().toString()));
 
-    installTable(new ZooKeeperWatcher(conf, "table-creation", null),
+    Table t = installTable(new ZooKeeperWatcher(conf, "table-creation", null),
         "table", "family", 40);
+    try {
+      makeWAL(hrs, ProtobufUtil.getOnlineRegions(hrs.getRSRpcServices()),
+          "table", "family", NUM_LOG_LINES, 100);
 
-    makeWAL(hrs, ProtobufUtil.getOnlineRegions(hrs.getRSRpcServices()),
-      "table", "family", NUM_LOG_LINES, 100);
-
-    new Thread() {
-      @Override
-      public void run() {
-        waitForCounter(tot_wkr_task_acquired, 0, 1, 1000);
-        for (RegionServerThread rst : rsts) {
-          rst.getRegionServer().abort("testing");
-          break;
+      new Thread() {
+        @Override
+        public void run() {
+          waitForCounter(tot_wkr_task_acquired, 0, 1, 1000);
+          for (RegionServerThread rst : rsts) {
+            rst.getRegionServer().abort("testing");
+            break;
+          }
         }
-      }
-    }.start();
-    // slm.splitLogDistributed(logDir);
-    FileStatus[] logfiles = fs.listStatus(logDir);
-    TaskBatch batch = new TaskBatch();
-    slm.enqueueSplitTask(logfiles[0].getPath().toString(), batch);
-    //waitForCounter but for one of the 2 counters
-    long curt = System.currentTimeMillis();
-    long waitTime = 80000;
-    long endt = curt + waitTime;
-    while (curt < endt) {
-      if ((tot_wkr_task_resigned.get() + tot_wkr_task_err.get() +
-          tot_wkr_final_transition_failed.get() + tot_wkr_task_done.get() +
-          tot_wkr_preempt_task.get()) == 0) {
-        Thread.yield();
-        curt = System.currentTimeMillis();
-      } else {
-        assertTrue(1 <= (tot_wkr_task_resigned.get() + tot_wkr_task_err.get() +
+      }.start();
+      // slm.splitLogDistributed(logDir);
+      FileStatus[] logfiles = fs.listStatus(logDir);
+      TaskBatch batch = new TaskBatch();
+      slm.enqueueSplitTask(logfiles[0].getPath().toString(), batch);
+      //waitForCounter but for one of the 2 counters
+      long curt = System.currentTimeMillis();
+      long waitTime = 80000;
+      long endt = curt + waitTime;
+      while (curt < endt) {
+        if ((tot_wkr_task_resigned.get() + tot_wkr_task_err.get() +
             tot_wkr_final_transition_failed.get() + tot_wkr_task_done.get() +
-            tot_wkr_preempt_task.get()));
-        return;
+            tot_wkr_preempt_task.get()) == 0) {
+          Thread.yield();
+          curt = System.currentTimeMillis();
+        } else {
+          assertTrue(1 <= (tot_wkr_task_resigned.get() + tot_wkr_task_err.get() +
+              tot_wkr_final_transition_failed.get() + tot_wkr_task_done.get() +
+              tot_wkr_preempt_task.get()));
+          return;
+        }
       }
+      fail("none of the following counters went up in " + waitTime +
+          " milliseconds - " +
+          "tot_wkr_task_resigned, tot_wkr_task_err, " +
+          "tot_wkr_final_transition_failed, tot_wkr_task_done, " +
+          "tot_wkr_preempt_task");
+    } finally {
+      if (t != null) t.close();
     }
-    fail("none of the following counters went up in " + waitTime +
-        " milliseconds - " +
-        "tot_wkr_task_resigned, tot_wkr_task_err, " +
-        "tot_wkr_final_transition_failed, tot_wkr_task_done, " +
-        "tot_wkr_preempt_task");
   }
 
   @Test (timeout=300000)
@@ -1042,46 +1069,49 @@ public class TestDistributedLogSplitting {
         "distributed log splitting test", null);
 
     Table ht = installTable(zkw, "table", "family", NUM_REGIONS_TO_CREATE);
-    populateDataInTable(NUM_ROWS_PER_REGION, "family");
+    try {
+      populateDataInTable(NUM_ROWS_PER_REGION, "family");
 
 
-    List<RegionServerThread> rsts = cluster.getLiveRegionServerThreads();
-    assertEquals(NUM_RS, rsts.size());
-    rsts.get(0).getRegionServer().abort("testing");
-    rsts.get(1).getRegionServer().abort("testing");
-    rsts.get(2).getRegionServer().abort("testing");
-
-    long start = EnvironmentEdgeManager.currentTime();
-    while (cluster.getLiveRegionServerThreads().size() > (NUM_RS - 3)) {
-      if (EnvironmentEdgeManager.currentTime() - start > 60000) {
-        assertTrue(false);
-      }
-      Thread.sleep(200);
-    }
+      List<RegionServerThread> rsts = cluster.getLiveRegionServerThreads();
+      assertEquals(NUM_RS, rsts.size());
+      rsts.get(0).getRegionServer().abort("testing");
+      rsts.get(1).getRegionServer().abort("testing");
+      rsts.get(2).getRegionServer().abort("testing");
 
-    start = EnvironmentEdgeManager.currentTime();
-    while (HBaseTestingUtility.getAllOnlineRegions(cluster).size()
-        < (NUM_REGIONS_TO_CREATE + 1)) {
-      if (EnvironmentEdgeManager.currentTime() - start > 60000) {
-        assertTrue("Timedout", false);
+      long start = EnvironmentEdgeManager.currentTime();
+      while (cluster.getLiveRegionServerThreads().size() > (NUM_RS - 3)) {
+        if (EnvironmentEdgeManager.currentTime() - start > 60000) {
+          assertTrue(false);
+        }
+        Thread.sleep(200);
       }
-      Thread.sleep(200);
-    }
 
-    // wait for all regions are fully recovered
-    TEST_UTIL.waitFor(180000, 200, new Waiter.Predicate<Exception>() {
-      @Override
-      public boolean evaluate() throws Exception {
-        List<String> recoveringRegions = zkw.getRecoverableZooKeeper().getChildren(
-          zkw.recoveringRegionsZNode, false);
-        return (recoveringRegions != null && recoveringRegions.size() == 0);
+      start = EnvironmentEdgeManager.currentTime();
+      while (HBaseTestingUtility.getAllOnlineRegions(cluster).size()
+          < (NUM_REGIONS_TO_CREATE + 1)) {
+        if (EnvironmentEdgeManager.currentTime() - start > 60000) {
+          assertTrue("Timedout", false);
+        }
+        Thread.sleep(200);
       }
-    });
 
-    assertEquals(NUM_REGIONS_TO_CREATE * NUM_ROWS_PER_REGION,
-        TEST_UTIL.countRows(ht));
-    ht.close();
-    zkw.close();
+      // wait for all regions are fully recovered
+      TEST_UTIL.waitFor(180000, 200, new Waiter.Predicate<Exception>() {
+        @Override
+        public boolean evaluate() throws Exception {
+          List<String> recoveringRegions = zkw.getRecoverableZooKeeper().getChildren(
+              zkw.recoveringRegionsZNode, false);
+          return (recoveringRegions != null && recoveringRegions.size() == 0);
+        }
+      });
+
+      assertEquals(NUM_REGIONS_TO_CREATE * NUM_ROWS_PER_REGION,
+          TEST_UTIL.countRows(ht));
+    } finally {
+      if (ht != null) ht.close();
+      if (zkw != null) zkw.close();
+    }
   }
 
 
@@ -1218,79 +1248,83 @@ public class TestDistributedLogSplitting {
     List<RegionServerThread> rsts = cluster.getLiveRegionServerThreads();
     final ZooKeeperWatcher zkw = new ZooKeeperWatcher(conf, "table-creation", null);
     Table ht = installTable(zkw, "table", "family", NUM_REGIONS_TO_CREATE);
-
-    List<HRegionInfo> regions = null;
-    HRegionServer hrs = null;
-    for (int i = 0; i < NUM_RS; i++) {
-      boolean isCarryingMeta = false;
-      hrs = rsts.get(i).getRegionServer();
-      regions = ProtobufUtil.getOnlineRegions(hrs.getRSRpcServices());
-      for (HRegionInfo region : regions) {
-        if (region.isMetaRegion()) {
-          isCarryingMeta = true;
-          break;
+    try {
+      List<HRegionInfo> regions = null;
+      HRegionServer hrs = null;
+      for (int i = 0; i < NUM_RS; i++) {
+        boolean isCarryingMeta = false;
+        hrs = rsts.get(i).getRegionServer();
+        regions = ProtobufUtil.getOnlineRegions(hrs.getRSRpcServices());
+        for (HRegionInfo region : regions) {
+          if (region.isMetaRegion()) {
+            isCarryingMeta = true;
+            break;
+          }
         }
+        if (isCarryingMeta) {
+          continue;
+        }
+        break;
       }
-      if (isCarryingMeta) {
-        continue;
-      }
-      break;
-    }
 
-    LOG.info("#regions = " + regions.size());
-    Iterator<HRegionInfo> it = regions.iterator();
-    while (it.hasNext()) {
-      HRegionInfo region = it.next();
-      if (region.isMetaTable()
-          || region.getEncodedName().equals(HRegionInfo.FIRST_META_REGIONINFO.getEncodedName())) {
-        it.remove();
+      LOG.info("#regions = " + regions.size());
+      Iterator<HRegionInfo> it = regions.iterator();
+      while (it.hasNext()) {
+        HRegionInfo region = it.next();
+        if (region.isMetaTable()
+            || region.getEncodedName().equals(
+              HRegionInfo.FIRST_META_REGIONINFO.getEncodedName())) {
+          it.remove();
+        }
       }
-    }
-    if (regions.size() == 0) return;
-    HRegionInfo curRegionInfo = regions.get(0);
-    byte[] startRow = curRegionInfo.getStartKey();
-    if (startRow == null || startRow.length == 0) {
-      startRow = new byte[] { 0, 0, 0, 0, 1 };
-    }
-    byte[] row = Bytes.incrementBytes(startRow, 1);
-    // use last 5 bytes because HBaseTestingUtility.createMultiRegions use 5 bytes key
-    row = Arrays.copyOfRange(row, 3, 8);
-    long value = 0;
-    TableName tableName = TableName.valueOf("table");
-    byte[] family = Bytes.toBytes("family");
-    byte[] qualifier = Bytes.toBytes("c1");
-    long timeStamp = System.currentTimeMillis();
-    HTableDescriptor htd = new HTableDescriptor();
-    htd.addFamily(new HColumnDescriptor(family));
-    final WAL wal = hrs.getWAL(curRegionInfo);
-    for (int i = 0; i < NUM_LOG_LINES; i += 1) {
-      WALEdit e = new WALEdit();
-      value++;
-      e.add(new KeyValue(row, family, qualifier, timeStamp, Bytes.toBytes(value)));
-      wal.append(htd, curRegionInfo,
+      if (regions.size() == 0) return;
+      HRegionInfo curRegionInfo = regions.get(0);
+      byte[] startRow = curRegionInfo.getStartKey();
+      if (startRow == null || startRow.length == 0) {
+        startRow = new byte[] { 0, 0, 0, 0, 1 };
+      }
+      byte[] row = Bytes.incrementBytes(startRow, 1);
+      // use last 5 bytes because HBaseTestingUtility.createMultiRegions use 5 bytes key
+      row = Arrays.copyOfRange(row, 3, 8);
+      long value = 0;
+      TableName tableName = TableName.valueOf("table");
+      byte[] family = Bytes.toBytes("family");
+      byte[] qualifier = Bytes.toBytes("c1");
+      long timeStamp = System.currentTimeMillis();
+      HTableDescriptor htd = new HTableDescriptor(tableName);
+      htd.addFamily(new HColumnDescriptor(family));
+      final WAL wal = hrs.getWAL(curRegionInfo);
+      for (int i = 0; i < NUM_LOG_LINES; i += 1) {
+        WALEdit e = new WALEdit();
+        value++;
+        e.add(new KeyValue(row, family, qualifier, timeStamp, Bytes.toBytes(value)));
+        wal.append(htd, curRegionInfo,
           new HLogKey(curRegionInfo.getEncodedNameAsBytes(), tableName, System.currentTimeMillis()),
-          e, sequenceId, true, null);
+            e, sequenceId, true, null);
+      }
+      wal.sync();
+      wal.shutdown();
+
+      // wait for abort completes
+      this.abortRSAndWaitForRecovery(hrs, zkw, NUM_REGIONS_TO_CREATE);
+
+      // verify we got the last value
+      LOG.info("Verification Starts...");
+      Get g = new Get(row);
+      Result r = ht.get(g);
+      long theStoredVal = Bytes.toLong(r.getValue(family, qualifier));
+      assertEquals(value, theStoredVal);
+
+      // after flush
+      LOG.info("Verification after flush...");
+      TEST_UTIL.getHBaseAdmin().flush(tableName);
+      r = ht.get(g);
+      theStoredVal = Bytes.toLong(r.getValue(family, qualifier));
+      assertEquals(value, theStoredVal);
+    } finally {
+      if (ht != null) ht.close();
+      if (zkw != null) zkw.close();
     }
-    wal.sync();
-    wal.shutdown();
-
-    // wait for abort completes
-    this.abortRSAndWaitForRecovery(hrs, zkw, NUM_REGIONS_TO_CREATE);
-
-    // verify we got the last value
-    LOG.info("Verification Starts...");
-    Get g = new Get(row);
-    Result r = ht.get(g);
-    long theStoredVal = Bytes.toLong(r.getValue(family, qualifier));
-    assertEquals(value, theStoredVal);
-
-    // after flush
-    LOG.info("Verification after flush...");
-    TEST_UTIL.getHBaseAdmin().flush(tableName);
-    r = ht.get(g);
-    theStoredVal = Bytes.toLong(r.getValue(family, qualifier));
-    assertEquals(value, theStoredVal);
-    ht.close();
   }
 
   @Test(timeout = 300000)
@@ -1311,88 +1345,91 @@ public class TestDistributedLogSplitting {
     List<RegionServerThread> rsts = cluster.getLiveRegionServerThreads();
     final ZooKeeperWatcher zkw = new ZooKeeperWatcher(conf, "table-creation", null);
     Table ht = installTable(zkw, "table", "family", NUM_REGIONS_TO_CREATE);
-
-    List<HRegionInfo> regions = null;
-    HRegionServer hrs = null;
-    for (int i = 0; i < NUM_RS; i++) {
-      boolean isCarryingMeta = false;
-      hrs = rsts.get(i).getRegionServer();
-      regions = ProtobufUtil.getOnlineRegions(hrs.getRSRpcServices());
-      for (HRegionInfo region : regions) {
-        if (region.isMetaRegion()) {
-          isCarryingMeta = true;
-          break;
+    try {
+      List<HRegionInfo> regions = null;
+      HRegionServer hrs = null;
+      for (int i = 0; i < NUM_RS; i++) {
+        boolean isCarryingMeta = false;
+        hrs = rsts.get(i).getRegionServer();
+        regions = ProtobufUtil.getOnlineRegions(hrs.getRSRpcServices());
+        for (HRegionInfo region : regions) {
+          if (region.isMetaRegion()) {
+            isCarryingMeta = true;
+            break;
+          }
         }
+        if (isCarryingMeta) {
+          continue;
+        }
+        break;
       }
-      if (isCarryingMeta) {
-        continue;
-      }
-      break;
-    }
 
-    LOG.info("#regions = " + regions.size());
-    Iterator<HRegionInfo> it = regions.iterator();
-    while (it.hasNext()) {
-      HRegionInfo region = it.next();
-      if (region.isMetaTable()
+      LOG.info("#regions = " + regions.size());
+      Iterator<HRegionInfo> it = regions.iterator();
+      while (it.hasNext()) {
+        HRegionInfo region = it.next();
+        if (region.isMetaTable()
           || region.getEncodedName().equals(HRegionInfo.FIRST_META_REGIONINFO.getEncodedName())) {
-        it.remove();
+            it.remove();
+        }
       }
-    }
-    if (regions.size() == 0) return;
-    HRegionInfo curRegionInfo = regions.get(0);
-    byte[] startRow = curRegionInfo.getStartKey();
-    if (startRow == null || startRow.length == 0) {
-      startRow = new byte[] { 0, 0, 0, 0, 1 };
-    }
-    byte[] row = Bytes.incrementBytes(startRow, 1);
-    // use last 5 bytes because HBaseTestingUtility.createMultiRegions use 5 bytes key
-    row = Arrays.copyOfRange(row, 3, 8);
-    long value = 0;
-    final TableName tableName = TableName.valueOf("table");
-    byte[] family = Bytes.toBytes("family");
-    byte[] qualifier = Bytes.toBytes("c1");
-    long timeStamp = System.currentTimeMillis();
-    HTableDescriptor htd = new HTableDescriptor(tableName);
-    htd.addFamily(new HColumnDescriptor(family));
-    final WAL wal = hrs.getWAL(curRegionInfo);
-    for (int i = 0; i < NUM_LOG_LINES; i += 1) {
-      WALEdit e = new WALEdit();
-      value++;
-      e.add(new KeyValue(row, family, qualifier, timeStamp, Bytes.toBytes(value)));
-      wal.append(htd, curRegionInfo, new HLogKey(curRegionInfo.getEncodedNameAsBytes(),
-          tableName, System.currentTimeMillis()), e, sequenceId, true, null);
-    }
-    wal.sync();
-    wal.shutdown();
-
-    // wait for abort completes
-    this.abortRSAndWaitForRecovery(hrs, zkw, NUM_REGIONS_TO_CREATE);
-
-    // verify we got the last value
-    LOG.info("Verification Starts...");
-    Get g = new Get(row);
-    Result r = ht.get(g);
-    long theStoredVal = Bytes.toLong(r.getValue(family, qualifier));
-    assertEquals(value, theStoredVal);
-
-    // after flush & compaction
-    LOG.info("Verification after flush...");
-    TEST_UTIL.getHBaseAdmin().flush(tableName);
-    TEST_UTIL.getHBaseAdmin().compact(tableName);
-
-    // wait for compaction completes
-    TEST_UTIL.waitFor(30000, 200, new Waiter.Predicate<Exception>() {
-      @Override
-      public boolean evaluate() throws Exception {
-        return (TEST_UTIL.getHBaseAdmin().getCompactionState(tableName) == CompactionState.NONE);
+      if (regions.size() == 0) return;
+      HRegionInfo curRegionInfo = regions.get(0);
+      byte[] startRow = curRegionInfo.getStartKey();
+      if (startRow == null || startRow.length == 0) {
+        startRow = new byte[] { 0, 0, 0, 0, 1 };
       }
-    });
+      byte[] row = Bytes.incrementBytes(startRow, 1);
+      // use last 5 bytes because HBaseTestingUtility.createMultiRegions use 5 bytes key
+      row = Arrays.copyOfRange(row, 3, 8);
+      long value = 0;
+      final TableName tableName = TableName.valueOf("table");
+      byte[] family = Bytes.toBytes("family");
+      byte[] qualifier = Bytes.toBytes("c1");
+      long timeStamp = System.currentTimeMillis();
+      HTableDescriptor htd = new HTableDescriptor(tableName);
+      htd.addFamily(new HColumnDescriptor(family));
+      final WAL wal = hrs.getWAL(curRegionInfo);
+      for (int i = 0; i < NUM_LOG_LINES; i += 1) {
+        WALEdit e = new WALEdit();
+        value++;
+        e.add(new KeyValue(row, family, qualifier, timeStamp, Bytes.toBytes(value)));
+        wal.append(htd, curRegionInfo, new HLogKey(curRegionInfo.getEncodedNameAsBytes(),
+            tableName, System.currentTimeMillis()), e, sequenceId, true, null);
+      }
+      wal.sync();
+      wal.shutdown();
+
+      // wait for abort completes
+      this.abortRSAndWaitForRecovery(hrs, zkw, NUM_REGIONS_TO_CREATE);
+
+      // verify we got the last value
+      LOG.info("Verification Starts...");
+      Get g = new Get(row);
+      Result r = ht.get(g);
+      long theStoredVal = Bytes.toLong(r.getValue(family, qualifier));
+      assertEquals(value, theStoredVal);
+
+      // after flush & compaction
+      LOG.info("Verification after flush...");
+      TEST_UTIL.getHBaseAdmin().flush(tableName);
+      TEST_UTIL.getHBaseAdmin().compact(tableName);
+
+      // wait for compaction completes
+      TEST_UTIL.waitFor(30000, 200, new Waiter.Predicate<Exception>() {
+        @Override
+        public boolean evaluate() throws Exception {
+          return (TEST_UTIL.getHBaseAdmin().getCompactionState(tableName) == CompactionState.NONE);
+        }
+      });
 
-    r = ht.get(g);
-    theStoredVal = Bytes.toLong(r.getValue(family, qualifier));
-    assertEquals(value, theStoredVal);
-    ht.close();
+      r = ht.get(g);
+      theStoredVal = Bytes.toLong(r.getValue(family, qualifier));
+      assertEquals(value, theStoredVal);
+    } finally {
+      if (ht != null) ht.close();
+      if (zkw != null) zkw.close();
+    }
   }
 
   @Test(timeout = 300000)
@@ -1400,45 +1437,49 @@ public class TestDistributedLogSplitting {
     LOG.info("testReadWriteSeqIdFiles");
     startCluster(2);
     final ZooKeeperWatcher zkw = new ZooKeeperWatcher(conf, "table-creation", null);
-    HTable ht = installTable(zkw, "table", "family", 10);
-    FileSystem fs = master.getMasterFileSystem().getFileSystem();
-    Path tableDir = FSUtils.getTableDir(FSUtils.getRootDir(conf), TableName.valueOf("table"));
-    List<Path> regionDirs = FSUtils.getRegionDirs(fs, tableDir);
-    long newSeqId = WALSplitter.writeRegionSequenceIdFile(fs, regionDirs.get(0), 1L, 1000L);
-    WALSplitter.writeRegionSequenceIdFile(fs, regionDirs.get(0) , 1L, 1000L);
-    assertEquals(newSeqId + 2000,
-      WALSplitter.writeRegionSequenceIdFile(fs, regionDirs.get(0), 3L, 1000L));
-    
-    Path editsdir = WALSplitter.getRegionDirRecoveredEditsDir(regionDirs.get(0));
-    FileStatus[] files = FSUtils.listStatus(fs, editsdir, new PathFilter() {
-      @Override
-      public boolean accept(Path p) {
-        return WALSplitter.isSequenceIdFile(p);
-      }
-    });
-    // only one seqid file should exist
-    assertEquals(1, files.length);
-
-    // verify all seqId files aren't treated as recovered.edits files
-    NavigableSet<Path> recoveredEdits = WALSplitter.getSplitEditFilesSorted(fs, regionDirs.get(0));
-    assertEquals(0, recoveredEdits.size());
-
-    ht.close();
-  }
+    Table ht = installTable(zkw, "table", "family", 10);
+    try {
+      FileSystem fs = master.getMasterFileSystem().getFileSystem();
+      Path tableDir = FSUtils.getTableDir(FSUtils.getRootDir(conf), TableName.valueOf("table"));
+      List<Path> regionDirs = FSUtils.getRegionDirs(fs, tableDir);
+      long newSeqId = WALSplitter.writeRegionSequenceIdFile(fs, regionDirs.get(0), 1L, 1000L);
+      WALSplitter.writeRegionSequenceIdFile(fs, regionDirs.get(0) , 1L, 1000L);
+      assertEquals(newSeqId + 2000,
+          WALSplitter.writeRegionSequenceIdFile(fs, regionDirs.get(0), 3L, 1000L));
+
+      Path editsdir = WALSplitter.getRegionDirRecoveredEditsDir(regionDirs.get(0));
+      FileStatus[] files = FSUtils.listStatus(fs, editsdir, new PathFilter() {
+        @Override
+        public boolean accept(Path p) {
+          return WALSplitter.isSequenceIdFile(p);
+        }
+      });
+      // only one seqid file should exist
+      assertEquals(1, files.length);
 
-  HTable installTable(ZooKeeperWatcher zkw, String tname, String fname, int nrs) throws Exception {
+      // verify all seqId files aren't treated as recovered.edits files
+      NavigableSet<Path> recoveredEdits =
+        WALSplitter.getSplitEditFilesSorted(fs, regionDirs.get(0));
+      assertEquals(0, recoveredEdits.size());
+    } finally {
+      if (ht != null) ht.close();
+      if (zkw != null) zkw.close();
+    }
+  } 
+  
+  Table installTable(ZooKeeperWatcher zkw, String tname, String fname, int nrs) throws Exception {
     return installTable(zkw, tname, fname, nrs, 0);
   }
 
-  HTable installTable(ZooKeeperWatcher zkw, String tname, String fname, int nrs,
+  Table installTable(ZooKeeperWatcher zkw, String tname, String fname, int nrs,
       int existingRegions) throws Exception {
     // Create a table with regions
     TableName table = TableName.valueOf(tname);
     byte [] family = Bytes.toBytes(fname);
     LOG.info("Creating table with " + nrs + " regions");
-    HTable ht = TEST_UTIL.createMultiRegionTable(table, family, nrs);
+    Table ht = TEST_UTIL.createMultiRegionTable(table, family, nrs);
     int numRegions = -1;
-    try (RegionLocator r = ht.getRegionLocator()) {
+    try (RegionLocator r = TEST_UTIL.getConnection().getRegionLocator(table)) {
       numRegions = r.getStartKeys().length;
     }
     assertEquals(nrs, numRegions);
@@ -1487,6 +1528,27 @@ public class TestDistributedLogSplitting {
         putData(region, hri.getStartKey(), nrows, Bytes.toBytes("q"), family);
       }
     }
+
+    for (MasterThread mt : cluster.getLiveMasterThreads()) {
+      HRegionServer hrs = mt.getMaster();
+      List<HRegionInfo> hris;
+      try {
+        hris = ProtobufUtil.getOnlineRegions(hrs.getRSRpcServices());
+      } catch (ServerNotRunningYetException e) {
+        // It's ok: this master may be a backup. Ignored.
+        continue;
+      }
+      for (HRegionInfo hri : hris) {
+        if (hri.getTable().isSystemTable()) {
+          continue;
+        }
+        LOG.debug("adding data to rs = " + mt.getName() +
+            " region = "+ hri.getRegionNameAsString());
+        Region region = hrs.getOnlineRegion(hri.getRegionName());
+        assertTrue(region != null);
+        putData(region, hri.getStartKey(), nrows, Bytes.toBytes("q"), family);
+      }
+    }
   }
 
   public void makeWAL(HRegionServer hrs, List<HRegionInfo> regions, String tname, String fname,
@@ -1544,7 +1606,8 @@ public class TestDistributedLogSplitting {
                                              // key
         byte[] qualifier = Bytes.toBytes("c" + Integer.toString(i));
         e.add(new KeyValue(row, family, qualifier, System.currentTimeMillis(), value));
-        log.append(htd, curRegionInfo, new HLogKey(curRegionInfo.getEncodedNameAsBytes(), fullTName,
+        log.append(htd, curRegionInfo,
+          new HLogKey(curRegionInfo.getEncodedNameAsBytes(), fullTName,
             System.currentTimeMillis()), e, sequenceId, true, null);
         if (0 == i % syncEvery) {
           log.sync();
@@ -1592,10 +1655,8 @@ public class TestDistributedLogSplitting {
     return count;
   }
 
-  private void blockUntilNoRIT(ZooKeeperWatcher zkw, HMaster master)
-  throws KeeperException, InterruptedException {
-    ZKAssign.blockUntilNoRIT(zkw);
-    master.assignmentManager.waitUntilNoRegionsInTransition(60000);
+  private void blockUntilNoRIT(ZooKeeperWatcher zkw, HMaster master) throws Exception {
+    TEST_UTIL.waitUntilNoRegionsInTransition(60000);
   }
 
   private void putData(Region region, byte[] startRow, int numRows, byte [] qf,
@@ -1677,14 +1738,19 @@ public class TestDistributedLogSplitting {
    */
   private HRegionServer findRSToKill(boolean hasMetaRegion, String tableName) throws Exception {
     List<RegionServerThread> rsts = cluster.getLiveRegionServerThreads();
-    int numOfRSs = rsts.size();
     List<HRegionInfo> regions = null;
     HRegionServer hrs = null;
 
-    for (int i = 0; i < numOfRSs; i++) {
+    for (RegionServerThread rst: rsts) {
+      hrs = rst.getRegionServer();
+      while (rst.isAlive() && !hrs.isOnline()) {
+        Thread.sleep(100);
+      }
+      if (!rst.isAlive()) {
+        continue;
+      }
       boolean isCarryingMeta = false;
       boolean foundTableRegion = false;
-      hrs = rsts.get(i).getRegionServer();
       regions = ProtobufUtil.getOnlineRegions(hrs.getRSRpcServices());
       for (HRegionInfo region : regions) {
         if (region.isMetaRegion()) {

http://git-wip-us.apache.org/repos/asf/hbase/blob/08ca8965/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestRollingRestart.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestRollingRestart.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestRollingRestart.java
index 1908edb..3258ebd 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestRollingRestart.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestRollingRestart.java
@@ -69,7 +69,6 @@ public class  TestRollingRestart {
     // Start the cluster
     log("Starting cluster");
     Configuration conf = HBaseConfiguration.create();
-    conf.setInt(HConstants.REGION_SERVER_HIGH_PRIORITY_HANDLER_COUNT, 40);
     HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(conf);
     TEST_UTIL.startMiniCluster(NUM_MASTERS, NUM_RS);
     MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
@@ -237,7 +236,7 @@ public class  TestRollingRestart {
     }
     return numFound;
   }
-  
+
   private void assertRegionsAssigned(MiniHBaseCluster cluster,
       Set<String> expectedRegions) throws IOException {
     int numFound = getNumberOfOnlineRegions(cluster);

http://git-wip-us.apache.org/repos/asf/hbase/blob/08ca8965/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionReplicaFailover.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionReplicaFailover.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionReplicaFailover.java
index 70bbb69..fab092e 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionReplicaFailover.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionReplicaFailover.java
@@ -111,7 +111,6 @@ public class TestRegionReplicaFailover {
     conf.setInt("replication.stats.thread.period.seconds", 5);
     conf.setBoolean("hbase.tests.use.shortcircuit.reads", false);
     conf.setBoolean(HConstants.DISTRIBUTED_LOG_REPLAY_KEY, distributedLogReplay);
-    conf.setInt(HConstants.REGION_SERVER_HIGH_PRIORITY_HANDLER_COUNT, 60);
 
     HTU.startMiniCluster(NB_SERVERS);
     htd = HTU.createTableDescriptor(

http://git-wip-us.apache.org/repos/asf/hbase/blob/08ca8965/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshot.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshot.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshot.java
index c23f1c3..7e629cd 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshot.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshot.java
@@ -81,7 +81,6 @@ public class TestExportSnapshot {
     conf.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 6);
     conf.setBoolean("hbase.master.enabletable.roundrobin", true);
     conf.setInt("mapreduce.map.maxattempts", 10);
-    conf.setInt(HConstants.REGION_SERVER_HIGH_PRIORITY_HANDLER_COUNT, 40);
   }
 
   @BeforeClass

http://git-wip-us.apache.org/repos/asf/hbase/blob/08ca8965/hbase-server/src/test/resources/hbase-site.xml
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/resources/hbase-site.xml b/hbase-server/src/test/resources/hbase-site.xml
index 34a1b20..2d06c0a 100644
--- a/hbase-server/src/test/resources/hbase-site.xml
+++ b/hbase-server/src/test/resources/hbase-site.xml
@@ -51,6 +51,14 @@
     <value>5</value>
   </property>
   <property>
+    <name>hbase.regionserver.metahandler.count</name>
+    <value>5</value>
+  </property>
+  <property>
+      <name>hbase.ipc.server.read.threadpool.size</name>
+    <value>3</value>
+  </property>
+  <property>
     <name>hbase.master.info.port</name>
     <value>-1</value>
     <description>The port for the hbase master web UI


Mime
View raw message