Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id B4AD3200D5C for ; Wed, 8 Nov 2017 02:54:14 +0100 (CET) Received: by cust-asf.ponee.io (Postfix) id B1A0E160C02; Wed, 8 Nov 2017 01:54:14 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id 68FC5160C00 for ; Wed, 8 Nov 2017 02:54:12 +0100 (CET) Received: (qmail 8418 invoked by uid 500); 8 Nov 2017 01:54:11 -0000 Mailing-List: contact commits-help@hbase.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@hbase.apache.org Delivered-To: mailing list commits@hbase.apache.org Received: (qmail 8388 invoked by uid 99); 8 Nov 2017 01:54:11 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 08 Nov 2017 01:54:11 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 88541DF9FD; Wed, 8 Nov 2017 01:54:09 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: appy@apache.org To: commits@hbase.apache.org Date: Wed, 08 Nov 2017 01:54:10 -0000 Message-Id: <515f50fe834c49adb8b7f36fc3e5036a@git.apache.org> In-Reply-To: References: X-Mailer: ASF-Git Admin Mailer Subject: [2/5] hbase git commit: HBASE-19128 Purge Distributed Log Replay from codebase, configurations, text; mark the feature as unsupported, broken. archived-at: Wed, 08 Nov 2017 01:54:14 -0000 http://git-wip-us.apache.org/repos/asf/hbase/blob/a6d8023f/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestDistributedLogSplitting.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestDistributedLogSplitting.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestDistributedLogSplitting.java index eafc412..3d59639 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestDistributedLogSplitting.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestDistributedLogSplitting.java @@ -33,12 +33,9 @@ import static org.junit.Assert.fail; import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; -import java.util.HashSet; import java.util.Iterator; -import java.util.LinkedList; import java.util.List; import java.util.NavigableSet; -import java.util.Set; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.Future; @@ -67,28 +64,18 @@ import org.apache.hadoop.hbase.ServerName; import org.apache.hadoop.hbase.SplitLogCounters; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.Waiter; -import org.apache.hadoop.hbase.client.ClusterConnection; -import org.apache.hadoop.hbase.client.CompactionState; -import org.apache.hadoop.hbase.client.ConnectionUtils; -import org.apache.hadoop.hbase.client.Delete; -import org.apache.hadoop.hbase.client.Get; -import org.apache.hadoop.hbase.client.Increment; -import org.apache.hadoop.hbase.client.NonceGenerator; -import org.apache.hadoop.hbase.client.PerClientRandomNonceGenerator; import org.apache.hadoop.hbase.client.Put; import org.apache.hadoop.hbase.client.RegionInfo; import org.apache.hadoop.hbase.client.RegionInfoBuilder; import org.apache.hadoop.hbase.client.RegionLocator; -import org.apache.hadoop.hbase.client.Result; -import org.apache.hadoop.hbase.client.RetriesExhaustedWithDetailsException; import org.apache.hadoop.hbase.client.Table; import org.apache.hadoop.hbase.coordination.ZKSplitLogManagerCoordination; -import org.apache.hadoop.hbase.exceptions.RegionInRecoveryException; import org.apache.hadoop.hbase.ipc.ServerNotRunningYetException; import org.apache.hadoop.hbase.master.SplitLogManager.TaskBatch; import org.apache.hadoop.hbase.master.assignment.RegionStates; import org.apache.hadoop.hbase.regionserver.HRegion; import org.apache.hadoop.hbase.regionserver.HRegionServer; +import org.apache.hadoop.hbase.regionserver.MultiVersionConcurrencyControl; import org.apache.hadoop.hbase.regionserver.Region; import org.apache.hadoop.hbase.testclassification.LargeTests; import org.apache.hadoop.hbase.testclassification.MasterTests; @@ -110,10 +97,8 @@ import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher; import org.apache.hadoop.hdfs.MiniDFSCluster; import org.junit.After; import org.junit.AfterClass; -import org.junit.Assert; import org.junit.Before; import org.junit.BeforeClass; -import org.junit.Ignore; import org.junit.Rule; import org.junit.Test; import org.junit.experimental.categories.Category; @@ -145,7 +130,6 @@ public class TestDistributedLogSplitting { Configuration conf; static Configuration originalConf; static HBaseTestingUtility TEST_UTIL; - static MiniDFSCluster dfsCluster; static MiniZooKeeperCluster zkCluster; @Rule @@ -154,7 +138,6 @@ public class TestDistributedLogSplitting { @BeforeClass public static void setup() throws Exception { TEST_UTIL = new HBaseTestingUtility(HBaseConfiguration.create()); - dfsCluster = TEST_UTIL.startMiniDFSCluster(1); zkCluster = TEST_UTIL.startMiniZKCluster(); originalConf = TEST_UTIL.getConfiguration(); } @@ -178,7 +161,6 @@ public class TestDistributedLogSplitting { conf.setInt(HConstants.REGION_SERVER_HIGH_PRIORITY_HANDLER_COUNT, 10); TEST_UTIL.shutdownMiniHBaseCluster(); TEST_UTIL = new HBaseTestingUtility(conf); - TEST_UTIL.setDFSCluster(dfsCluster); TEST_UTIL.setZkCluster(zkCluster); TEST_UTIL.startMiniHBaseCluster(NUM_MASTERS, num_rs); cluster = TEST_UTIL.getHBaseCluster(); @@ -211,14 +193,12 @@ public class TestDistributedLogSplitting { } } - @Ignore("DLR is broken by HBASE-12751") @Test (timeout=300000) + @Test (timeout=300000) public void testRecoveredEdits() throws Exception { - LOG.info("testRecoveredEdits"); conf.setLong("hbase.regionserver.hlog.blocksize", 30 * 1024); // create more than one wal - conf.setBoolean(HConstants.DISTRIBUTED_LOG_REPLAY_KEY, false); startCluster(NUM_RS); - final int NUM_LOG_LINES = 1000; + final int NUM_LOG_LINES = 10000; final SplitLogManager slm = master.getMasterWalManager().getSplitLogManager(); // turn off load balancing to prevent regions from moving around otherwise // they will consume recovered.edits @@ -229,23 +209,18 @@ public class TestDistributedLogSplitting { Path rootdir = FSUtils.getRootDir(conf); + int numRegions = 50; Table t = installTable(new ZooKeeperWatcher(conf, "table-creation", null), - "table", "family", 40); + "table", "family", numRegions); try { TableName table = t.getName(); List regions = null; HRegionServer hrs = null; for (int i = 0; i < NUM_RS; i++) { - boolean foundRs = false; hrs = rsts.get(i).getRegionServer(); regions = ProtobufUtil.getOnlineRegions(hrs.getRSRpcServices()); - for (RegionInfo region : regions) { - if (region.getTable().getNameAsString().equalsIgnoreCase("table")) { - foundRs = true; - break; - } - } - if (foundRs) break; + // At least one RS will have >= to average number of regions. + if (regions.size() >= numRegions/NUM_RS) break; } final Path logDir = new Path(rootdir, AbstractFSWALProvider.getWALDirectoryName(hrs .getServerName().toString())); @@ -266,11 +241,9 @@ public class TestDistributedLogSplitting { int count = 0; for (RegionInfo hri : regions) { - Path tdir = FSUtils.getTableDir(rootdir, table); - Path editsdir = - WALSplitter.getRegionDirRecoveredEditsDir( - HRegion.getRegionDir(tdir, hri.getEncodedName())); + Path editsdir = WALSplitter.getRegionDirRecoveredEditsDir( + HRegion.getRegionDir(tdir, hri.getEncodedName())); LOG.debug("checking edits dir " + editsdir); FileStatus[] files = fs.listStatus(editsdir, new PathFilter() { @Override @@ -293,195 +266,14 @@ public class TestDistributedLogSplitting { // check that the log file is moved assertFalse(fs.exists(logDir)); - assertEquals(NUM_LOG_LINES, count); } finally { if (t != null) t.close(); } } - @Ignore("DLR is broken by HBASE-12751") @Test(timeout = 300000) - public void testLogReplayWithNonMetaRSDown() throws Exception { - LOG.info("testLogReplayWithNonMetaRSDown"); - conf.setLong("hbase.regionserver.hlog.blocksize", 30 * 1024); // create more than one wal - conf.setBoolean(HConstants.DISTRIBUTED_LOG_REPLAY_KEY, true); - startCluster(NUM_RS); - final int NUM_REGIONS_TO_CREATE = 40; - final int NUM_LOG_LINES = 1000; - // turn off load balancing to prevent regions from moving around otherwise - // they will consume recovered.edits - master.balanceSwitch(false); - - final ZooKeeperWatcher zkw = new ZooKeeperWatcher(conf, "table-creation", null); - Table ht = installTable(zkw, "table", "family", NUM_REGIONS_TO_CREATE); - try { - HRegionServer hrs = findRSToKill(false, "table"); - List regions = ProtobufUtil.getOnlineRegions(hrs.getRSRpcServices()); - makeWAL(hrs, regions, "table", "family", NUM_LOG_LINES, 100); - - // wait for abort completes - this.abortRSAndVerifyRecovery(hrs, ht, zkw, NUM_REGIONS_TO_CREATE, NUM_LOG_LINES); - } finally { - if (ht != null) ht.close(); - if (zkw != null) zkw.close(); - } - } - - private static class NonceGeneratorWithDups implements NonceGenerator { - - private final PerClientRandomNonceGenerator delegate = PerClientRandomNonceGenerator.get(); - private boolean isDups = false; - private LinkedList nonces = new LinkedList<>(); - - public void startDups() { - isDups = true; - } - - @Override - public long newNonce() { - long nonce = isDups ? nonces.removeFirst() : delegate.newNonce(); - if (!isDups) { - nonces.add(nonce); - } - return nonce; - } - - @Override - public long getNonceGroup() { - return delegate.getNonceGroup(); - } - } - - @Ignore("DLR is broken by HBASE-12751") @Test(timeout = 300000) - public void testNonceRecovery() throws Exception { - LOG.info("testNonceRecovery"); - final String TABLE_NAME = "table"; - final String FAMILY_NAME = "family"; - final int NUM_REGIONS_TO_CREATE = 40; - - conf.setLong("hbase.regionserver.hlog.blocksize", 100*1024); - conf.setBoolean(HConstants.DISTRIBUTED_LOG_REPLAY_KEY, true); - startCluster(NUM_RS); - master.balanceSwitch(false); - - final ZooKeeperWatcher zkw = new ZooKeeperWatcher(conf, "table-creation", null); - Table ht = installTable(zkw, TABLE_NAME, FAMILY_NAME, NUM_REGIONS_TO_CREATE); - NonceGeneratorWithDups ng = new NonceGeneratorWithDups(); - NonceGenerator oldNg = - ConnectionUtils.injectNonceGeneratorForTesting( - (ClusterConnection)TEST_UTIL.getConnection(), ng); - - try { - List reqs = new ArrayList<>(); - for (RegionServerThread rst : cluster.getLiveRegionServerThreads()) { - HRegionServer hrs = rst.getRegionServer(); - List hris = ProtobufUtil.getOnlineRegions(hrs.getRSRpcServices()); - for (RegionInfo hri : hris) { - if (TABLE_NAME.equalsIgnoreCase(hri.getTable().getNameAsString())) { - byte[] key = hri.getStartKey(); - if (key == null || key.length == 0) { - key = Bytes.copy(hri.getEndKey()); - --(key[key.length - 1]); - } - Increment incr = new Increment(key); - incr.addColumn(Bytes.toBytes(FAMILY_NAME), Bytes.toBytes("q"), 1); - ht.increment(incr); - reqs.add(incr); - } - } - } - - HRegionServer hrs = findRSToKill(false, "table"); - abortRSAndWaitForRecovery(hrs, zkw, NUM_REGIONS_TO_CREATE); - ng.startDups(); - for (Increment incr : reqs) { - try { - ht.increment(incr); - fail("should have thrown"); - } catch (IOException ope) { - LOG.debug("Caught as expected: " + ope.getMessage()); - } - } - } finally { - ConnectionUtils.injectNonceGeneratorForTesting((ClusterConnection) - TEST_UTIL.getConnection(), oldNg); - if (ht != null) ht.close(); - if (zkw != null) zkw.close(); - } - } - - @Ignore("DLR is broken by HBASE-12751") @Test(timeout = 300000) - public void testLogReplayWithMetaRSDown() throws Exception { - LOG.info("testRecoveredEditsReplayWithMetaRSDown"); - conf.setBoolean(HConstants.DISTRIBUTED_LOG_REPLAY_KEY, true); - startCluster(NUM_RS); - final int NUM_REGIONS_TO_CREATE = 40; - final int NUM_LOG_LINES = 1000; - // turn off load balancing to prevent regions from moving around otherwise - // they will consume recovered.edits - master.balanceSwitch(false); - - final ZooKeeperWatcher zkw = new ZooKeeperWatcher(conf, "table-creation", null); - Table ht = installTable(zkw, "table", "family", NUM_REGIONS_TO_CREATE); - try { - HRegionServer hrs = findRSToKill(true, "table"); - List regions = ProtobufUtil.getOnlineRegions(hrs.getRSRpcServices()); - makeWAL(hrs, regions, "table", "family", NUM_LOG_LINES, 100); - - this.abortRSAndVerifyRecovery(hrs, ht, zkw, NUM_REGIONS_TO_CREATE, NUM_LOG_LINES); - } finally { - if (ht != null) ht.close(); - if (zkw != null) zkw.close(); - } - } - - private void abortRSAndVerifyRecovery(HRegionServer hrs, Table ht, final ZooKeeperWatcher zkw, - final int numRegions, final int numofLines) throws Exception { - - abortRSAndWaitForRecovery(hrs, zkw, numRegions); - assertEquals(numofLines, TEST_UTIL.countRows(ht)); - } - - private void abortRSAndWaitForRecovery(HRegionServer hrs, final ZooKeeperWatcher zkw, - final int numRegions) throws Exception { - final MiniHBaseCluster tmpCluster = this.cluster; - - // abort RS - LOG.info("Aborting region server: " + hrs.getServerName()); - hrs.abort("testing"); - - // wait for abort completes - TEST_UTIL.waitFor(120000, 200, new Waiter.Predicate() { - @Override - public boolean evaluate() throws Exception { - return (tmpCluster.getLiveRegionServerThreads().size() <= (NUM_RS - 1)); - } - }); - - // wait for regions come online - TEST_UTIL.waitFor(180000, 200, new Waiter.Predicate() { - @Override - public boolean evaluate() throws Exception { - return (HBaseTestingUtility.getAllOnlineRegions(tmpCluster).size() - >= (numRegions + 1)); - } - }); - - // wait for all regions are fully recovered - TEST_UTIL.waitFor(180000, 200, new Waiter.Predicate() { - @Override - public boolean evaluate() throws Exception { - List recoveringRegions = zkw.getRecoverableZooKeeper().getChildren( - zkw.znodePaths.recoveringRegionsZNode, false); - return (recoveringRegions != null && recoveringRegions.isEmpty()); - } - }); - } - - @Ignore("DLR is broken by HBASE-12751") @Test(timeout = 300000) + @Test(timeout = 300000) public void testMasterStartsUpWithLogSplittingWork() throws Exception { - LOG.info("testMasterStartsUpWithLogSplittingWork"); - conf.setBoolean(HConstants.DISTRIBUTED_LOG_REPLAY_KEY, false); conf.setInt(ServerManager.WAIT_ON_REGIONSERVERS_MINTOSTART, NUM_RS - 1); startCluster(NUM_RS); @@ -536,473 +328,6 @@ public class TestDistributedLogSplitting { } } - @Ignore("DLR is broken by HBASE-12751") @Test(timeout = 300000) - public void testMasterStartsUpWithLogReplayWork() throws Exception { - LOG.info("testMasterStartsUpWithLogReplayWork"); - conf.setBoolean(HConstants.DISTRIBUTED_LOG_REPLAY_KEY, true); - conf.setInt(ServerManager.WAIT_ON_REGIONSERVERS_MINTOSTART, NUM_RS - 1); - startCluster(NUM_RS); - - final int NUM_REGIONS_TO_CREATE = 40; - final int NUM_LOG_LINES = 1000; - // turn off load balancing to prevent regions from moving around otherwise - // they will consume recovered.edits - master.balanceSwitch(false); - - final ZooKeeperWatcher zkw = new ZooKeeperWatcher(conf, "table-creation", null); - Table ht = installTable(zkw, "table", "family", NUM_REGIONS_TO_CREATE); - try { - HRegionServer hrs = findRSToKill(false, "table"); - List regions = ProtobufUtil.getOnlineRegions(hrs.getRSRpcServices()); - makeWAL(hrs, regions, "table", "family", NUM_LOG_LINES, 100); - - // abort master - abortMaster(cluster); - - // abort RS - LOG.info("Aborting region server: " + hrs.getServerName()); - hrs.abort("testing"); - - // wait for the RS dies - TEST_UTIL.waitFor(120000, 200, new Waiter.Predicate() { - @Override - public boolean evaluate() throws Exception { - return (cluster.getLiveRegionServerThreads().size() <= (NUM_RS - 1)); - } - }); - - Thread.sleep(2000); - LOG.info("Current Open Regions:" + HBaseTestingUtility.getAllOnlineRegions(cluster).size()); - - // wait for all regions are fully recovered - TEST_UTIL.waitFor(180000, 200, new Waiter.Predicate() { - @Override - public boolean evaluate() throws Exception { - List recoveringRegions = zkw.getRecoverableZooKeeper().getChildren( - zkw.znodePaths.recoveringRegionsZNode, false); - boolean done = recoveringRegions != null && recoveringRegions.isEmpty(); - if (!done) { - LOG.info("Recovering regions: " + recoveringRegions); - } - return done; - } - }); - - LOG.info("Current Open Regions After Master Node Starts Up:" - + HBaseTestingUtility.getAllOnlineRegions(cluster).size()); - - assertEquals(NUM_LOG_LINES, TEST_UTIL.countRows(ht)); - } finally { - if (ht != null) ht.close(); - if (zkw != null) zkw.close(); - } - } - - - @Ignore("DLR is broken by HBASE-12751") @Test(timeout = 300000) - public void testLogReplayTwoSequentialRSDown() throws Exception { - LOG.info("testRecoveredEditsReplayTwoSequentialRSDown"); - conf.setBoolean(HConstants.DISTRIBUTED_LOG_REPLAY_KEY, true); - startCluster(NUM_RS); - final int NUM_REGIONS_TO_CREATE = 40; - final int NUM_LOG_LINES = 1000; - // turn off load balancing to prevent regions from moving around otherwise - // they will consume recovered.edits - master.balanceSwitch(false); - - List rsts = cluster.getLiveRegionServerThreads(); - final ZooKeeperWatcher zkw = new ZooKeeperWatcher(conf, "table-creation", null); - Table ht = installTable(zkw, "table", "family", NUM_REGIONS_TO_CREATE); - try { - List regions = null; - HRegionServer hrs1 = findRSToKill(false, "table"); - regions = ProtobufUtil.getOnlineRegions(hrs1.getRSRpcServices()); - - makeWAL(hrs1, regions, "table", "family", NUM_LOG_LINES, 100); - - // abort RS1 - LOG.info("Aborting region server: " + hrs1.getServerName()); - hrs1.abort("testing"); - - // wait for abort completes - TEST_UTIL.waitFor(120000, 200, new Waiter.Predicate() { - @Override - public boolean evaluate() throws Exception { - return (cluster.getLiveRegionServerThreads().size() <= (NUM_RS - 1)); - } - }); - - // wait for regions come online - TEST_UTIL.waitFor(180000, 200, new Waiter.Predicate() { - @Override - public boolean evaluate() throws Exception { - return (HBaseTestingUtility.getAllOnlineRegions(cluster).size() - >= (NUM_REGIONS_TO_CREATE + 1)); - } - }); - - // sleep a little bit in order to interrupt recovering in the middle - Thread.sleep(300); - // abort second region server - rsts = cluster.getLiveRegionServerThreads(); - HRegionServer hrs2 = rsts.get(0).getRegionServer(); - LOG.info("Aborting one more region server: " + hrs2.getServerName()); - hrs2.abort("testing"); - - // wait for abort completes - TEST_UTIL.waitFor(120000, 200, new Waiter.Predicate() { - @Override - public boolean evaluate() throws Exception { - return (cluster.getLiveRegionServerThreads().size() <= (NUM_RS - 2)); - } - }); - - // wait for regions come online - TEST_UTIL.waitFor(180000, 200, new Waiter.Predicate() { - @Override - public boolean evaluate() throws Exception { - return (HBaseTestingUtility.getAllOnlineRegions(cluster).size() - >= (NUM_REGIONS_TO_CREATE + 1)); - } - }); - - // wait for all regions are fully recovered - TEST_UTIL.waitFor(180000, 200, new Waiter.Predicate() { - @Override - public boolean evaluate() throws Exception { - List recoveringRegions = zkw.getRecoverableZooKeeper().getChildren( - zkw.znodePaths.recoveringRegionsZNode, false); - return (recoveringRegions != null && recoveringRegions.isEmpty()); - } - }); - - assertEquals(NUM_LOG_LINES, TEST_UTIL.countRows(ht)); - } finally { - if (ht != null) ht.close(); - if (zkw != null) zkw.close(); - } - } - - @Ignore("DLR is broken by HBASE-12751") @Test(timeout = 300000) - public void testMarkRegionsRecoveringInZK() throws Exception { - LOG.info("testMarkRegionsRecoveringInZK"); - conf.setBoolean(HConstants.DISTRIBUTED_LOG_REPLAY_KEY, true); - startCluster(NUM_RS); - master.balanceSwitch(false); - List rsts = cluster.getLiveRegionServerThreads(); - final ZooKeeperWatcher zkw = master.getZooKeeper(); - Table ht = installTable(zkw, "table", "family", 40); - try { - final SplitLogManager slm = master.getMasterWalManager().getSplitLogManager(); - - Set regionSet = new HashSet<>(); - RegionInfo region = null; - HRegionServer hrs = null; - ServerName firstFailedServer = null; - ServerName secondFailedServer = null; - for (int i = 0; i < NUM_RS; i++) { - hrs = rsts.get(i).getRegionServer(); - List regions = ProtobufUtil.getOnlineRegions(hrs.getRSRpcServices()); - if (regions.isEmpty()) continue; - region = regions.get(0); - regionSet.add(region); - firstFailedServer = hrs.getServerName(); - secondFailedServer = rsts.get((i + 1) % NUM_RS).getRegionServer().getServerName(); - break; - } - - slm.markRegionsRecovering(firstFailedServer, regionSet); - slm.markRegionsRecovering(secondFailedServer, regionSet); - - List recoveringRegions = ZKUtil.listChildrenNoWatch(zkw, - ZKUtil.joinZNode(zkw.znodePaths.recoveringRegionsZNode, region.getEncodedName())); - - assertEquals(recoveringRegions.size(), 2); - - // wait for splitLogWorker to mark them up because there is no WAL files recorded in ZK - final HRegionServer tmphrs = hrs; - TEST_UTIL.waitFor(60000, 1000, new Waiter.Predicate() { - @Override - public boolean evaluate() throws Exception { - return (tmphrs.getRecoveringRegions().isEmpty()); - } - }); - } finally { - if (ht != null) ht.close(); - if (zkw != null) zkw.close(); - } - } - - @Ignore("DLR is broken by HBASE-12751") @Test(timeout = 300000) - public void testReplayCmd() throws Exception { - LOG.info("testReplayCmd"); - conf.setBoolean(HConstants.DISTRIBUTED_LOG_REPLAY_KEY, true); - startCluster(NUM_RS); - final int NUM_REGIONS_TO_CREATE = 40; - // turn off load balancing to prevent regions from moving around otherwise - // they will consume recovered.edits - master.balanceSwitch(false); - - List rsts = cluster.getLiveRegionServerThreads(); - final ZooKeeperWatcher zkw = new ZooKeeperWatcher(conf, "table-creation", null); - Table ht = installTable(zkw, "table", "family", NUM_REGIONS_TO_CREATE); - try { - List regions = null; - HRegionServer hrs = null; - for (int i = 0; i < NUM_RS; i++) { - boolean isCarryingMeta = false; - hrs = rsts.get(i).getRegionServer(); - regions = ProtobufUtil.getOnlineRegions(hrs.getRSRpcServices()); - for (RegionInfo region : regions) { - if (region.isMetaRegion()) { - isCarryingMeta = true; - break; - } - } - if (isCarryingMeta) { - continue; - } - if (regions.size() > 0) break; - } - - this.prepareData(ht, Bytes.toBytes("family"), Bytes.toBytes("c1")); - String originalCheckSum = TEST_UTIL.checksumRows(ht); - - // abort RA and trigger replay - abortRSAndWaitForRecovery(hrs, zkw, NUM_REGIONS_TO_CREATE); - - assertEquals("Data should remain after reopening of regions", originalCheckSum, - TEST_UTIL.checksumRows(ht)); - } finally { - if (ht != null) ht.close(); - if (zkw != null) zkw.close(); - } - } - - @Ignore("DLR is broken by HBASE-12751") @Test(timeout = 300000) - public void testLogReplayForDisablingTable() throws Exception { - LOG.info("testLogReplayForDisablingTable"); - conf.setBoolean(HConstants.DISTRIBUTED_LOG_REPLAY_KEY, true); - startCluster(NUM_RS); - final int NUM_REGIONS_TO_CREATE = 40; - final int NUM_LOG_LINES = 1000; - - List rsts = cluster.getLiveRegionServerThreads(); - final ZooKeeperWatcher zkw = new ZooKeeperWatcher(conf, "table-creation", null); - Table disablingHT = installTable(zkw, "disableTable", "family", NUM_REGIONS_TO_CREATE); - Table ht = installTable(zkw, "table", "family", NUM_REGIONS_TO_CREATE, NUM_REGIONS_TO_CREATE); - try { - // turn off load balancing to prevent regions from moving around otherwise - // they will consume recovered.edits - master.balanceSwitch(false); - - List regions = null; - HRegionServer hrs = null; - boolean hasRegionsForBothTables = false; - String tableName = null; - for (int i = 0; i < NUM_RS; i++) { - tableName = null; - hasRegionsForBothTables = false; - boolean isCarryingSystem = false; - hrs = rsts.get(i).getRegionServer(); - regions = ProtobufUtil.getOnlineRegions(hrs.getRSRpcServices()); - for (RegionInfo region : regions) { - if (region.getTable().isSystemTable()) { - isCarryingSystem = true; - break; - } - if (tableName != null && - !tableName.equalsIgnoreCase(region.getTable().getNameAsString())) { - // make sure that we find a RS has online regions for both "table" and "disableTable" - hasRegionsForBothTables = true; - break; - } else if (tableName == null) { - tableName = region.getTable().getNameAsString(); - } - } - if (isCarryingSystem) { - continue; - } - if (hasRegionsForBothTables) { - break; - } - } - - // make sure we found a good RS - Assert.assertTrue(hasRegionsForBothTables); - - LOG.info("#regions = " + regions.size()); - Iterator it = regions.iterator(); - while (it.hasNext()) { - RegionInfo region = it.next(); - if (region.isMetaRegion()) { - it.remove(); - } - } - makeWAL(hrs, regions, "disableTable", "family", NUM_LOG_LINES, 100, false); - makeWAL(hrs, regions, "table", "family", NUM_LOG_LINES, 100); - - LOG.info("Disabling table\n"); - TEST_UTIL.getAdmin().disableTable(TableName.valueOf(name.getMethodName())); - TEST_UTIL.waitTableDisabled(TableName.valueOf(name.getMethodName()).getName()); - - // abort RS - LOG.info("Aborting region server: " + hrs.getServerName()); - hrs.abort("testing"); - - // wait for abort completes - TEST_UTIL.waitFor(120000, 200, new Waiter.Predicate() { - @Override - public boolean evaluate() throws Exception { - return (cluster.getLiveRegionServerThreads().size() <= (NUM_RS - 1)); - } - }); - - // wait for regions come online - TEST_UTIL.waitFor(180000, 200, new Waiter.Predicate() { - @Override - public boolean evaluate() throws Exception { - return (HBaseTestingUtility.getAllOnlineRegions(cluster).size() - >= (NUM_REGIONS_TO_CREATE + 1)); - } - }); - - // wait for all regions are fully recovered - TEST_UTIL.waitFor(180000, 200, new Waiter.Predicate() { - @Override - public boolean evaluate() throws Exception { - List recoveringRegions = zkw.getRecoverableZooKeeper().getChildren( - zkw.znodePaths.recoveringRegionsZNode, false); - ServerManager serverManager = master.getServerManager(); - return (!serverManager.areDeadServersInProgress() && - recoveringRegions != null && recoveringRegions.isEmpty()); - } - }); - - int count = 0; - FileSystem fs = master.getMasterFileSystem().getFileSystem(); - Path rootdir = FSUtils.getRootDir(conf); - Path tdir = FSUtils.getTableDir(rootdir, TableName.valueOf(name.getMethodName())); - for (RegionInfo hri : regions) { - Path editsdir = - WALSplitter.getRegionDirRecoveredEditsDir( - HRegion.getRegionDir(tdir, hri.getEncodedName())); - LOG.debug("checking edits dir " + editsdir); - if(!fs.exists(editsdir)) continue; - FileStatus[] files = fs.listStatus(editsdir, new PathFilter() { - @Override - public boolean accept(Path p) { - if (WALSplitter.isSequenceIdFile(p)) { - return false; - } - return true; - } - }); - if(files != null) { - for(FileStatus file : files) { - int c = countWAL(file.getPath(), fs, conf); - count += c; - LOG.info(c + " edits in " + file.getPath()); - } - } - } - - LOG.info("Verify edits in recovered.edits files"); - assertEquals(NUM_LOG_LINES, count); - LOG.info("Verify replayed edits"); - assertEquals(NUM_LOG_LINES, TEST_UTIL.countRows(ht)); - - // clean up - for (RegionInfo hri : regions) { - Path editsdir = - WALSplitter.getRegionDirRecoveredEditsDir( - HRegion.getRegionDir(tdir, hri.getEncodedName())); - fs.delete(editsdir, true); - } - disablingHT.close(); - } finally { - if (ht != null) ht.close(); - if (zkw != null) zkw.close(); - } - } - - @Ignore("DLR is broken by HBASE-12751") @Test(timeout = 300000) - public void testDisallowWritesInRecovering() throws Exception { - LOG.info("testDisallowWritesInRecovering"); - conf.setBoolean(HConstants.DISTRIBUTED_LOG_REPLAY_KEY, true); - conf.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 3); - conf.setBoolean(HConstants.DISALLOW_WRITES_IN_RECOVERING, true); - startCluster(NUM_RS); - final int NUM_REGIONS_TO_CREATE = 40; - // turn off load balancing to prevent regions from moving around otherwise - // they will consume recovered.edits - master.balanceSwitch(false); - - List rsts = cluster.getLiveRegionServerThreads(); - final ZooKeeperWatcher zkw = new ZooKeeperWatcher(conf, "table-creation", null); - Table ht = installTable(zkw, "table", "family", NUM_REGIONS_TO_CREATE); - try { - final SplitLogManager slm = master.getMasterWalManager().getSplitLogManager(); - - Set regionSet = new HashSet<>(); - RegionInfo region = null; - HRegionServer hrs = null; - HRegionServer dstRS = null; - for (int i = 0; i < NUM_RS; i++) { - hrs = rsts.get(i).getRegionServer(); - List regions = ProtobufUtil.getOnlineRegions(hrs.getRSRpcServices()); - if (regions.isEmpty()) continue; - region = regions.get(0); - regionSet.add(region); - dstRS = rsts.get((i+1) % NUM_RS).getRegionServer(); - break; - } - - slm.markRegionsRecovering(hrs.getServerName(), regionSet); - // move region in order for the region opened in recovering state - final RegionInfo hri = region; - final HRegionServer tmpRS = dstRS; - TEST_UTIL.getAdmin().move(region.getEncodedNameAsBytes(), - Bytes.toBytes(dstRS.getServerName().getServerName())); - // wait for region move completes - final RegionStates regionStates = - TEST_UTIL.getHBaseCluster().getMaster().getAssignmentManager().getRegionStates(); - TEST_UTIL.waitFor(45000, 200, new Waiter.Predicate() { - @Override - public boolean evaluate() throws Exception { - ServerName sn = regionStates.getRegionServerOfRegion(hri); - return (sn != null && sn.equals(tmpRS.getServerName())); - } - }); - - try { - byte[] key = region.getStartKey(); - if (key == null || key.length == 0) { - key = new byte[] { 0, 0, 0, 0, 1 }; - } - Put put = new Put(key); - put.addColumn(Bytes.toBytes("family"), Bytes.toBytes("c1"), new byte[]{'b'}); - ht.put(put); - } catch (IOException ioe) { - Assert.assertTrue(ioe instanceof RetriesExhaustedWithDetailsException); - RetriesExhaustedWithDetailsException re = (RetriesExhaustedWithDetailsException) ioe; - boolean foundRegionInRecoveryException = false; - for (Throwable t : re.getCauses()) { - if (t instanceof RegionInRecoveryException) { - foundRegionInRecoveryException = true; - break; - } - } - Assert.assertTrue( - "No RegionInRecoveryException. Following exceptions returned=" + re.getCauses(), - foundRegionInRecoveryException); - } - } finally { - if (ht != null) ht.close(); - if (ht != null) zkw.close(); - } - } - /** * The original intention of this test was to force an abort of a region * server and to make sure that the failure path in the region servers is @@ -1012,7 +337,8 @@ public class TestDistributedLogSplitting { * detects that the region server has aborted. * @throws Exception */ - @Ignore ("Disabled because flakey") @Test (timeout=300000) + // Was marked flaky before Distributed Log Replay cleanup. + @Test (timeout=300000) public void testWorkerAbort() throws Exception { LOG.info("testWorkerAbort"); startCluster(3); @@ -1112,16 +438,6 @@ public class TestDistributedLogSplitting { Thread.sleep(200); } - // wait for all regions are fully recovered - TEST_UTIL.waitFor(180000, 200, new Waiter.Predicate() { - @Override - public boolean evaluate() throws Exception { - List recoveringRegions = zkw.getRecoverableZooKeeper().getChildren( - zkw.znodePaths.recoveringRegionsZNode, false); - return (recoveringRegions != null && recoveringRegions.isEmpty()); - } - }); - assertEquals(NUM_REGIONS_TO_CREATE * NUM_ROWS_PER_REGION, TEST_UTIL.countRows(ht)); } finally { @@ -1200,255 +516,6 @@ public class TestDistributedLogSplitting { } } - @Ignore("DLR is broken by HBASE-12751") @Test(timeout = 300000) - public void testMetaRecoveryInZK() throws Exception { - LOG.info("testMetaRecoveryInZK"); - conf.setBoolean(HConstants.DISTRIBUTED_LOG_REPLAY_KEY, true); - startCluster(NUM_RS); - - // turn off load balancing to prevent regions from moving around otherwise - // they will consume recovered.edits - master.balanceSwitch(false); - final ZooKeeperWatcher zkw = new ZooKeeperWatcher(conf, "table-creation", null); - - // only testing meta recovery in ZK operation - HRegionServer hrs = findRSToKill(true, null); - List regions = ProtobufUtil.getOnlineRegions(hrs.getRSRpcServices()); - - LOG.info("#regions = " + regions.size()); - Set tmpRegions = new HashSet<>(); - tmpRegions.add(RegionInfoBuilder.FIRST_META_REGIONINFO); - master.getMasterWalManager().prepareLogReplay(hrs.getServerName(), tmpRegions); - Set userRegionSet = new HashSet<>(); - userRegionSet.addAll(regions); - master.getMasterWalManager().prepareLogReplay(hrs.getServerName(), userRegionSet); - boolean isMetaRegionInRecovery = false; - List recoveringRegions = - zkw.getRecoverableZooKeeper().getChildren(zkw.znodePaths.recoveringRegionsZNode, false); - for (String curEncodedRegionName : recoveringRegions) { - if (curEncodedRegionName.equals(RegionInfoBuilder.FIRST_META_REGIONINFO.getEncodedName())) { - isMetaRegionInRecovery = true; - break; - } - } - assertTrue(isMetaRegionInRecovery); - - master.getMasterWalManager().splitMetaLog(hrs.getServerName()); - - isMetaRegionInRecovery = false; - recoveringRegions = - zkw.getRecoverableZooKeeper().getChildren(zkw.znodePaths.recoveringRegionsZNode, false); - for (String curEncodedRegionName : recoveringRegions) { - if (curEncodedRegionName.equals(RegionInfoBuilder.FIRST_META_REGIONINFO.getEncodedName())) { - isMetaRegionInRecovery = true; - break; - } - } - // meta region should be recovered - assertFalse(isMetaRegionInRecovery); - zkw.close(); - } - - @Ignore("DLR is broken by HBASE-12751") @Test(timeout = 300000) - public void testSameVersionUpdatesRecovery() throws Exception { - LOG.info("testSameVersionUpdatesRecovery"); - conf.setLong("hbase.regionserver.hlog.blocksize", 15 * 1024); - conf.setBoolean(HConstants.DISTRIBUTED_LOG_REPLAY_KEY, true); - startCluster(NUM_RS); - final AtomicLong sequenceId = new AtomicLong(100); - final int NUM_REGIONS_TO_CREATE = 40; - final int NUM_LOG_LINES = 1000; - // turn off load balancing to prevent regions from moving around otherwise - // they will consume recovered.edits - master.balanceSwitch(false); - - List rsts = cluster.getLiveRegionServerThreads(); - final ZooKeeperWatcher zkw = new ZooKeeperWatcher(conf, "table-creation", null); - Table ht = installTable(zkw, name.getMethodName(), "family", NUM_REGIONS_TO_CREATE); - try { - List regions = null; - HRegionServer hrs = null; - for (int i = 0; i < NUM_RS; i++) { - boolean isCarryingMeta = false; - hrs = rsts.get(i).getRegionServer(); - regions = ProtobufUtil.getOnlineRegions(hrs.getRSRpcServices()); - for (RegionInfo region : regions) { - if (region.isMetaRegion()) { - isCarryingMeta = true; - break; - } - } - if (isCarryingMeta) { - continue; - } - break; - } - - LOG.info("#regions = " + regions.size()); - Iterator it = regions.iterator(); - while (it.hasNext()) { - RegionInfo region = it.next(); - if (region.isMetaRegion() - || region.getEncodedName().equals( - RegionInfoBuilder.FIRST_META_REGIONINFO.getEncodedName())) { - it.remove(); - } - } - if (regions.isEmpty()) return; - RegionInfo curRegionInfo = regions.get(0); - byte[] startRow = curRegionInfo.getStartKey(); - if (startRow == null || startRow.length == 0) { - startRow = new byte[] { 0, 0, 0, 0, 1 }; - } - byte[] row = Bytes.incrementBytes(startRow, 1); - // use last 5 bytes because HBaseTestingUtility.createMultiRegions use 5 bytes key - row = Arrays.copyOfRange(row, 3, 8); - long value = 0; - TableName tableName = TableName.valueOf(name.getMethodName()); - byte[] family = Bytes.toBytes("family"); - byte[] qualifier = Bytes.toBytes("c1"); - long timeStamp = System.currentTimeMillis(); - HTableDescriptor htd = new HTableDescriptor(tableName); - htd.addFamily(new HColumnDescriptor(family)); - final WAL wal = hrs.getWAL(curRegionInfo); - for (int i = 0; i < NUM_LOG_LINES; i += 1) { - WALEdit e = new WALEdit(); - value++; - e.add(new KeyValue(row, family, qualifier, timeStamp, Bytes.toBytes(value))); - wal.append(curRegionInfo, new WALKey(curRegionInfo.getEncodedNameAsBytes(), tableName, - System.currentTimeMillis()), e, true); - } - wal.sync(); - wal.shutdown(); - - // wait for abort completes - this.abortRSAndWaitForRecovery(hrs, zkw, NUM_REGIONS_TO_CREATE); - - // verify we got the last value - LOG.info("Verification Starts..."); - Get g = new Get(row); - Result r = ht.get(g); - long theStoredVal = Bytes.toLong(r.getValue(family, qualifier)); - assertEquals(value, theStoredVal); - - // after flush - LOG.info("Verification after flush..."); - TEST_UTIL.getAdmin().flush(tableName); - r = ht.get(g); - theStoredVal = Bytes.toLong(r.getValue(family, qualifier)); - assertEquals(value, theStoredVal); - } finally { - if (ht != null) ht.close(); - if (zkw != null) zkw.close(); - } - } - - @Ignore("DLR is broken by HBASE-12751") @Test(timeout = 300000) - public void testSameVersionUpdatesRecoveryWithCompaction() throws Exception { - LOG.info("testSameVersionUpdatesRecoveryWithWrites"); - conf.setLong("hbase.regionserver.hlog.blocksize", 15 * 1024); - conf.setBoolean(HConstants.DISTRIBUTED_LOG_REPLAY_KEY, true); - conf.setInt(HConstants.HREGION_MEMSTORE_FLUSH_SIZE, 30 * 1024); - conf.setInt("hbase.hstore.compactionThreshold", 3); - startCluster(NUM_RS); - final AtomicLong sequenceId = new AtomicLong(100); - final int NUM_REGIONS_TO_CREATE = 40; - final int NUM_LOG_LINES = 2000; - // turn off load balancing to prevent regions from moving around otherwise - // they will consume recovered.edits - master.balanceSwitch(false); - - List rsts = cluster.getLiveRegionServerThreads(); - final ZooKeeperWatcher zkw = new ZooKeeperWatcher(conf, "table-creation", null); - Table ht = installTable(zkw, "table", "family", NUM_REGIONS_TO_CREATE); - try { - List regions = null; - HRegionServer hrs = null; - for (int i = 0; i < NUM_RS; i++) { - boolean isCarryingMeta = false; - hrs = rsts.get(i).getRegionServer(); - regions = ProtobufUtil.getOnlineRegions(hrs.getRSRpcServices()); - for (RegionInfo region : regions) { - if (region.isMetaRegion()) { - isCarryingMeta = true; - break; - } - } - if (isCarryingMeta) { - continue; - } - break; - } - - LOG.info("#regions = " + regions.size()); - Iterator it = regions.iterator(); - while (it.hasNext()) { - RegionInfo region = it.next(); - if (region.isMetaRegion() - || region.getEncodedName().equals(RegionInfoBuilder.FIRST_META_REGIONINFO.getEncodedName())) { - it.remove(); - } - } - if (regions.isEmpty()) return; - RegionInfo curRegionInfo = regions.get(0); - byte[] startRow = curRegionInfo.getStartKey(); - if (startRow == null || startRow.length == 0) { - startRow = new byte[] { 0, 0, 0, 0, 1 }; - } - byte[] row = Bytes.incrementBytes(startRow, 1); - // use last 5 bytes because HBaseTestingUtility.createMultiRegions use 5 bytes key - row = Arrays.copyOfRange(row, 3, 8); - long value = 0; - final TableName tableName = TableName.valueOf(name.getMethodName()); - byte[] family = Bytes.toBytes("family"); - byte[] qualifier = Bytes.toBytes("c1"); - long timeStamp = System.currentTimeMillis(); - HTableDescriptor htd = new HTableDescriptor(tableName); - htd.addFamily(new HColumnDescriptor(family)); - final WAL wal = hrs.getWAL(curRegionInfo); - for (int i = 0; i < NUM_LOG_LINES; i += 1) { - WALEdit e = new WALEdit(); - value++; - e.add(new KeyValue(row, family, qualifier, timeStamp, Bytes.toBytes(value))); - wal.append(curRegionInfo, new WALKey(curRegionInfo.getEncodedNameAsBytes(), - tableName, System.currentTimeMillis()), e, true); - } - wal.sync(); - wal.shutdown(); - - // wait for abort completes - this.abortRSAndWaitForRecovery(hrs, zkw, NUM_REGIONS_TO_CREATE); - - // verify we got the last value - LOG.info("Verification Starts..."); - Get g = new Get(row); - Result r = ht.get(g); - long theStoredVal = Bytes.toLong(r.getValue(family, qualifier)); - assertEquals(value, theStoredVal); - - // after flush & compaction - LOG.info("Verification after flush..."); - TEST_UTIL.getAdmin().flush(tableName); - TEST_UTIL.getAdmin().compact(tableName); - - // wait for compaction completes - TEST_UTIL.waitFor(30000, 200, new Waiter.Predicate() { - @Override - public boolean evaluate() throws Exception { - return (TEST_UTIL.getAdmin() - .getCompactionState(tableName) == CompactionState.NONE); - } - }); - - r = ht.get(g); - theStoredVal = Bytes.toLong(r.getValue(family, qualifier)); - assertEquals(value, theStoredVal); - } finally { - if (ht != null) ht.close(); - if (zkw != null) zkw.close(); - } - } - @Test(timeout = 300000) public void testReadWriteSeqIdFiles() throws Exception { LOG.info("testReadWriteSeqIdFiles"); @@ -1578,8 +645,6 @@ public class TestDistributedLogSplitting { TableName fullTName = TableName.valueOf(tname); // remove root and meta region regions.remove(RegionInfoBuilder.FIRST_META_REGIONINFO); - // using one sequenceId for edits across all regions is ok. - final AtomicLong sequenceId = new AtomicLong(10); for(Iterator iter = regions.iterator(); iter.hasNext(); ) { @@ -1608,6 +673,7 @@ public class TestDistributedLogSplitting { int[] counts = new int[n]; // sync every ~30k to line up with desired wal rolls final int syncEvery = 30 * 1024 / edit_size; + MultiVersionConcurrencyControl mvcc = new MultiVersionConcurrencyControl(); if (n > 0) { for (int i = 0; i < num_edits; i += 1) { WALEdit e = new WALEdit(); @@ -1619,13 +685,12 @@ public class TestDistributedLogSplitting { } byte[] row = Bytes.incrementBytes(startRow, counts[i % n]); row = Arrays.copyOfRange(row, 3, 8); // use last 5 bytes because - // HBaseTestingUtility.createMultiRegions use 5 bytes - // key + // HBaseTestingUtility.createMultiRegions use 5 bytes key byte[] qualifier = Bytes.toBytes("c" + Integer.toString(i)); e.add(new KeyValue(row, family, qualifier, System.currentTimeMillis(), value)); log.append(curRegionInfo, new WALKey(curRegionInfo.getEncodedNameAsBytes(), fullTName, - System.currentTimeMillis()), e, true); + System.currentTimeMillis(), mvcc), e, true); if (0 == i % syncEvery) { log.sync(); } @@ -1688,37 +753,6 @@ public class TestDistributedLogSplitting { } } - /** - * Load table with puts and deletes with expected values so that we can verify later - */ - private void prepareData(final Table t, final byte[] f, final byte[] column) throws IOException { - byte[] k = new byte[3]; - - // add puts - List puts = new ArrayList<>(); - for (byte b1 = 'a'; b1 <= 'z'; b1++) { - for (byte b2 = 'a'; b2 <= 'z'; b2++) { - for (byte b3 = 'a'; b3 <= 'z'; b3++) { - k[0] = b1; - k[1] = b2; - k[2] = b3; - Put put = new Put(k); - put.addColumn(f, column, k); - puts.add(put); - } - } - } - t.put(puts); - // add deletes - for (byte b3 = 'a'; b3 <= 'z'; b3++) { - k[0] = 'a'; - k[1] = 'a'; - k[2] = b3; - Delete del = new Delete(k); - t.delete(del); - } - } - private void waitForCounter(LongAdder ctr, long oldval, long newval, long timems) { long curt = System.currentTimeMillis(); http://git-wip-us.apache.org/repos/asf/hbase/blob/a6d8023f/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterWalManager.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterWalManager.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterWalManager.java deleted file mode 100644 index 8641b20..0000000 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterWalManager.java +++ /dev/null @@ -1,105 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hbase.master; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; - -import java.util.HashSet; -import java.util.Set; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hbase.HBaseTestingUtility; -import org.apache.hadoop.hbase.HConstants; -import org.apache.hadoop.hbase.ServerName; -import org.apache.hadoop.hbase.SplitLogTask; -import org.apache.hadoop.hbase.testclassification.MasterTests; -import org.apache.hadoop.hbase.testclassification.MediumTests; -import org.apache.hadoop.hbase.util.FSUtils; -import org.apache.hadoop.hbase.zookeeper.ZKSplitLog; -import org.apache.hadoop.hbase.zookeeper.ZKUtil; -import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher; -import org.apache.zookeeper.CreateMode; -import org.apache.zookeeper.ZooDefs.Ids; -import org.junit.AfterClass; -import org.junit.BeforeClass; -import org.junit.Test; -import org.junit.experimental.categories.Category; - -/** - * Test the master wal manager in a local cluster - */ -@Category({MasterTests.class, MediumTests.class}) -public class TestMasterWalManager { - private static final Log LOG = LogFactory.getLog(TestMasterWalManager.class); - private static final HBaseTestingUtility UTIL = new HBaseTestingUtility(); - - @BeforeClass - public static void setupTest() throws Exception { - UTIL.startMiniCluster(); - } - - @AfterClass - public static void teardownTest() throws Exception { - UTIL.shutdownMiniCluster(); - } - - @Test - public void testRemoveStaleRecoveringRegionsDuringMasterInitialization() throws Exception { - // this test is for when distributed log replay is enabled - if (!UTIL.getConfiguration().getBoolean(HConstants.DISTRIBUTED_LOG_REPLAY_KEY, false)) return; - - LOG.info("Starting testRemoveStaleRecoveringRegionsDuringMasterInitialization"); - HMaster master = UTIL.getMiniHBaseCluster().getMaster(); - MasterWalManager mwm = master.getMasterWalManager(); - - String failedRegion = "failedRegoin1"; - String staleRegion = "staleRegion"; - ServerName inRecoveryServerName = ServerName.valueOf("mgr,1,1"); - ServerName previouselyFaildServerName = ServerName.valueOf("previous,1,1"); - String walPath = "/hbase/data/.logs/" + inRecoveryServerName.getServerName() - + "-splitting/test"; - // Create a ZKW to use in the test - ZooKeeperWatcher zkw = HBaseTestingUtility.getZooKeeperWatcher(UTIL); - zkw.getRecoverableZooKeeper().create(ZKSplitLog.getEncodedNodeName(zkw, walPath), - new SplitLogTask.Owned(inRecoveryServerName).toByteArray(), - Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT); - String staleRegionPath = ZKUtil.joinZNode(zkw.znodePaths.recoveringRegionsZNode, staleRegion); - ZKUtil.createWithParents(zkw, staleRegionPath); - String inRecoveringRegionPath = ZKUtil.joinZNode(zkw.znodePaths.recoveringRegionsZNode, - failedRegion); - inRecoveringRegionPath = ZKUtil.joinZNode(inRecoveringRegionPath, - inRecoveryServerName.getServerName()); - ZKUtil.createWithParents(zkw, inRecoveringRegionPath); - Set servers = new HashSet<>(); - servers.add(previouselyFaildServerName); - mwm.removeStaleRecoveringRegionsFromZK(servers); - - // verification - assertFalse(ZKUtil.checkExists(zkw, staleRegionPath) != -1); - assertTrue(ZKUtil.checkExists(zkw, inRecoveringRegionPath) != -1); - - ZKUtil.deleteChildrenRecursively(zkw, zkw.znodePaths.recoveringRegionsZNode); - ZKUtil.deleteChildrenRecursively(zkw, zkw.znodePaths.splitLogZNode); - zkw.close(); - } -} http://git-wip-us.apache.org/repos/asf/hbase/blob/a6d8023f/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestSplitLogManager.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestSplitLogManager.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestSplitLogManager.java index 3dcd849..ca64326 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestSplitLogManager.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestSplitLogManager.java @@ -36,7 +36,6 @@ import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; import java.io.IOException; -import java.util.List; import java.util.Map; import java.util.UUID; import java.util.concurrent.atomic.LongAdder; @@ -49,7 +48,6 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.CoordinatedStateManager; import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.HConstants; -import org.apache.hadoop.hbase.HRegionInfo; import org.apache.hadoop.hbase.ServerName; import org.apache.hadoop.hbase.SplitLogCounters; import org.apache.hadoop.hbase.SplitLogTask; @@ -59,7 +57,6 @@ import org.apache.hadoop.hbase.coordination.ZkCoordinatedStateManager; import org.apache.hadoop.hbase.master.SplitLogManager.Task; import org.apache.hadoop.hbase.master.SplitLogManager.TaskBatch; import org.apache.hadoop.hbase.regionserver.TestMasterAddressTracker.NodeCreationListener; -import org.apache.hadoop.hbase.shaded.protobuf.generated.ZooKeeperProtos.SplitLogTask.RecoveryMode; import org.apache.hadoop.hbase.testclassification.MasterTests; import org.apache.hadoop.hbase.testclassification.MediumTests; import org.apache.hadoop.hbase.zookeeper.ZKSplitLog; @@ -73,7 +70,6 @@ import org.apache.zookeeper.ZooDefs.Ids; import org.junit.After; import org.junit.Assert; import org.junit.Before; -import org.junit.Ignore; import org.junit.Test; import org.junit.experimental.categories.Category; import org.mockito.Mockito; @@ -93,7 +89,6 @@ public class TestSplitLogManager { private SplitLogManager slm; private Configuration conf; private int to; - private RecoveryMode mode; private static HBaseTestingUtility TEST_UTIL; @@ -153,10 +148,6 @@ public class TestSplitLogManager { conf.setInt("hbase.splitlog.manager.timeoutmonitor.period", 100); to = to + 16 * 100; - - this.mode = - (conf.getBoolean(HConstants.DISTRIBUTED_LOG_REPLAY_KEY, false) ? RecoveryMode.LOG_REPLAY - : RecoveryMode.LOG_SPLITTING); } @After @@ -245,7 +236,7 @@ public class TestSplitLogManager { LOG.info("TestOrphanTaskAcquisition"); String tasknode = ZKSplitLog.getEncodedNodeName(zkw, "orphan/test/slash"); - SplitLogTask slt = new SplitLogTask.Owned(master.getServerName(), this.mode); + SplitLogTask slt = new SplitLogTask.Owned(master.getServerName()); zkw.getRecoverableZooKeeper().create(tasknode, slt.toByteArray(), Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT); @@ -270,7 +261,7 @@ public class TestSplitLogManager { " startup"); String tasknode = ZKSplitLog.getEncodedNodeName(zkw, "orphan/test/slash"); //create an unassigned orphan task - SplitLogTask slt = new SplitLogTask.Unassigned(master.getServerName(), this.mode); + SplitLogTask slt = new SplitLogTask.Unassigned(master.getServerName()); zkw.getRecoverableZooKeeper().create(tasknode, slt.toByteArray(), Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT); int version = ZKUtil.checkExists(zkw, tasknode); @@ -305,19 +296,19 @@ public class TestSplitLogManager { final ServerName worker1 = ServerName.valueOf("worker1,1,1"); final ServerName worker2 = ServerName.valueOf("worker2,1,1"); final ServerName worker3 = ServerName.valueOf("worker3,1,1"); - SplitLogTask slt = new SplitLogTask.Owned(worker1, this.mode); + SplitLogTask slt = new SplitLogTask.Owned(worker1); ZKUtil.setData(zkw, tasknode, slt.toByteArray()); waitForCounter(tot_mgr_heartbeat, 0, 1, to/2); waitForCounter(tot_mgr_resubmit, 0, 1, to + to/2); int version1 = ZKUtil.checkExists(zkw, tasknode); assertTrue(version1 > version); - slt = new SplitLogTask.Owned(worker2, this.mode); + slt = new SplitLogTask.Owned(worker2); ZKUtil.setData(zkw, tasknode, slt.toByteArray()); waitForCounter(tot_mgr_heartbeat, 1, 2, to/2); waitForCounter(tot_mgr_resubmit, 1, 2, to + to/2); int version2 = ZKUtil.checkExists(zkw, tasknode); assertTrue(version2 > version1); - slt = new SplitLogTask.Owned(worker3, this.mode); + slt = new SplitLogTask.Owned(worker3); ZKUtil.setData(zkw, tasknode, slt.toByteArray()); waitForCounter(tot_mgr_heartbeat, 2, 3, to/2); waitForCounter(tot_mgr_resubmit_threshold_reached, 0, 1, to + to/2); @@ -335,7 +326,7 @@ public class TestSplitLogManager { String tasknode = submitTaskAndWait(batch, "foo/1"); int version = ZKUtil.checkExists(zkw, tasknode); final ServerName worker1 = ServerName.valueOf("worker1,1,1"); - SplitLogTask slt = new SplitLogTask.Owned(worker1, this.mode); + SplitLogTask slt = new SplitLogTask.Owned(worker1); ZKUtil.setData(zkw, tasknode, slt.toByteArray()); waitForCounter(tot_mgr_heartbeat, 0, 1, to/2); waitForCounter(new Expr() { @@ -362,7 +353,7 @@ public class TestSplitLogManager { TaskBatch batch = new TaskBatch(); String tasknode = submitTaskAndWait(batch, "foo/1"); final ServerName worker1 = ServerName.valueOf("worker1,1,1"); - SplitLogTask slt = new SplitLogTask.Done(worker1, this.mode); + SplitLogTask slt = new SplitLogTask.Done(worker1); ZKUtil.setData(zkw, tasknode, slt.toByteArray()); synchronized (batch) { while (batch.installed != batch.done) { @@ -383,7 +374,7 @@ public class TestSplitLogManager { String tasknode = submitTaskAndWait(batch, "foo/1"); final ServerName worker1 = ServerName.valueOf("worker1,1,1"); - SplitLogTask slt = new SplitLogTask.Err(worker1, this.mode); + SplitLogTask slt = new SplitLogTask.Err(worker1); ZKUtil.setData(zkw, tasknode, slt.toByteArray()); synchronized (batch) { @@ -407,7 +398,7 @@ public class TestSplitLogManager { assertEquals(tot_mgr_resubmit.sum(), 0); final ServerName worker1 = ServerName.valueOf("worker1,1,1"); assertEquals(tot_mgr_resubmit.sum(), 0); - SplitLogTask slt = new SplitLogTask.Resigned(worker1, this.mode); + SplitLogTask slt = new SplitLogTask.Resigned(worker1); assertEquals(tot_mgr_resubmit.sum(), 0); ZKUtil.setData(zkw, tasknode, slt.toByteArray()); ZKUtil.checkExists(zkw, tasknode); @@ -430,7 +421,7 @@ public class TestSplitLogManager { // create an orphan task in OWNED state String tasknode1 = ZKSplitLog.getEncodedNodeName(zkw, "orphan/1"); final ServerName worker1 = ServerName.valueOf("worker1,1,1"); - SplitLogTask slt = new SplitLogTask.Owned(worker1, this.mode); + SplitLogTask slt = new SplitLogTask.Owned(worker1); zkw.getRecoverableZooKeeper().create(tasknode1, slt.toByteArray(), Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT); @@ -445,7 +436,7 @@ public class TestSplitLogManager { for (int i = 0; i < (3 * to)/100; i++) { Thread.sleep(100); final ServerName worker2 = ServerName.valueOf("worker1,1,1"); - slt = new SplitLogTask.Owned(worker2, this.mode); + slt = new SplitLogTask.Owned(worker2); ZKUtil.setData(zkw, tasknode1, slt.toByteArray()); } @@ -469,7 +460,7 @@ public class TestSplitLogManager { String tasknode = submitTaskAndWait(batch, "foo/1"); int version = ZKUtil.checkExists(zkw, tasknode); final ServerName worker1 = ServerName.valueOf("worker1,1,1"); - SplitLogTask slt = new SplitLogTask.Owned(worker1, this.mode); + SplitLogTask slt = new SplitLogTask.Owned(worker1); ZKUtil.setData(zkw, tasknode, slt.toByteArray()); if (tot_mgr_heartbeat.sum() == 0) waitForCounter(tot_mgr_heartbeat, 0, 1, to/2); slm.handleDeadWorker(worker1); @@ -494,7 +485,7 @@ public class TestSplitLogManager { String tasknode = submitTaskAndWait(batch, "foo/1"); final ServerName worker1 = ServerName.valueOf("worker1,1,1"); - SplitLogTask slt = new SplitLogTask.Owned(worker1, this.mode); + SplitLogTask slt = new SplitLogTask.Owned(worker1); ZKUtil.setData(zkw, tasknode, slt.toByteArray()); if (tot_mgr_heartbeat.sum() == 0) waitForCounter(tot_mgr_heartbeat, 0, 1, to/2); @@ -544,7 +535,7 @@ public class TestSplitLogManager { while (!done) { for (Map.Entry entry : slm.getTasks().entrySet()) { final ServerName worker1 = ServerName.valueOf("worker1,1,1"); - SplitLogTask slt = new SplitLogTask.Done(worker1, RecoveryMode.LOG_SPLITTING); + SplitLogTask slt = new SplitLogTask.Done(worker1); boolean encounteredZKException = false; try { ZKUtil.setData(zkw, entry.getKey(), slt.toByteArray()); @@ -564,50 +555,4 @@ public class TestSplitLogManager { assertFalse(fs.exists(logDirPath)); } - - /** - * The following test case is aiming to test the situation when distributedLogReplay is turned off - * and restart a cluster there should no recovery regions in ZK left. - * @throws Exception - */ - @Test(timeout = 300000) - public void testRecoveryRegionRemovedFromZK() throws Exception { - LOG.info("testRecoveryRegionRemovedFromZK"); - conf.setBoolean(HConstants.DISTRIBUTED_LOG_REPLAY_KEY, false); - String nodePath = - ZKUtil.joinZNode(zkw.znodePaths.recoveringRegionsZNode, - HRegionInfo.FIRST_META_REGIONINFO.getEncodedName()); - ZKUtil.createSetData(zkw, nodePath, ZKUtil.positionToByteArray(0L)); - - slm = new SplitLogManager(master, conf); - slm.removeStaleRecoveringRegions(null); - - List recoveringRegions = - zkw.getRecoverableZooKeeper().getChildren(zkw.znodePaths.recoveringRegionsZNode, false); - - assertTrue("Recovery regions isn't cleaned", recoveringRegions.isEmpty()); - } - - @Ignore("DLR is broken by HBASE-12751") @Test(timeout=60000) - public void testGetPreviousRecoveryMode() throws Exception { - LOG.info("testGetPreviousRecoveryMode"); - SplitLogCounters.resetCounters(); - // Not actually enabling DLR for the cluster, just for the ZkCoordinatedStateManager to use. - // The test is just manipulating ZK manually anyways. - conf.setBoolean(HConstants.DISTRIBUTED_LOG_REPLAY_KEY, true); - - zkw.getRecoverableZooKeeper().create(ZKSplitLog.getEncodedNodeName(zkw, "testRecovery"), - new SplitLogTask.Unassigned( - ServerName.valueOf("mgr,1,1"), RecoveryMode.LOG_SPLITTING).toByteArray(), - Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT); - - slm = new SplitLogManager(master, conf); - LOG.info("Mode1=" + slm.getRecoveryMode()); - assertTrue(slm.isLogSplitting()); - zkw.getRecoverableZooKeeper().delete(ZKSplitLog.getEncodedNodeName(zkw, "testRecovery"), -1); - LOG.info("Mode2=" + slm.getRecoveryMode()); - slm.setRecoveryMode(false); - LOG.info("Mode3=" + slm.getRecoveryMode()); - assertTrue("Mode4=" + slm.getRecoveryMode(), slm.isLogReplaying()); - } } http://git-wip-us.apache.org/repos/asf/hbase/blob/a6d8023f/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java index fec7151..127f949 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java @@ -48,7 +48,6 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; import java.util.Collections; -import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.NavigableMap; @@ -134,7 +133,6 @@ import org.apache.hadoop.hbase.monitoring.TaskMonitor; import org.apache.hadoop.hbase.regionserver.HRegion.RegionScannerImpl; import org.apache.hadoop.hbase.regionserver.Region.RowLock; import org.apache.hadoop.hbase.regionserver.TestHStore.FaultyFileSystem; -import org.apache.hadoop.hbase.regionserver.handler.FinishRegionRecoveringHandler; import org.apache.hadoop.hbase.regionserver.wal.FSHLog; import org.apache.hadoop.hbase.regionserver.wal.MetricsWAL; import org.apache.hadoop.hbase.regionserver.wal.MetricsWALSource; @@ -176,7 +174,6 @@ import org.mockito.invocation.InvocationOnMock; import org.mockito.stubbing.Answer; import org.apache.hadoop.hbase.shaded.com.google.common.collect.Lists; -import org.apache.hadoop.hbase.shaded.com.google.common.collect.Maps; import org.apache.hadoop.hbase.shaded.com.google.protobuf.ByteString; import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil; import org.apache.hadoop.hbase.shaded.protobuf.generated.WALProtos.CompactionDescriptor; @@ -5902,93 +5899,6 @@ public class TestHRegion { } } - @Test - public void testOpenRegionWrittenToWALForLogReplay() throws Exception { - // similar to the above test but with distributed log replay - final ServerName serverName = ServerName.valueOf(name.getMethodName(), 100, 42); - final RegionServerServices rss = spy(TEST_UTIL.createMockRegionServerService(serverName)); - - HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(name.getMethodName())); - htd.addFamily(new HColumnDescriptor(fam1)); - htd.addFamily(new HColumnDescriptor(fam2)); - - HRegionInfo hri = new HRegionInfo(htd.getTableName(), - HConstants.EMPTY_BYTE_ARRAY, HConstants.EMPTY_BYTE_ARRAY); - - // open the region w/o rss and wal and flush some files - HRegion region = - HBaseTestingUtility.createRegionAndWAL(hri, TEST_UTIL.getDataTestDir(), TEST_UTIL - .getConfiguration(), htd); - assertNotNull(region); - - // create a file in fam1 for the region before opening in OpenRegionHandler - region.put(new Put(Bytes.toBytes("a")).addColumn(fam1, fam1, fam1)); - region.flush(true); - HBaseTestingUtility.closeRegionAndWAL(region); - - ArgumentCaptor editCaptor = ArgumentCaptor.forClass(WALEdit.class); - - // capture append() calls - WAL wal = mockWAL(); - when(rss.getWAL((HRegionInfo) any())).thenReturn(wal); - - // add the region to recovering regions - HashMap recoveringRegions = Maps.newHashMap(); - recoveringRegions.put(region.getRegionInfo().getEncodedName(), null); - when(rss.getRecoveringRegions()).thenReturn(recoveringRegions); - - try { - Configuration conf = new Configuration(TEST_UTIL.getConfiguration()); - conf.set(HConstants.REGION_IMPL, HRegionWithSeqId.class.getName()); - region = HRegion.openHRegion(hri, htd, rss.getWAL(hri), - conf, rss, null); - - // verify that we have not appended region open event to WAL because this region is still - // recovering - verify(wal, times(0)).append((HRegionInfo)any(), (WALKey)any() - , editCaptor.capture(), anyBoolean()); - - // not put the region out of recovering state - new FinishRegionRecoveringHandler(rss, region.getRegionInfo().getEncodedName(), "/foo") - .prepare().process(); - - // now we should have put the entry - verify(wal, times(1)).append((HRegionInfo)any(), (WALKey)any() - , editCaptor.capture(), anyBoolean()); - - WALEdit edit = editCaptor.getValue(); - assertNotNull(edit); - assertNotNull(edit.getCells()); - assertEquals(1, edit.getCells().size()); - RegionEventDescriptor desc = WALEdit.getRegionEventDescriptor(edit.getCells().get(0)); - assertNotNull(desc); - - LOG.info("RegionEventDescriptor from WAL: " + desc); - - assertEquals(RegionEventDescriptor.EventType.REGION_OPEN, desc.getEventType()); - assertTrue(Bytes.equals(desc.getTableName().toByteArray(), htd.getTableName().toBytes())); - assertTrue(Bytes.equals(desc.getEncodedRegionName().toByteArray(), - hri.getEncodedNameAsBytes())); - assertTrue(desc.getLogSequenceNumber() > 0); - assertEquals(serverName, ProtobufUtil.toServerName(desc.getServer())); - assertEquals(2, desc.getStoresCount()); - - StoreDescriptor store = desc.getStores(0); - assertTrue(Bytes.equals(store.getFamilyName().toByteArray(), fam1)); - assertEquals(store.getStoreHomeDir(), Bytes.toString(fam1)); - assertEquals(1, store.getStoreFileCount()); // 1store file - assertFalse(store.getStoreFile(0).contains("/")); // ensure path is relative - - store = desc.getStores(1); - assertTrue(Bytes.equals(store.getFamilyName().toByteArray(), fam2)); - assertEquals(store.getStoreHomeDir(), Bytes.toString(fam2)); - assertEquals(0, store.getStoreFileCount()); // no store files - - } finally { - HBaseTestingUtility.closeRegionAndWAL(region); - } - } - /** * Utility method to setup a WAL mock. * Needs to do the bit where we close latch on the WALKey on append else test hangs. http://git-wip-us.apache.org/repos/asf/hbase/blob/a6d8023f/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestPerColumnFamilyFlush.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestPerColumnFamilyFlush.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestPerColumnFamilyFlush.java index 49a61c5..b8155e4 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestPerColumnFamilyFlush.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestPerColumnFamilyFlush.java @@ -46,7 +46,6 @@ import org.apache.hadoop.hbase.util.JVMClusterUtil; import org.apache.hadoop.hbase.util.Pair; import org.apache.hadoop.hbase.wal.AbstractFSWALProvider; import org.apache.hadoop.hbase.wal.WAL; -import org.junit.Ignore; import org.junit.Test; import org.junit.experimental.categories.Category; @@ -417,20 +416,9 @@ public class TestPerColumnFamilyFlush { } } - // Test Log Replay with Distributed Replay on. - // In distributed log replay, the log splitters ask the master for the - // last flushed sequence id for a region. This test would ensure that we - // are doing the book-keeping correctly. - @Ignore("DLR is broken by HBASE-12751") @Test(timeout = 180000) - public void testLogReplayWithDistributedReplay() throws Exception { - TEST_UTIL.getConfiguration().setBoolean(HConstants.DISTRIBUTED_LOG_REPLAY_KEY, true); - doTestLogReplay(); - } - // Test Log Replay with Distributed log split on. @Test(timeout = 180000) public void testLogReplayWithDistributedLogSplit() throws Exception { - TEST_UTIL.getConfiguration().setBoolean(HConstants.DISTRIBUTED_LOG_REPLAY_KEY, false); doTestLogReplay(); } http://git-wip-us.apache.org/repos/asf/hbase/blob/a6d8023f/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionReplicaFailover.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionReplicaFailover.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionReplicaFailover.java index ffcc5c0..7a6e2fb 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionReplicaFailover.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionReplicaFailover.java @@ -22,8 +22,6 @@ package org.apache.hadoop.hbase.regionserver; import static org.junit.Assert.*; import java.io.IOException; -import java.util.Arrays; -import java.util.Collection; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicReference; @@ -57,14 +55,10 @@ import org.junit.Rule; import org.junit.Test; import org.junit.experimental.categories.Category; import org.junit.rules.TestName; -import org.junit.runner.RunWith; -import org.junit.runners.Parameterized; -import org.junit.runners.Parameterized.Parameters; /** * Tests failover of secondary region replicas. */ -@RunWith(Parameterized.class) @Category(LargeTests.class) public class TestRegionReplicaFailover { @@ -90,19 +84,6 @@ public class TestRegionReplicaFailover { private HTableDescriptor htd; - /* - * We are testing with dist log split and dist log replay separately - */ - @Parameters - public static Collection getParameters() { - Object[][] params = - new Boolean[][] { /*{true}, Disable DLR!!! It is going to be removed*/ {false} }; - return Arrays.asList(params); - } - - @Parameterized.Parameter(0) - public boolean distributedLogReplay; - @Before public void before() throws Exception { Configuration conf = HTU.getConfiguration(); @@ -112,7 +93,6 @@ public class TestRegionReplicaFailover { conf.setBoolean(ServerRegionReplicaUtil.REGION_REPLICA_WAIT_FOR_PRIMARY_FLUSH_CONF_KEY, true); conf.setInt("replication.stats.thread.period.seconds", 5); conf.setBoolean("hbase.tests.use.shortcircuit.reads", false); - conf.setBoolean(HConstants.DISTRIBUTED_LOG_REPLAY_KEY, distributedLogReplay); HTU.startMiniCluster(NB_SERVERS); htd = HTU.createTableDescriptor( http://git-wip-us.apache.org/repos/asf/hbase/blob/a6d8023f/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerNoMaster.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerNoMaster.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerNoMaster.java index f8b9f6e..1f7320e 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerNoMaster.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerNoMaster.java @@ -153,7 +153,7 @@ public class TestRegionServerNoMaster { public static void openRegion(HBaseTestingUtility HTU, HRegionServer rs, HRegionInfo hri) throws Exception { AdminProtos.OpenRegionRequest orr = - RequestConverter.buildOpenRegionRequest(rs.getServerName(), hri, null, null); + RequestConverter.buildOpenRegionRequest(rs.getServerName(), hri, null); AdminProtos.OpenRegionResponse responseOpen = rs.rpcServices.openRegion(null, orr); Assert.assertTrue(responseOpen.getOpeningStateCount() == 1); @@ -295,7 +295,7 @@ public class TestRegionServerNoMaster { closeRegionNoZK(); try { AdminProtos.OpenRegionRequest orr = RequestConverter.buildOpenRegionRequest( - earlierServerName, hri, null, null); + earlierServerName, hri, null); getRS().getRSRpcServices().openRegion(null, orr); Assert.fail("The openRegion should have been rejected"); } catch (org.apache.hadoop.hbase.shaded.com.google.protobuf.ServiceException se) { http://git-wip-us.apache.org/repos/asf/hbase/blob/a6d8023f/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitLogWorker.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitLogWorker.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitLogWorker.java index 40077f9..3eec9d8 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitLogWorker.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitLogWorker.java @@ -37,7 +37,6 @@ import org.apache.hadoop.hbase.ChoreService; import org.apache.hadoop.hbase.CoordinatedStateManager; import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.HBaseTestingUtility; -import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.Server; import org.apache.hadoop.hbase.ServerName; import org.apache.hadoop.hbase.SplitLogCounters; @@ -47,7 +46,6 @@ import org.apache.hadoop.hbase.client.ClusterConnection; import org.apache.hadoop.hbase.coordination.ZkCoordinatedStateManager; import org.apache.hadoop.hbase.executor.ExecutorService; import org.apache.hadoop.hbase.executor.ExecutorType; -import org.apache.hadoop.hbase.shaded.protobuf.generated.ZooKeeperProtos.SplitLogTask.RecoveryMode; import org.apache.hadoop.hbase.testclassification.MediumTests; import org.apache.hadoop.hbase.testclassification.RegionServerTests; import org.apache.hadoop.hbase.util.CancelableProgressable; @@ -78,7 +76,6 @@ public class TestSplitLogWorker { private ZooKeeperWatcher zkw; private SplitLogWorker slw; private ExecutorService executorService; - private RecoveryMode mode; class DummyServer implements Server { private ZooKeeperWatcher zkw; @@ -212,8 +209,6 @@ public class TestSplitLogWorker { SplitLogCounters.resetCounters(); executorService = new ExecutorService("TestSplitLogWorker"); executorService.startExecutorService(ExecutorType.RS_LOG_REPLAY_OPS, 10); - this.mode = (conf.getBoolean(HConstants.DISTRIBUTED_LOG_REPLAY_KEY, false) ? - RecoveryMode.LOG_REPLAY : RecoveryMode.LOG_SPLITTING); } @After @@ -228,7 +223,7 @@ public class TestSplitLogWorker { new SplitLogWorker.TaskExecutor() { @Override - public Status exec(String name, RecoveryMode mode, CancelableProgressable p) { + public Status exec(String name, CancelableProgressable p) { while (true) { try { Thread.sleep(1000); @@ -251,7 +246,7 @@ public class TestSplitLogWorker { final ServerName RS = ServerName.valueOf("rs,1,1"); RegionServerServices mockedRS = getRegionServer(RS); zkw.getRecoverableZooKeeper().create(ZKSplitLog.getEncodedNodeName(zkw, TATAS), - new SplitLogTask.Unassigned(ServerName.valueOf("mgr,1,1"), this.mode).toByteArray(), + new SplitLogTask.Unassigned(ServerName.valueOf("mgr,1,1")).toByteArray(), Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT); @@ -287,7 +282,7 @@ public class TestSplitLogWorker { final ServerName SVR1 = ServerName.valueOf("svr1,1,1"); final ServerName SVR2 = ServerName.valueOf("svr2,1,1"); zkw.getRecoverableZooKeeper().create(ZKSplitLog.getEncodedNodeName(zkw, TRFT), - new SplitLogTask.Unassigned(MANAGER, this.mode).toByteArray(), + new SplitLogTask.Unassigned(MANAGER).toByteArray(), Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT); RegionServerServices mockedRS1 = getRegionServer(SVR1); RegionServerServices mockedRS2 = getRegionServer(SVR2); @@ -330,7 +325,7 @@ public class TestSplitLogWorker { // this time create a task node after starting the splitLogWorker zkw.getRecoverableZooKeeper().create(PATH, - new SplitLogTask.Unassigned(MANAGER, this.mode).toByteArray(), + new SplitLogTask.Unassigned(MANAGER).toByteArray(), Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT); waitForCounter(SplitLogCounters.tot_wkr_task_acquired, 0, 1, WAIT_TIME); @@ -338,7 +333,7 @@ public class TestSplitLogWorker { byte [] bytes = ZKUtil.getData(zkw, PATH); SplitLogTask slt = SplitLogTask.parseFrom(bytes); assertTrue(slt.isOwned(SRV)); - slt = new SplitLogTask.Owned(MANAGER, this.mode); + slt = new SplitLogTask.Owned(MANAGER); ZKUtil.setData(zkw, PATH, slt.toByteArray()); waitForCounter(SplitLogCounters.tot_wkr_preempt_task, 0, 1, WAIT_TIME); } finally { @@ -362,7 +357,7 @@ public class TestSplitLogWorker { waitForCounter(SplitLogCounters.tot_wkr_task_grabing, 0, 1, WAIT_TIME); SplitLogTask unassignedManager = - new SplitLogTask.Unassigned(MANAGER, this.mode); + new SplitLogTask.Unassigned(MANAGER); zkw.getRecoverableZooKeeper().create(PATH1, unassignedManager.toByteArray(), Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT); @@ -376,7 +371,7 @@ public class TestSplitLogWorker { // preempt the first task, have it owned by another worker final ServerName anotherWorker = ServerName.valueOf("another-worker,1,1"); - SplitLogTask slt = new SplitLogTask.Owned(anotherWorker, this.mode); + SplitLogTask slt = new SplitLogTask.Owned(anotherWorker); ZKUtil.setData(zkw, PATH1, slt.toByteArray()); waitForCounter(SplitLogCounters.tot_wkr_preempt_task, 0, 1, WAIT_TIME); @@ -402,7 +397,7 @@ public class TestSplitLogWorker { Thread.sleep(100); String task = ZKSplitLog.getEncodedNodeName(zkw, "task"); - SplitLogTask slt = new SplitLogTask.Unassigned(MANAGER, this.mode); + SplitLogTask slt = new SplitLogTask.Unassigned(MANAGER); zkw.getRecoverableZooKeeper().create(task,slt.toByteArray(), Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT); @@ -454,7 +449,7 @@ public class TestSplitLogWorker { RegionServerServices mockedRS = getRegionServer(RS); for (int i = 0; i < maxTasks; i++) { zkw.getRecoverableZooKeeper().create(ZKSplitLog.getEncodedNodeName(zkw, TATAS + i), - new SplitLogTask.Unassigned(ServerName.valueOf("mgr,1,1"), this.mode).toByteArray(), + new SplitLogTask.Unassigned(ServerName.valueOf("mgr,1,1")).toByteArray(), Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT); } @@ -497,7 +492,7 @@ public class TestSplitLogWorker { for (int i = 0; i < maxTasks; i++) { zkw.getRecoverableZooKeeper().create(ZKSplitLog.getEncodedNodeName(zkw, TATAS + i), - new SplitLogTask.Unassigned(ServerName.valueOf("mgr,1,1"), this.mode).toByteArray(), + new SplitLogTask.Unassigned(ServerName.valueOf("mgr,1,1")).toByteArray(), Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT); } @@ -521,8 +516,6 @@ public class TestSplitLogWorker { /** * Create a mocked region server service instance - * @param server - * @return */ private RegionServerServices getRegionServer(ServerName name) { http://git-wip-us.apache.org/repos/asf/hbase/blob/a6d8023f/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitWalDataLoss.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitWalDataLoss.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitWalDataLoss.java index 99cf91d..4db5734 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitWalDataLoss.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestSplitWalDataLoss.java @@ -79,7 +79,6 @@ public class TestSplitWalDataLoss { @Before public void setUp() throws Exception { testUtil.getConfiguration().setInt("hbase.regionserver.msginterval", 30000); - testUtil.getConfiguration().setBoolean(HConstants.DISTRIBUTED_LOG_REPLAY_KEY, false); testUtil.startMiniCluster(2); Admin admin = testUtil.getAdmin(); admin.createNamespace(namespace); http://git-wip-us.apache.org/repos/asf/hbase/blob/a6d8023f/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/AbstractTestWALReplay.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/AbstractTestWALReplay.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/AbstractTestWALReplay.java index 283b85d..5acbf23 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/AbstractTestWALReplay.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/AbstractTestWALReplay.java @@ -112,8 +112,6 @@ import org.mockito.Mockito; import org.mockito.invocation.InvocationOnMock; import org.mockito.stubbing.Answer; -import org.apache.hadoop.hbase.shaded.protobuf.generated.ZooKeeperProtos.SplitLogTask.RecoveryMode; - /** * Test replay of edits out of a WAL split. */ @@ -127,7 +125,6 @@ public abstract class AbstractTestWALReplay { private Path logDir; private FileSystem fs; private Configuration conf; - private RecoveryMode mode; private WALFactory wals; @Rule @@ -165,9 +162,6 @@ public abstract class AbstractTestWALReplay { if (TEST_UTIL.getDFSCluster().getFileSystem().exists(this.hbaseRootDir)) { TEST_UTIL.getDFSCluster().getFileSystem().delete(this.hbaseRootDir, true); } - this.mode = (conf.getBoolean(HConstants.DISTRIBUTED_LOG_REPLAY_KEY, - HConstants.DEFAULT_DISTRIBUTED_LOG_REPLAY_CONFIG) ? - RecoveryMode.LOG_REPLAY : RecoveryMode.LOG_SPLITTING); this.wals = new WALFactory(conf, null, currentTest.getMethodName()); } @@ -908,7 +902,7 @@ public abstract class AbstractTestWALReplay { assertNotNull(listStatus); assertTrue(listStatus.length > 0); WALSplitter.splitLogFile(hbaseRootDir, listStatus[0], - this.fs, this.conf, null, null, null, null, mode, wals); + this.fs, this.conf, null, null, null, wals); FileStatus[] listStatus1 = this.fs.listStatus( new Path(FSUtils.getTableDir(hbaseRootDir, tableName), new Path(hri.getEncodedName(), "recovered.edits")), new PathFilter() { @@ -1062,10 +1056,8 @@ public abstract class AbstractTestWALReplay { first = fs.getFileStatus(smallFile); second = fs.getFileStatus(largeFile); } - WALSplitter.splitLogFile(hbaseRootDir, first, fs, conf, null, null, null, null, - RecoveryMode.LOG_SPLITTING, wals); - WALSplitter.splitLogFile(hbaseRootDir, second, fs, conf, null, null, null, null, - RecoveryMode.LOG_SPLITTING, wals); + WALSplitter.splitLogFile(hbaseRootDir, first, fs, conf, null, null, null, wals); + WALSplitter.splitLogFile(hbaseRootDir, second, fs, conf, null, null, null, wals); WAL wal = createWAL(this.conf, hbaseRootDir, logName); region = HRegion.openHRegion(conf, this.fs, hbaseRootDir, hri, htd, wal); assertTrue(region.getOpenSeqNum() > mvcc.getWritePoint()); http://git-wip-us.apache.org/repos/asf/hbase/blob/a6d8023f/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestRegionReplicaReplicationEndpointNoMaster.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestRegionReplicaReplicationEndpointNoMaster.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestRegionReplicaReplicationEndpointNoMaster.java index 1089b7a..45682fc 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestRegionReplicaReplicationEndpointNoMaster.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/regionserver/TestRegionReplicaReplicationEndpointNoMaster.java @@ -276,56 +276,4 @@ public class TestRegionReplicaReplicationEndpointNoMaster { closeRegion(HTU, rs0, hriSecondary); connection.close(); } - - @Test (timeout = 240000) - public void testReplayedEditsAreSkipped() throws Exception { - openRegion(HTU, rs0, hriSecondary); - ClusterConnection connection = - (ClusterConnection) ConnectionFactory.createConnection(HTU.getConfiguration()); - RegionReplicaReplicationEndpoint replicator = new RegionReplicaReplicationEndpoint(); - - ReplicationEndpoint.Context context = mock(ReplicationEndpoint.Context.class); - when(context.getConfiguration()).thenReturn(HTU.getConfiguration()); - when(context.getMetrics()).thenReturn(mock(MetricsSource.class)); - - ReplicationPeer mockPeer = mock(ReplicationPeer.class); - when(mockPeer.getNamespaces()).thenReturn(null); - when(mockPeer.getTableCFs()).thenReturn(null); - when(mockPeer.getPeerConfig()).thenReturn(new ReplicationPeerConfig()); - when(context.getReplicationPeer()).thenReturn(mockPeer); - - replicator.init(context); - replicator.startAsync(); - - // test the filter for the RE, not actual replication - WALEntryFilter filter = replicator.getWALEntryfilter(); - - //load some data to primary - HTU.loadNumericRows(table, f, 0, 1000); - - Assert.assertEquals(1000, entries.size()); - for (Entry e: entries) { - Cell _c = e.getEdit().getCells().get(0); - if (Integer.parseInt( - Bytes.toString(_c.getValueArray(), _c.getValueOffset(), _c.getValueLength())) % 2 == 0) { - e.getKey().setOrigLogSeqNum(1); // simulate dist log replay by setting orig seq id - } - } - - long skipped = 0, replayed = 0; - for (Entry e : entries) { - if (filter.filter(e) == null) { - skipped++; - } else { - replayed++; - } - } - - assertEquals(500, skipped); - assertEquals(500, replayed); - - HTU.deleteNumericRows(table, f, 0, 1000); - closeRegion(HTU, rs0, hriSecondary); - connection.close(); - } }