Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id 51567200C40 for ; Thu, 23 Mar 2017 16:43:10 +0100 (CET) Received: by cust-asf.ponee.io (Postfix) id 4FCED160B9A; Thu, 23 Mar 2017 15:43:10 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id 92E4C160B9B for ; Thu, 23 Mar 2017 16:43:07 +0100 (CET) Received: (qmail 50946 invoked by uid 500); 23 Mar 2017 15:43:06 -0000 Mailing-List: contact commits-help@hbase.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@hbase.apache.org Delivered-To: mailing list commits@hbase.apache.org Received: (qmail 50447 invoked by uid 99); 23 Mar 2017 15:43:05 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Thu, 23 Mar 2017 15:43:05 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 347EAE0ACF; Thu, 23 Mar 2017 15:43:05 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: stack@apache.org To: commits@hbase.apache.org Date: Thu, 23 Mar 2017 15:43:07 -0000 Message-Id: <214afd2de5704bd7b6dcde6bda69f80a@git.apache.org> In-Reply-To: References: X-Mailer: ASF-Git Admin Mailer Subject: [03/21] hbase git commit: HBASE-14614 Procedure v2 - Core Assignment Manager (Matteo Bertozzi) archived-at: Thu, 23 Mar 2017 15:43:10 -0000 http://git-wip-us.apache.org/repos/asf/hbase/blob/8faab93a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/MockMasterServices.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/MockMasterServices.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/MockMasterServices.java new file mode 100644 index 0000000..fdee307 --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/MockMasterServices.java @@ -0,0 +1,201 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.master.assignment; + +import java.io.IOException; +import java.util.HashSet; +import java.util.NavigableMap; +import java.util.SortedSet; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.ServerLoad; +import org.apache.hadoop.hbase.ServerName; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.master.LoadBalancer; +import org.apache.hadoop.hbase.master.MasterFileSystem; +import org.apache.hadoop.hbase.master.MasterServices; +import org.apache.hadoop.hbase.master.MasterWalManager; +import org.apache.hadoop.hbase.master.MockNoopMasterServices; +import org.apache.hadoop.hbase.master.RegionState.State; +import org.apache.hadoop.hbase.master.ServerManager; +import org.apache.hadoop.hbase.master.balancer.LoadBalancerFactory; +import org.apache.hadoop.hbase.master.procedure.MasterProcedureConstants; +import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv; +import org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher; +import org.apache.hadoop.hbase.procedure2.Procedure; +import org.apache.hadoop.hbase.procedure2.ProcedureExecutor; +import org.apache.hadoop.hbase.procedure2.store.NoopProcedureStore; +import org.apache.hadoop.hbase.procedure2.store.ProcedureStore; +import org.apache.hadoop.hbase.security.Superusers; + +public class MockMasterServices extends MockNoopMasterServices { + private final MasterFileSystem fileSystemManager; + private final MasterWalManager walManager; + private final AssignmentManager assignmentManager; + + private MasterProcedureEnv procedureEnv; + private ProcedureExecutor procedureExecutor; + private ProcedureStore procedureStore; + + private LoadBalancer balancer; + private ServerManager serverManager; + // Set of regions on a 'server'. Populated externally. Used in below faking 'cluster'. + private final NavigableMap> regionsToRegionServers; + + public MockMasterServices(Configuration conf, + NavigableMap> regionsToRegionServers) + throws IOException { + super(conf); + this.regionsToRegionServers = regionsToRegionServers; + Superusers.initialize(conf); + this.fileSystemManager = new MasterFileSystem(this); + this.walManager = new MasterWalManager(this); + this.assignmentManager = new AssignmentManager(this, new MockRegionStateStore(this)) { + public boolean isTableEnabled(final TableName tableName) { + return true; + } + + public boolean isTableDisabled(final TableName tableName) { + return false; + } + + @Override + protected boolean waitServerReportEvent(ServerName serverName, Procedure proc) { + // Make a report with current state of the server 'serverName' before we call wait.. + SortedSet regions = regionsToRegionServers.get(serverName); + getAssignmentManager().reportOnlineRegions(serverName, 0, + regions == null? new HashSet(): regions); + return super.waitServerReportEvent(serverName, proc); + } + }; + this.balancer = LoadBalancerFactory.getLoadBalancer(conf); + this.serverManager = new ServerManager(this); + } + + public void start(final int numServes, final RSProcedureDispatcher remoteDispatcher) + throws IOException { + startProcedureExecutor(remoteDispatcher); + assignmentManager.start(); + for (int i = 0; i < numServes; ++i) { + serverManager.regionServerReport( + ServerName.valueOf("localhost", 100 + i, 1), ServerLoad.EMPTY_SERVERLOAD); + } + } + + @Override + public void stop(String why) { + stopProcedureExecutor(); + this.assignmentManager.stop(); + } + + private void startProcedureExecutor(final RSProcedureDispatcher remoteDispatcher) + throws IOException { + final Configuration conf = getConfiguration(); + final Path logDir = new Path(fileSystemManager.getRootDir(), + MasterProcedureConstants.MASTER_PROCEDURE_LOGDIR); + + //procedureStore = new WALProcedureStore(conf, fileSystemManager.getFileSystem(), logDir, + // new MasterProcedureEnv.WALStoreLeaseRecovery(this)); + procedureStore = new NoopProcedureStore(); + procedureStore.registerListener(new MasterProcedureEnv.MasterProcedureStoreListener(this)); + + procedureEnv = new MasterProcedureEnv(this, + remoteDispatcher != null ? remoteDispatcher : new RSProcedureDispatcher(this)); + + procedureExecutor = new ProcedureExecutor(conf, procedureEnv, procedureStore, + procedureEnv.getProcedureScheduler()); + + final int numThreads = conf.getInt(MasterProcedureConstants.MASTER_PROCEDURE_THREADS, + Math.max(Runtime.getRuntime().availableProcessors(), + MasterProcedureConstants.DEFAULT_MIN_MASTER_PROCEDURE_THREADS)); + final boolean abortOnCorruption = conf.getBoolean( + MasterProcedureConstants.EXECUTOR_ABORT_ON_CORRUPTION, + MasterProcedureConstants.DEFAULT_EXECUTOR_ABORT_ON_CORRUPTION); + procedureStore.start(numThreads); + procedureExecutor.start(numThreads, abortOnCorruption); + procedureEnv.getRemoteDispatcher().start(); + } + + private void stopProcedureExecutor() { + if (procedureEnv != null) { + procedureEnv.getRemoteDispatcher().stop(); + } + + if (procedureExecutor != null) { + procedureExecutor.stop(); + } + + if (procedureStore != null) { + procedureStore.stop(isAborted()); + } + } + + @Override + public boolean isInitialized() { + return true; + } + + @Override + public MasterFileSystem getMasterFileSystem() { + return fileSystemManager; + } + + @Override + public MasterWalManager getMasterWalManager() { + return walManager; + } + + @Override + public ProcedureExecutor getMasterProcedureExecutor() { + return procedureExecutor; + } + + @Override + public LoadBalancer getLoadBalancer() { + return balancer; + } + + @Override + public ServerManager getServerManager() { + return serverManager; + } + + @Override + public AssignmentManager getAssignmentManager() { + return assignmentManager; + } + + private static class MockRegionStateStore extends RegionStateStore { + public MockRegionStateStore(final MasterServices master) { + super(master); + } + + public void start() throws IOException { + } + + public void stop() { + } + + public void updateRegionLocation(final HRegionInfo regionInfo, final State state, + final ServerName regionLocation, final ServerName lastHost, final long openSeqNum) + throws IOException { + } + } +} http://git-wip-us.apache.org/repos/asf/hbase/blob/8faab93a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestAssignmentManager.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestAssignmentManager.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestAssignmentManager.java new file mode 100644 index 0000000..b75f711 --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestAssignmentManager.java @@ -0,0 +1,567 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.master.assignment; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotEquals; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +import java.io.IOException; +import java.net.SocketTimeoutException; +import java.util.NavigableMap; +import java.util.Random; +import java.util.Set; +import java.util.SortedSet; +import java.util.concurrent.ConcurrentSkipListMap; +import java.util.concurrent.ConcurrentSkipListSet; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Future; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.CategoryBasedTimeout; +import org.apache.hadoop.hbase.DoNotRetryIOException; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.ServerName; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.RetriesExhaustedException; +import org.apache.hadoop.hbase.ipc.ServerNotRunningYetException; +import org.apache.hadoop.hbase.master.MasterServices; +import org.apache.hadoop.hbase.master.RegionState.State; +import org.apache.hadoop.hbase.master.procedure.MasterProcedureConstants; +import org.apache.hadoop.hbase.master.procedure.MasterProcedureScheduler; +import org.apache.hadoop.hbase.master.procedure.ProcedureSyncWait; +import org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher; +import org.apache.hadoop.hbase.procedure2.Procedure; +import org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility; +import org.apache.hadoop.hbase.procedure2.store.wal.WALProcedureStore; +import org.apache.hadoop.hbase.procedure2.util.StringUtils; +import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil; +import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.CloseRegionRequest; +import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.CloseRegionResponse; +import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.ExecuteProceduresRequest; +import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.ExecuteProceduresResponse; +import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.OpenRegionRequest; +import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.OpenRegionRequest.RegionOpenInfo; +import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.OpenRegionResponse; +import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos.OpenRegionResponse.RegionOpeningState; +import org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos.RegionInfo; +import org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.RegionStateTransition; +import org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.RegionStateTransition.TransitionCode; +import org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos.ReportRegionStateTransitionRequest; +import org.apache.hadoop.hbase.testclassification.MasterTests; +import org.apache.hadoop.hbase.testclassification.MediumTests; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.FSUtils; +import org.apache.hadoop.ipc.RemoteException; +import org.apache.log4j.Level; +import org.apache.log4j.Logger; +import org.junit.After; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.rules.TestName; +import org.junit.rules.TestRule; + +@Category({MasterTests.class, MediumTests.class}) +public class TestAssignmentManager { + private static final Log LOG = LogFactory.getLog(TestAssignmentManager.class); + static { + Logger.getLogger(MasterProcedureScheduler.class).setLevel(Level.TRACE); + } + @Rule public TestName name = new TestName(); + @Rule public final TestRule timeout = + CategoryBasedTimeout.builder().withTimeout(this.getClass()). + withLookingForStuckThread(true).build(); + + private static final int PROC_NTHREADS = 64; + private static final int NREGIONS = 1 * 1000; + private static final int NSERVERS = Math.max(1, NREGIONS / 200); + + private HBaseTestingUtility UTIL; + private MockRSProcedureDispatcher rsDispatcher; + private MockMasterServices master; + private AssignmentManager am; + private NavigableMap> regionsToRegionServers = + new ConcurrentSkipListMap>(); + + private void setupConfiguration(Configuration conf) throws Exception { + FSUtils.setRootDir(conf, UTIL.getDataTestDir()); + conf.setBoolean(WALProcedureStore.USE_HSYNC_CONF_KEY, false); + conf.setInt(WALProcedureStore.SYNC_WAIT_MSEC_CONF_KEY, 10); + conf.setInt(MasterProcedureConstants.MASTER_PROCEDURE_THREADS, PROC_NTHREADS); + conf.setInt(RSProcedureDispatcher.RS_RPC_STARTUP_WAIT_TIME_CONF_KEY, 1000); + conf.setInt(AssignmentManager.ASSIGN_MAX_ATTEMPTS, 5); + } + + @Before + public void setUp() throws Exception { + UTIL = new HBaseTestingUtility(); + setupConfiguration(UTIL.getConfiguration()); + master = new MockMasterServices(UTIL.getConfiguration(), this.regionsToRegionServers); + rsDispatcher = new MockRSProcedureDispatcher(master); + master.start(NSERVERS, rsDispatcher); + am = master.getAssignmentManager(); + setUpMeta(); + } + + private void setUpMeta() throws Exception { + rsDispatcher.setMockRsExecutor(new GoodRsExecutor()); + am.assignMeta(HRegionInfo.FIRST_META_REGIONINFO); + am.wakeMetaLoadedEvent(); + am.setFailoverCleanupDone(true); + } + + @After + public void tearDown() throws Exception { + master.stop("tearDown"); + } + + @Test + public void testAssignWithGoodExec() throws Exception { + testAssign(new GoodRsExecutor()); + } + + @Test + public void testAssignWithRandExec() throws Exception { + final TableName tableName = TableName.valueOf("testAssignWithRandExec"); + final HRegionInfo hri = createRegionInfo(tableName, 1); + + rsDispatcher.setMockRsExecutor(new RandRsExecutor()); + + AssignProcedure proc = am.createAssignProcedure(hri, false); + //waitOnFuture(submitProcedure(am.createAssignProcedure(hri, false, false))); + // TODO + } + + @Test + public void testSocketTimeout() throws Exception { + final TableName tableName = TableName.valueOf(this.name.getMethodName()); + final HRegionInfo hri = createRegionInfo(tableName, 1); + + rsDispatcher.setMockRsExecutor(new SocketTimeoutRsExecutor(20, 3)); + waitOnFuture(submitProcedure(am.createAssignProcedure(hri, false))); + + rsDispatcher.setMockRsExecutor(new SocketTimeoutRsExecutor(20, 3)); + waitOnFuture(submitProcedure(am.createUnassignProcedure(hri, null, false))); + } + + @Test + public void testServerNotYetRunning() throws Exception { + testRetriesExhaustedFailure(TableName.valueOf("testServerNotYetRunning"), + new ServerNotYetRunningRsExecutor()); + } + + private void testRetriesExhaustedFailure(final TableName tableName, + final MockRSExecutor executor) throws Exception { + final HRegionInfo hri = createRegionInfo(tableName, 1); + + // Test Assign operation failure + rsDispatcher.setMockRsExecutor(executor); + try { + waitOnFuture(submitProcedure(am.createAssignProcedure(hri, false))); + fail("unexpected assign completion"); + } catch (RetriesExhaustedException e) { + // expected exception + LOG.info("expected exception from assign operation: " + e.getMessage(), e); + } + + // Assign the region (without problems) + rsDispatcher.setMockRsExecutor(new GoodRsExecutor()); + waitOnFuture(submitProcedure(am.createAssignProcedure(hri, false))); + + // Test Unassign operation failure + rsDispatcher.setMockRsExecutor(executor); + waitOnFuture(submitProcedure(am.createUnassignProcedure(hri, null, false))); + } + + + @Test + public void testIOExceptionOnAssignment() throws Exception { + testFailedOpen(TableName.valueOf("testExceptionOnAssignment"), + new FaultyRsExecutor(new IOException("test fault"))); + } + + @Test + public void testDoNotRetryExceptionOnAssignment() throws Exception { + testFailedOpen(TableName.valueOf("testDoNotRetryExceptionOnAssignment"), + new FaultyRsExecutor(new DoNotRetryIOException("test do not retry fault"))); + } + + private void testFailedOpen(final TableName tableName, + final MockRSExecutor executor) throws Exception { + final HRegionInfo hri = createRegionInfo(tableName, 1); + + // Test Assign operation failure + rsDispatcher.setMockRsExecutor(executor); + try { + waitOnFuture(submitProcedure(am.createAssignProcedure(hri, false))); + fail("unexpected assign completion"); + } catch (RetriesExhaustedException e) { + // expected exception + LOG.info("REGION STATE " + am.getRegionStates().getRegionNode(hri)); + LOG.info("expected exception from assign operation: " + e.getMessage(), e); + assertEquals(true, am.getRegionStates().getRegionState(hri).isFailedOpen()); + } + } + + private void testAssign(final MockRSExecutor executor) throws Exception { + testAssign(executor, NREGIONS); + } + + private void testAssign(final MockRSExecutor executor, final int nregions) throws Exception { + rsDispatcher.setMockRsExecutor(executor); + + AssignProcedure[] assignments = new AssignProcedure[nregions]; + + long st = System.currentTimeMillis(); + bulkSubmit(assignments); + + for (int i = 0; i < assignments.length; ++i) { + ProcedureTestingUtility.waitProcedure( + master.getMasterProcedureExecutor(), assignments[i]); + assertTrue(assignments[i].toString(), assignments[i].isSuccess()); + } + long et = System.currentTimeMillis(); + float sec = ((et - st) / 1000.0f); + LOG.info(String.format("[T] Assigning %dprocs in %s (%.2fproc/sec)", + assignments.length, StringUtils.humanTimeDiff(et - st), assignments.length / sec)); + } + + @Test + public void testAssignAnAssignedRegion() throws Exception { + final TableName tableName = TableName.valueOf("testAssignAnAssignedRegion"); + final HRegionInfo hri = createRegionInfo(tableName, 1); + + rsDispatcher.setMockRsExecutor(new GoodRsExecutor()); + + final Future futureA = submitProcedure(am.createAssignProcedure(hri, false)); + + // wait first assign + waitOnFuture(futureA); + am.getRegionStates().isRegionInState(hri, State.OPEN); + // Second should be a noop. We should recognize region is already OPEN internally + // and skip out doing nothing. + // wait second assign + final Future futureB = submitProcedure(am.createAssignProcedure(hri, false)); + waitOnFuture(futureB); + am.getRegionStates().isRegionInState(hri, State.OPEN); + // TODO: What else can we do to ensure just a noop. + } + + @Test + public void testUnassignAnUnassignedRegion() throws Exception { + final TableName tableName = TableName.valueOf("testUnassignAnUnassignedRegion"); + final HRegionInfo hri = createRegionInfo(tableName, 1); + + rsDispatcher.setMockRsExecutor(new GoodRsExecutor()); + + // assign the region first + waitOnFuture(submitProcedure(am.createAssignProcedure(hri, false))); + + final Future futureA = submitProcedure(am.createUnassignProcedure(hri, null, false)); + + // Wait first unassign. + waitOnFuture(futureA); + am.getRegionStates().isRegionInState(hri, State.CLOSED); + // Second should be a noop. We should recognize region is already CLOSED internally + // and skip out doing nothing. + final Future futureB = submitProcedure(am.createUnassignProcedure(hri, null, false)); + waitOnFuture(futureB); + // Ensure we are still CLOSED. + am.getRegionStates().isRegionInState(hri, State.CLOSED); + // TODO: What else can we do to ensure just a noop. + } + + private Future submitProcedure(final Procedure proc) { + return ProcedureSyncWait.submitProcedure(master.getMasterProcedureExecutor(), proc); + } + + private byte[] waitOnFuture(final Future future) throws Exception { + try { + return future.get(); + } catch (ExecutionException e) { + throw (Exception)e.getCause(); + } + } + + // ============================================================================================ + // Helpers + // ============================================================================================ + private void bulkSubmit(final AssignProcedure[] procs) throws Exception { + final Thread[] threads = new Thread[PROC_NTHREADS]; + for (int i = 0; i < threads.length; ++i) { + final int threadId = i; + threads[i] = new Thread() { + @Override + public void run() { + TableName tableName = TableName.valueOf("table-" + threadId); + int n = (procs.length / threads.length); + int start = threadId * n; + int stop = start + n; + for (int j = start; j < stop; ++j) { + procs[j] = createAndSubmitAssign(tableName, j); + } + } + }; + threads[i].start(); + } + for (int i = 0; i < threads.length; ++i) { + threads[i].join(); + } + for (int i = procs.length - 1; i >= 0 && procs[i] == null; --i) { + procs[i] = createAndSubmitAssign(TableName.valueOf("table-sync"), i); + } + } + + private AssignProcedure createAndSubmitAssign(TableName tableName, int regionId) { + HRegionInfo hri = createRegionInfo(tableName, regionId); + AssignProcedure proc = am.createAssignProcedure(hri, false); + master.getMasterProcedureExecutor().submitProcedure(proc); + return proc; + } + + private UnassignProcedure createAndSubmitUnassign(TableName tableName, int regionId) { + HRegionInfo hri = createRegionInfo(tableName, regionId); + UnassignProcedure proc = am.createUnassignProcedure(hri, null, false); + master.getMasterProcedureExecutor().submitProcedure(proc); + return proc; + } + + private HRegionInfo createRegionInfo(final TableName tableName, final long regionId) { + return new HRegionInfo(tableName, + Bytes.toBytes(regionId), Bytes.toBytes(regionId + 1), false, 0); + } + + private void sendTransitionReport(final ServerName serverName, + final RegionInfo regionInfo, final TransitionCode state) throws IOException { + ReportRegionStateTransitionRequest.Builder req = + ReportRegionStateTransitionRequest.newBuilder(); + req.setServer(ProtobufUtil.toServerName(serverName)); + req.addTransition(RegionStateTransition.newBuilder() + .addRegionInfo(regionInfo) + .setTransitionCode(state) + .setOpenSeqNum(1) + .build()); + am.reportRegionStateTransition(req.build()); + } + + private class NoopRsExecutor implements MockRSExecutor { + public ExecuteProceduresResponse sendRequest(ServerName server, + ExecuteProceduresRequest request) throws IOException { + ExecuteProceduresResponse.Builder builder = ExecuteProceduresResponse.newBuilder(); + if (request.getOpenRegionCount() > 0) { + for (OpenRegionRequest req: request.getOpenRegionList()) { + OpenRegionResponse.Builder resp = OpenRegionResponse.newBuilder(); + for (RegionOpenInfo openReq: req.getOpenInfoList()) { + RegionOpeningState state = execOpenRegion(server, openReq); + if (state != null) { + resp.addOpeningState(state); + } + } + builder.addOpenRegion(resp.build()); + } + } + if (request.getCloseRegionCount() > 0) { + for (CloseRegionRequest req: request.getCloseRegionList()) { + CloseRegionResponse resp = execCloseRegion(server, + req.getRegion().getValue().toByteArray()); + if (resp != null) { + builder.addCloseRegion(resp); + } + } + } + return ExecuteProceduresResponse.newBuilder().build(); + } + + protected RegionOpeningState execOpenRegion(ServerName server, RegionOpenInfo regionInfo) + throws IOException { + return null; + } + + protected CloseRegionResponse execCloseRegion(ServerName server, byte[] regionName) + throws IOException { + return null; + } + } + + private class GoodRsExecutor extends NoopRsExecutor { + @Override + protected RegionOpeningState execOpenRegion(ServerName server, RegionOpenInfo openReq) + throws IOException { + sendTransitionReport(server, openReq.getRegion(), TransitionCode.OPENED); + // Concurrency? + // Now update the state of our cluster in regionsToRegionServers. + SortedSet regions = regionsToRegionServers.get(server); + if (regions == null) { + regions = new ConcurrentSkipListSet(Bytes.BYTES_COMPARATOR); + regionsToRegionServers.put(server, regions); + } + HRegionInfo hri = HRegionInfo.convert(openReq.getRegion()); + if (regions.contains(hri.getRegionName())) { + throw new UnsupportedOperationException(hri.getRegionNameAsString()); + } + regions.add(hri.getRegionName()); + return RegionOpeningState.OPENED; + } + + @Override + protected CloseRegionResponse execCloseRegion(ServerName server, byte[] regionName) + throws IOException { + HRegionInfo hri = am.getRegionInfo(regionName); + sendTransitionReport(server, HRegionInfo.convert(hri), TransitionCode.CLOSED); + return CloseRegionResponse.newBuilder().setClosed(true).build(); + } + } + + private static class ServerNotYetRunningRsExecutor implements MockRSExecutor { + public ExecuteProceduresResponse sendRequest(ServerName server, ExecuteProceduresRequest req) + throws IOException { + throw new ServerNotRunningYetException("wait on server startup"); + } + } + + private static class FaultyRsExecutor implements MockRSExecutor { + private final IOException exception; + + public FaultyRsExecutor(final IOException exception) { + this.exception = exception; + } + + public ExecuteProceduresResponse sendRequest(ServerName server, ExecuteProceduresRequest req) + throws IOException { + throw exception; + } + } + + private class SocketTimeoutRsExecutor extends GoodRsExecutor { + private final int maxSocketTimeoutRetries; + private final int maxServerRetries; + + private ServerName lastServer; + private int sockTimeoutRetries; + private int serverRetries; + + public SocketTimeoutRsExecutor(int maxSocketTimeoutRetries, int maxServerRetries) { + this.maxServerRetries = maxServerRetries; + this.maxSocketTimeoutRetries = maxSocketTimeoutRetries; + } + + public ExecuteProceduresResponse sendRequest(ServerName server, ExecuteProceduresRequest req) + throws IOException { + // SocketTimeoutException should be a temporary problem + // unless the server will be declared dead. + if (sockTimeoutRetries++ < maxSocketTimeoutRetries) { + if (sockTimeoutRetries == 1) assertNotEquals(lastServer, server); + lastServer = server; + LOG.debug("Socket timeout for server=" + server + " retries=" + sockTimeoutRetries); + throw new SocketTimeoutException("simulate socket timeout"); + } else if (serverRetries++ < maxServerRetries) { + LOG.info("Mark server=" + server + " as dead. serverRetries=" + serverRetries); + master.getServerManager().moveFromOnlineToDeadServers(server); + sockTimeoutRetries = 0; + throw new SocketTimeoutException("simulate socket timeout"); + } else { + return super.sendRequest(server, req); + } + } + } + + private class RandRsExecutor extends NoopRsExecutor { + private final Random rand = new Random(); + + public ExecuteProceduresResponse sendRequest(ServerName server, ExecuteProceduresRequest req) + throws IOException { + switch (rand.nextInt(5)) { + case 0: throw new ServerNotRunningYetException("wait on server startup"); + case 1: throw new SocketTimeoutException("simulate socket timeout"); + case 2: throw new RemoteException("java.io.IOException", "unexpected exception"); + } + return super.sendRequest(server, req); + } + + @Override + protected RegionOpeningState execOpenRegion(ServerName server, RegionOpenInfo openReq) + throws IOException { + switch (rand.nextInt(6)) { + case 0: + return OpenRegionResponse.RegionOpeningState.OPENED; + case 1: + sendTransitionReport(server, openReq.getRegion(), TransitionCode.OPENED); + return OpenRegionResponse.RegionOpeningState.ALREADY_OPENED; + case 2: + sendTransitionReport(server, openReq.getRegion(), TransitionCode.FAILED_OPEN); + return OpenRegionResponse.RegionOpeningState.FAILED_OPENING; + } + return null; + } + + @Override + protected CloseRegionResponse execCloseRegion(ServerName server, byte[] regionName) + throws IOException { + CloseRegionResponse.Builder resp = CloseRegionResponse.newBuilder(); + boolean closed = rand.nextBoolean(); + if (closed) { + HRegionInfo hri = am.getRegionInfo(regionName); + sendTransitionReport(server, HRegionInfo.convert(hri), TransitionCode.CLOSED); + } + resp.setClosed(closed); + return resp.build(); + } + } + + private interface MockRSExecutor { + ExecuteProceduresResponse sendRequest(ServerName server, ExecuteProceduresRequest req) + throws IOException; + } + + private class MockRSProcedureDispatcher extends RSProcedureDispatcher { + private MockRSExecutor mockRsExec; + + public MockRSProcedureDispatcher(final MasterServices master) { + super(master); + } + + public void setMockRsExecutor(final MockRSExecutor mockRsExec) { + this.mockRsExec = mockRsExec; + } + + @Override + protected void remoteDispatch(ServerName serverName, Set operations) { + submitTask(new MockRemoteCall(serverName, operations)); + } + + private class MockRemoteCall extends ExecuteProceduresRemoteCall { + public MockRemoteCall(final ServerName serverName, + final Set operations) { + super(serverName, operations); + } + + @Override + protected ExecuteProceduresResponse sendRequest(final ServerName serverName, + final ExecuteProceduresRequest request) throws IOException { + return mockRsExec.sendRequest(serverName, request); + } + } + } +} http://git-wip-us.apache.org/repos/asf/hbase/blob/8faab93a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestAssignmentOnRSCrash.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestAssignmentOnRSCrash.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestAssignmentOnRSCrash.java new file mode 100644 index 0000000..e4cec45 --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestAssignmentOnRSCrash.java @@ -0,0 +1,185 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.master.assignment; + +import java.io.IOException; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.ServerName; +import org.apache.hadoop.hbase.client.Get; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.client.Table; +import org.apache.hadoop.hbase.testclassification.MasterTests; +import org.apache.hadoop.hbase.testclassification.LargeTests; +import org.apache.hadoop.hbase.master.procedure.MasterProcedureConstants; +import org.apache.hadoop.hbase.util.Bytes; + +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotEquals; +import static org.junit.Assert.assertTrue; + +@Category({MasterTests.class, LargeTests.class}) +public class TestAssignmentOnRSCrash { + private static final Log LOG = LogFactory.getLog(TestAssignmentOnRSCrash.class); + + private static final TableName TEST_TABLE = TableName.valueOf("testb"); + private static final String FAMILY_STR = "f"; + private static final byte[] FAMILY = Bytes.toBytes(FAMILY_STR); + private static final int NUM_RS = 3; + + private HBaseTestingUtility UTIL; + + private static void setupConf(Configuration conf) { + conf.setInt(MasterProcedureConstants.MASTER_PROCEDURE_THREADS, 1); + conf.set("hbase.balancer.tablesOnMaster", "none"); + } + + @Before + public void setup() throws Exception { + UTIL = new HBaseTestingUtility(); + + setupConf(UTIL.getConfiguration()); + UTIL.startMiniCluster(NUM_RS); + + UTIL.createTable(TEST_TABLE, new byte[][] { FAMILY }, new byte[][] { + Bytes.toBytes("B"), Bytes.toBytes("D"), Bytes.toBytes("F"), Bytes.toBytes("L") + }); + } + + @After + public void tearDown() throws Exception { + UTIL.shutdownMiniCluster(); + } + + @Test(timeout=30000) + public void testKillRsWithUserRegionWithData() throws Exception { + testCrashRsWithUserRegion(true, true); + } + + @Test(timeout=30000) + public void testKillRsWithUserRegionWithoutData() throws Exception { + testCrashRsWithUserRegion(true, false); + } + + @Test(timeout=30000) + public void testStopRsWithUserRegionWithData() throws Exception { + testCrashRsWithUserRegion(false, true); + } + + @Test(timeout=30000) + public void testStopRsWithUserRegionWithoutData() throws Exception { + testCrashRsWithUserRegion(false, false); + } + + private void testCrashRsWithUserRegion(final boolean kill, final boolean withData) + throws Exception { + final int NROWS = 100; + int nkilled = 0; + for (HRegionInfo hri: UTIL.getHBaseAdmin().getTableRegions(TEST_TABLE)) { + ServerName serverName = AssignmentTestingUtil.getServerHoldingRegion(UTIL, hri); + if (AssignmentTestingUtil.isServerHoldingMeta(UTIL, serverName)) continue; + + if (withData) { + testInsert(hri, NROWS); + } + + // wait for regions to enter in transition and then to get out of transition + AssignmentTestingUtil.crashRs(UTIL, serverName, kill); + AssignmentTestingUtil.waitForRegionToBeInTransition(UTIL, hri); + UTIL.waitUntilNoRegionsInTransition(); + + if (withData) { + assertEquals(NROWS, testGet(hri, NROWS)); + } + + // region should be moved to another RS + assertNotEquals(serverName, AssignmentTestingUtil.getServerHoldingRegion(UTIL, hri)); + + if (++nkilled == (NUM_RS - 1)) { + break; + } + } + assertTrue("expected RSs to be killed", nkilled > 0); + } + + @Test(timeout=60000) + public void testKillRsWithMetaRegion() throws Exception { + testCrashRsWithMetaRegion(true); + } + + @Test(timeout=60000) + public void testStopRsWithMetaRegion() throws Exception { + testCrashRsWithMetaRegion(false); + } + + private void testCrashRsWithMetaRegion(final boolean kill) throws Exception { + int nkilled = 0; + for (HRegionInfo hri: AssignmentTestingUtil.getMetaRegions(UTIL)) { + ServerName serverName = AssignmentTestingUtil.crashRsWithRegion(UTIL, hri, kill); + + // wait for region to enter in transition and then to get out of transition + AssignmentTestingUtil.waitForRegionToBeInTransition(UTIL, hri); + UTIL.waitUntilNoRegionsInTransition(); + testGet(hri, 10); + + // region should be moved to another RS + assertNotEquals(serverName, AssignmentTestingUtil.getServerHoldingRegion(UTIL, hri)); + + if (++nkilled == (NUM_RS - 1)) { + break; + } + } + assertTrue("expected RSs to be killed", nkilled > 0); + } + + private void testInsert(final HRegionInfo hri, final int nrows) throws IOException { + final Table table = UTIL.getConnection().getTable(hri.getTable()); + for (int i = 0; i < nrows; ++i) { + final byte[] row = Bytes.add(hri.getStartKey(), Bytes.toBytes(i)); + final Put put = new Put(row); + put.addColumn(FAMILY, null, row); + table.put(put); + } + } + + public int testGet(final HRegionInfo hri, final int nrows) throws IOException { + int nresults = 0; + final Table table = UTIL.getConnection().getTable(hri.getTable()); + for (int i = 0; i < nrows; ++i) { + final byte[] row = Bytes.add(hri.getStartKey(), Bytes.toBytes(i)); + final Result result = table.get(new Get(row)); + if (result != null && !result.isEmpty() && + Bytes.equals(row, result.getValue(FAMILY, null))) { + nresults++; + } + } + return nresults; + } +} http://git-wip-us.apache.org/repos/asf/hbase/blob/8faab93a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestMergeTableRegionsProcedure.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestMergeTableRegionsProcedure.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestMergeTableRegionsProcedure.java new file mode 100644 index 0000000..51f28af --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestMergeTableRegionsProcedure.java @@ -0,0 +1,239 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.master.assignment; + +import java.util.List; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HColumnDescriptor; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.Admin; +import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv; +import org.apache.hadoop.hbase.master.procedure.MasterProcedureConstants; +import org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility; +import org.apache.hadoop.hbase.procedure2.ProcedureExecutor; +import org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility; +import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProcedureProtos.MergeTableRegionsState; +import org.apache.hadoop.hbase.testclassification.MasterTests; +import org.apache.hadoop.hbase.testclassification.MediumTests; +import org.apache.hadoop.hbase.util.Bytes; +import org.junit.After; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +@Category({MasterTests.class, MediumTests.class}) +public class TestMergeTableRegionsProcedure { + private static final Log LOG = LogFactory.getLog(TestMergeTableRegionsProcedure.class); + + protected static final HBaseTestingUtility UTIL = new HBaseTestingUtility(); + private static long nonceGroup = HConstants.NO_NONCE; + private static long nonce = HConstants.NO_NONCE; + + private static final int initialRegionCount = 4; + private final static byte[] FAMILY = Bytes.toBytes("FAMILY"); + final static Configuration conf = UTIL.getConfiguration(); + private static Admin admin; + + private static void setupConf(Configuration conf) { + // Reduce the maximum attempts to speed up the test + conf.setInt("hbase.assignment.maximum.attempts", 3); + conf.setInt("hbase.master.maximum.ping.server.attempts", 3); + conf.setInt("hbase.master.ping.server.retry.sleep.interval", 1); + + conf.setInt(MasterProcedureConstants.MASTER_PROCEDURE_THREADS, 1); + } + + @BeforeClass + public static void setupCluster() throws Exception { + setupConf(conf); + UTIL.startMiniCluster(1); + admin = UTIL.getHBaseAdmin(); + } + + @AfterClass + public static void cleanupTest() throws Exception { + try { + UTIL.shutdownMiniCluster(); + } catch (Exception e) { + LOG.warn("failure shutting down cluster", e); + } + } + + @Before + public void setup() throws Exception { + resetProcExecutorTestingKillFlag(); + nonceGroup = + MasterProcedureTestingUtility.generateNonceGroup(UTIL.getHBaseCluster().getMaster()); + nonce = MasterProcedureTestingUtility.generateNonce(UTIL.getHBaseCluster().getMaster()); + // Turn off balancer so it doesn't cut in and mess up our placements. + UTIL.getHBaseAdmin().setBalancerRunning(false, true); + // Turn off the meta scanner so it don't remove parent on us. + UTIL.getHBaseCluster().getMaster().setCatalogJanitorEnabled(false); + resetProcExecutorTestingKillFlag(); + } + + @After + public void tearDown() throws Exception { + resetProcExecutorTestingKillFlag(); + for (HTableDescriptor htd: UTIL.getHBaseAdmin().listTables()) { + LOG.info("Tear down, remove table=" + htd.getTableName()); + UTIL.deleteTable(htd.getTableName()); + } + } + + private void resetProcExecutorTestingKillFlag() { + final ProcedureExecutor procExec = getMasterProcedureExecutor(); + ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, false); + assertTrue("expected executor to be running", procExec.isRunning()); + } + + /** + * This tests two region merges + */ + @Test(timeout=60000) + public void testMergeTwoRegions() throws Exception { + final TableName tableName = TableName.valueOf("testMergeTwoRegions"); + final ProcedureExecutor procExec = getMasterProcedureExecutor(); + + List tableRegions = createTable(tableName); + + HRegionInfo[] regionsToMerge = new HRegionInfo[2]; + regionsToMerge[0] = tableRegions.get(0); + regionsToMerge[1] = tableRegions.get(1); + + long procId = procExec.submitProcedure(new MergeTableRegionsProcedure( + procExec.getEnvironment(), regionsToMerge, true)); + ProcedureTestingUtility.waitProcedure(procExec, procId); + ProcedureTestingUtility.assertProcNotFailed(procExec, procId); + + assertRegionCount(tableName, initialRegionCount - 1); + } + + /** + * This tests two concurrent region merges + */ + @Test(timeout=60000) + public void testMergeRegionsConcurrently() throws Exception { + final TableName tableName = TableName.valueOf("testMergeRegionsConcurrently"); + final ProcedureExecutor procExec = getMasterProcedureExecutor(); + + List tableRegions = createTable(tableName); + + HRegionInfo[] regionsToMerge1 = new HRegionInfo[2]; + HRegionInfo[] regionsToMerge2 = new HRegionInfo[2]; + regionsToMerge1[0] = tableRegions.get(0); + regionsToMerge1[1] = tableRegions.get(1); + regionsToMerge2[0] = tableRegions.get(2); + regionsToMerge2[1] = tableRegions.get(3); + + long procId1 = procExec.submitProcedure(new MergeTableRegionsProcedure( + procExec.getEnvironment(), regionsToMerge1, true)); + long procId2 = procExec.submitProcedure(new MergeTableRegionsProcedure( + procExec.getEnvironment(), regionsToMerge2, true)); + ProcedureTestingUtility.waitProcedure(procExec, procId1); + ProcedureTestingUtility.waitProcedure(procExec, procId2); + ProcedureTestingUtility.assertProcNotFailed(procExec, procId1); + ProcedureTestingUtility.assertProcNotFailed(procExec, procId2); + assertRegionCount(tableName, initialRegionCount - 2); + } + + @Test(timeout=60000) + public void testRecoveryAndDoubleExecution() throws Exception { + final TableName tableName = TableName.valueOf("testRecoveryAndDoubleExecution"); + final ProcedureExecutor procExec = getMasterProcedureExecutor(); + + List tableRegions = createTable(tableName); + + ProcedureTestingUtility.waitNoProcedureRunning(procExec); + ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, true); + + HRegionInfo[] regionsToMerge = new HRegionInfo[2]; + regionsToMerge[0] = tableRegions.get(0); + regionsToMerge[1] = tableRegions.get(1); + + long procId = procExec.submitProcedure( + new MergeTableRegionsProcedure(procExec.getEnvironment(), regionsToMerge, true)); + + // Restart the executor and execute the step twice + MasterProcedureTestingUtility.testRecoveryAndDoubleExecution(procExec, procId); + ProcedureTestingUtility.assertProcNotFailed(procExec, procId); + + assertRegionCount(tableName, initialRegionCount - 1); + } + + @Test(timeout = 60000) + public void testRollbackAndDoubleExecution() throws Exception { + final TableName tableName = TableName.valueOf("testRollbackAndDoubleExecution"); + final ProcedureExecutor procExec = getMasterProcedureExecutor(); + + List tableRegions = createTable(tableName); + + ProcedureTestingUtility.waitNoProcedureRunning(procExec); + ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, true); + + HRegionInfo[] regionsToMerge = new HRegionInfo[2]; + regionsToMerge[0] = tableRegions.get(0); + regionsToMerge[1] = tableRegions.get(1); + + long procId = procExec.submitProcedure( + new MergeTableRegionsProcedure(procExec.getEnvironment(), regionsToMerge, true)); + + // Failing before MERGE_TABLE_REGIONS_UPDATE_META we should trigger the rollback + // NOTE: the 5 (number before MERGE_TABLE_REGIONS_UPDATE_META step) is + // hardcoded, so you have to look at this test at least once when you add a new step. + int numberOfSteps = 5; + MasterProcedureTestingUtility.testRollbackAndDoubleExecution(procExec, procId, numberOfSteps); + } + + private List createTable(final TableName tableName) + throws Exception { + HTableDescriptor desc = new HTableDescriptor(tableName); + desc.addFamily(new HColumnDescriptor(FAMILY)); + byte[][] splitRows = new byte[initialRegionCount - 1][]; + for (int i = 0; i < splitRows.length; ++i) { + splitRows[i] = Bytes.toBytes(String.format("%d", i)); + } + admin.createTable(desc, splitRows); + return assertRegionCount(tableName, initialRegionCount); + } + + public List assertRegionCount(final TableName tableName, final int nregions) + throws Exception { + UTIL.waitUntilNoRegionsInTransition(); + List tableRegions = admin.getTableRegions(tableName); + assertEquals(nregions, tableRegions.size()); + return tableRegions; + } + + private ProcedureExecutor getMasterProcedureExecutor() { + return UTIL.getHBaseCluster().getMaster().getMasterProcedureExecutor(); + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hbase/blob/8faab93a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestRegionStates.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestRegionStates.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestRegionStates.java new file mode 100644 index 0000000..1d2481d --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestRegionStates.java @@ -0,0 +1,226 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.master.assignment; + +import java.lang.Thread.UncaughtExceptionHandler; +import java.util.concurrent.Callable; +import java.util.concurrent.ExecutorCompletionService; +import java.util.concurrent.Future; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.List; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.master.assignment.RegionStates.RegionStateNode; +import org.apache.hadoop.hbase.procedure2.util.StringUtils; +import org.apache.hadoop.hbase.testclassification.MasterTests; +import org.apache.hadoop.hbase.testclassification.MediumTests; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.Threads; +import org.junit.After; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +@Category({MasterTests.class, MediumTests.class}) +public class TestRegionStates { + private static final Log LOG = LogFactory.getLog(TestRegionStates.class); + + protected static final HBaseTestingUtility UTIL = new HBaseTestingUtility(); + + private static ThreadPoolExecutor threadPool; + private static ExecutorCompletionService executorService; + + @BeforeClass + public static void setUp() throws Exception { + threadPool = Threads.getBoundedCachedThreadPool(32, 60L, TimeUnit.SECONDS, + Threads.newDaemonThreadFactory("ProcedureDispatcher", + new UncaughtExceptionHandler() { + @Override + public void uncaughtException(Thread t, Throwable e) { + LOG.warn("Failed thread " + t.getName(), e); + } + })); + executorService = new ExecutorCompletionService(threadPool); + } + + @AfterClass + public static void tearDown() throws Exception { + threadPool.shutdown(); + } + + @Before + public void testSetup() { + } + + @After + public void testTearDown() throws Exception { + while (true) { + Future f = executorService.poll(); + if (f == null) break; + f.get(); + } + } + + private static void waitExecutorService(final int count) throws Exception { + for (int i = 0; i < count; ++i) { + executorService.take().get(); + } + } + + // ========================================================================== + // Regions related + // ========================================================================== + + @Test + public void testRegionDoubleCreation() throws Exception { + // NOTE: HRegionInfo sort by table first, so we are relying on that + final TableName TABLE_NAME_A = TableName.valueOf("testOrderedByTableA"); + final TableName TABLE_NAME_B = TableName.valueOf("testOrderedByTableB"); + final TableName TABLE_NAME_C = TableName.valueOf("testOrderedByTableC"); + final RegionStates stateMap = new RegionStates(); + final int NRUNS = 1000; + final int NSMALL_RUNS = 3; + + // add some regions for table B + for (int i = 0; i < NRUNS; ++i) { + addRegionNode(stateMap, TABLE_NAME_B, i); + } + // re-add the regions for table B + for (int i = 0; i < NRUNS; ++i) { + addRegionNode(stateMap, TABLE_NAME_B, i); + } + waitExecutorService(NRUNS * 2); + + // add two other tables A and C that will be placed before and after table B (sort order) + for (int i = 0; i < NSMALL_RUNS; ++i) { + addRegionNode(stateMap, TABLE_NAME_A, i); + addRegionNode(stateMap, TABLE_NAME_C, i); + } + + // check for the list of regions of the 3 tables + checkTableRegions(stateMap, TABLE_NAME_A, NSMALL_RUNS); + checkTableRegions(stateMap, TABLE_NAME_B, NRUNS); + checkTableRegions(stateMap, TABLE_NAME_C, NSMALL_RUNS); + } + + private void checkTableRegions(final RegionStates stateMap, + final TableName tableName, final int nregions) { + List hris = stateMap.getRegionsOfTable(tableName); + assertEquals(nregions, hris.size()); + for (int i = 1; i < hris.size(); ++i) { + long a = Bytes.toLong(hris.get(i - 1).getStartKey()); + long b = Bytes.toLong(hris.get(i + 0).getStartKey()); + assertEquals(b, a + 1); + } + } + + private void addRegionNode(final RegionStates stateMap, + final TableName tableName, final long regionId) { + executorService.submit(new Callable() { + @Override + public Object call() { + HRegionInfo hri = new HRegionInfo(tableName, + Bytes.toBytes(regionId), Bytes.toBytes(regionId + 1), false, 0); + return stateMap.getOrCreateRegionNode(hri); + } + }); + } + + private Object createRegionNode(final RegionStates stateMap, + final TableName tableName, final long regionId) { + return stateMap.getOrCreateRegionNode(createRegionInfo(tableName, regionId)); + } + + private HRegionInfo createRegionInfo(final TableName tableName, final long regionId) { + return new HRegionInfo(tableName, + Bytes.toBytes(regionId), Bytes.toBytes(regionId + 1), false, 0); + } + + @Test + public void testPerf() throws Exception { + final TableName TABLE_NAME = TableName.valueOf("testPerf"); + final int NRUNS = 1000000; // 1M + final RegionStates stateMap = new RegionStates(); + + long st = System.currentTimeMillis(); + for (int i = 0; i < NRUNS; ++i) { + final int regionId = i; + executorService.submit(new Callable() { + @Override + public Object call() { + HRegionInfo hri = createRegionInfo(TABLE_NAME, regionId); + return stateMap.getOrCreateRegionNode(hri); + } + }); + } + waitExecutorService(NRUNS); + long et = System.currentTimeMillis(); + LOG.info(String.format("PERF STATEMAP INSERT: %s %s/sec", + StringUtils.humanTimeDiff(et - st), + StringUtils.humanSize(NRUNS / ((et - st) / 1000.0f)))); + + st = System.currentTimeMillis(); + for (int i = 0; i < NRUNS; ++i) { + final int regionId = i; + executorService.submit(new Callable() { + @Override + public Object call() { + HRegionInfo hri = createRegionInfo(TABLE_NAME, regionId); + return stateMap.getRegionState(hri); + } + }); + } + + waitExecutorService(NRUNS); + et = System.currentTimeMillis(); + LOG.info(String.format("PERF STATEMAP GET: %s %s/sec", + StringUtils.humanTimeDiff(et - st), + StringUtils.humanSize(NRUNS / ((et - st) / 1000.0f)))); + } + + @Test + public void testPerfSingleThread() { + final TableName TABLE_NAME = TableName.valueOf("testPerf"); + final int NRUNS = 1 * 1000000; // 1M + + final RegionStates stateMap = new RegionStates(); + long st = System.currentTimeMillis(); + for (int i = 0; i < NRUNS; ++i) { + stateMap.createRegionNode(createRegionInfo(TABLE_NAME, i)); + } + long et = System.currentTimeMillis(); + LOG.info(String.format("PERF SingleThread: %s %s/sec", + StringUtils.humanTimeDiff(et - st), + StringUtils.humanSize(NRUNS / ((et - st) / 1000.0f)))); + } + + // ========================================================================== + // Server related + // ========================================================================== +} http://git-wip-us.apache.org/repos/asf/hbase/blob/8faab93a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestSplitTableRegionProcedure.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestSplitTableRegionProcedure.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestSplitTableRegionProcedure.java new file mode 100644 index 0000000..7af9d67 --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/assignment/TestSplitTableRegionProcedure.java @@ -0,0 +1,427 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.master.assignment; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +import java.io.IOException; +import java.util.List; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.CategoryBasedTimeout; +import org.apache.hadoop.hbase.Cell; +import org.apache.hadoop.hbase.CellUtil; +import org.apache.hadoop.hbase.DoNotRetryIOException; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.Waiter; +import org.apache.hadoop.hbase.client.CompactionState; +import org.apache.hadoop.hbase.client.Delete; +import org.apache.hadoop.hbase.client.Get; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.client.Table; +import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv; +import org.apache.hadoop.hbase.master.procedure.MasterProcedureConstants; +import org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility; +import org.apache.hadoop.hbase.procedure2.ProcedureExecutor; +import org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility; +import org.apache.hadoop.hbase.regionserver.HRegion; +import org.apache.hadoop.hbase.testclassification.MasterTests; +import org.apache.hadoop.hbase.testclassification.MediumTests; +import org.apache.hadoop.hbase.util.Bytes; +import org.junit.After; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Rule; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.rules.TestName; +import org.junit.rules.TestRule; + +@Category({MasterTests.class, MediumTests.class}) +public class TestSplitTableRegionProcedure { + private static final Log LOG = LogFactory.getLog(TestSplitTableRegionProcedure.class); + @Rule public final TestRule timeout = CategoryBasedTimeout.builder().withTimeout(this.getClass()). + withLookingForStuckThread(true).build(); + + protected static final HBaseTestingUtility UTIL = new HBaseTestingUtility(); + + private static String ColumnFamilyName1 = "cf1"; + private static String ColumnFamilyName2 = "cf2"; + + private static final int startRowNum = 11; + private static final int rowCount = 60; + + @Rule + public TestName name = new TestName(); + + private static void setupConf(Configuration conf) { + conf.setInt(MasterProcedureConstants.MASTER_PROCEDURE_THREADS, 1); + conf.setLong(HConstants.MAJOR_COMPACTION_PERIOD, 0); + } + + @BeforeClass + public static void setupCluster() throws Exception { + setupConf(UTIL.getConfiguration()); + UTIL.startMiniCluster(3); + } + + @AfterClass + public static void cleanupTest() throws Exception { + try { + UTIL.shutdownMiniCluster(); + } catch (Exception e) { + LOG.warn("failure shutting down cluster", e); + } + } + + @Before + public void setup() throws Exception { + ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(getMasterProcedureExecutor(), false); + + // Turn off balancer so it doesn't cut in and mess up our placements. + UTIL.getAdmin().setBalancerRunning(false, true); + // Turn off the meta scanner so it don't remove parent on us. + UTIL.getHBaseCluster().getMaster().setCatalogJanitorEnabled(false); + } + + @After + public void tearDown() throws Exception { + ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(getMasterProcedureExecutor(), false); + for (HTableDescriptor htd: UTIL.getAdmin().listTables()) { + UTIL.deleteTable(htd.getTableName()); + } + } + + @Test(timeout=60000) + public void testSplitTableRegion() throws Exception { + final TableName tableName = TableName.valueOf(name.getMethodName()); + final ProcedureExecutor procExec = getMasterProcedureExecutor(); + + HRegionInfo [] regions = MasterProcedureTestingUtility.createTable( + procExec, tableName, null, ColumnFamilyName1, ColumnFamilyName2); + insertData(tableName); + int splitRowNum = startRowNum + rowCount / 2; + byte[] splitKey = Bytes.toBytes("" + splitRowNum); + + assertTrue("not able to find a splittable region", regions != null); + assertTrue("not able to find a splittable region", regions.length == 1); + + // Split region of the table + long procId = procExec.submitProcedure( + new SplitTableRegionProcedure(procExec.getEnvironment(), regions[0], splitKey)); + // Wait the completion + ProcedureTestingUtility.waitProcedure(procExec, procId); + ProcedureTestingUtility.assertProcNotFailed(procExec, procId); + + verify(tableName, splitRowNum); + } + + @Test(timeout=60000) + public void testSplitTableRegionNoStoreFile() throws Exception { + final TableName tableName = TableName.valueOf(name.getMethodName()); + final ProcedureExecutor procExec = getMasterProcedureExecutor(); + + HRegionInfo [] regions = MasterProcedureTestingUtility.createTable( + procExec, tableName, null, ColumnFamilyName1, ColumnFamilyName2); + int splitRowNum = startRowNum + rowCount / 2; + byte[] splitKey = Bytes.toBytes("" + splitRowNum); + + assertTrue("not able to find a splittable region", regions != null); + assertTrue("not able to find a splittable region", regions.length == 1); + + // Split region of the table + long procId = procExec.submitProcedure( + new SplitTableRegionProcedure(procExec.getEnvironment(), regions[0], splitKey)); + // Wait the completion + ProcedureTestingUtility.waitProcedure(procExec, procId); + ProcedureTestingUtility.assertProcNotFailed(procExec, procId); + + assertTrue(UTIL.getMiniHBaseCluster().getRegions(tableName).size() == 2); + assertTrue(UTIL.countRows(tableName) == 0); + } + + @Test(timeout=60000) + public void testSplitTableRegionUnevenDaughter() throws Exception { + final TableName tableName = TableName.valueOf(name.getMethodName()); + final ProcedureExecutor procExec = getMasterProcedureExecutor(); + + HRegionInfo [] regions = MasterProcedureTestingUtility.createTable( + procExec, tableName, null, ColumnFamilyName1, ColumnFamilyName2); + insertData(tableName); + // Split to two daughters with one of them only has 1 row + int splitRowNum = startRowNum + rowCount / 4; + byte[] splitKey = Bytes.toBytes("" + splitRowNum); + + assertTrue("not able to find a splittable region", regions != null); + assertTrue("not able to find a splittable region", regions.length == 1); + + // Split region of the table + long procId = procExec.submitProcedure( + new SplitTableRegionProcedure(procExec.getEnvironment(), regions[0], splitKey)); + // Wait the completion + ProcedureTestingUtility.waitProcedure(procExec, procId); + ProcedureTestingUtility.assertProcNotFailed(procExec, procId); + + verify(tableName, splitRowNum); + } + + @Test(timeout=60000) + public void testSplitTableRegionEmptyDaughter() throws Exception { + final TableName tableName = TableName.valueOf(name.getMethodName()); + final ProcedureExecutor procExec = getMasterProcedureExecutor(); + + HRegionInfo [] regions = MasterProcedureTestingUtility.createTable( + procExec, tableName, null, ColumnFamilyName1, ColumnFamilyName2); + insertData(tableName); + // Split to two daughters with one of them only has 1 row + int splitRowNum = startRowNum + rowCount; + byte[] splitKey = Bytes.toBytes("" + splitRowNum); + + assertTrue("not able to find a splittable region", regions != null); + assertTrue("not able to find a splittable region", regions.length == 1); + + // Split region of the table + long procId = procExec.submitProcedure( + new SplitTableRegionProcedure(procExec.getEnvironment(), regions[0], splitKey)); + // Wait the completion + ProcedureTestingUtility.waitProcedure(procExec, procId); + ProcedureTestingUtility.assertProcNotFailed(procExec, procId); + + // Make sure one daughter has 0 rows. + List daughters = UTIL.getMiniHBaseCluster().getRegions(tableName); + assertTrue(daughters.size() == 2); + assertTrue(UTIL.countRows(tableName) == rowCount); + assertTrue(UTIL.countRows(daughters.get(0)) == 0 || UTIL.countRows(daughters.get(1)) == 0); + } + + @Test(timeout=60000) + public void testSplitTableRegionDeletedRowsDaughter() throws Exception { + final TableName tableName = TableName.valueOf(name.getMethodName()); + final ProcedureExecutor procExec = getMasterProcedureExecutor(); + + HRegionInfo [] regions = MasterProcedureTestingUtility.createTable( + procExec, tableName, null, ColumnFamilyName1, ColumnFamilyName2); + insertData(tableName); + // Split to two daughters with one of them only has 1 row + int splitRowNum = rowCount; + deleteData(tableName, splitRowNum); + byte[] splitKey = Bytes.toBytes("" + splitRowNum); + + assertTrue("not able to find a splittable region", regions != null); + assertTrue("not able to find a splittable region", regions.length == 1); + + // Split region of the table + long procId = procExec.submitProcedure( + new SplitTableRegionProcedure(procExec.getEnvironment(), regions[0], splitKey)); + // Wait the completion + ProcedureTestingUtility.waitProcedure(procExec, procId); + ProcedureTestingUtility.assertProcNotFailed(procExec, procId); + + UTIL.getAdmin().majorCompact(tableName); + // waiting for the major compaction to complete + UTIL.waitFor(6000, new Waiter.Predicate() { + @Override + public boolean evaluate() throws IOException { + return UTIL.getAdmin().getCompactionState(tableName) == CompactionState.NONE; + } + }); + + // Make sure one daughter has 0 rows. + List daughters = UTIL.getMiniHBaseCluster().getRegions(tableName); + assertTrue(daughters.size() == 2); + final int currentRowCount = splitRowNum - startRowNum; + assertTrue(UTIL.countRows(tableName) == currentRowCount); + assertTrue(UTIL.countRows(daughters.get(0)) == 0 || UTIL.countRows(daughters.get(1)) == 0); + } + + @Test(timeout=60000) + public void testInvalidSplitKey() throws Exception { + final TableName tableName = TableName.valueOf(name.getMethodName()); + final ProcedureExecutor procExec = getMasterProcedureExecutor(); + + HRegionInfo [] regions = MasterProcedureTestingUtility.createTable( + procExec, tableName, null, ColumnFamilyName1, ColumnFamilyName2); + insertData(tableName); + + assertTrue("not able to find a splittable region", regions != null); + assertTrue("not able to find a splittable region", regions.length == 1); + + // Split region of the table with null split key + try { + long procId1 = procExec.submitProcedure( + new SplitTableRegionProcedure(procExec.getEnvironment(), regions[0], null)); + ProcedureTestingUtility.waitProcedure(procExec, procId1); + fail("unexpected procedure start with invalid split-key"); + } catch (DoNotRetryIOException e) { + LOG.debug("Expected Split procedure construction failure: " + e.getMessage()); + } + } + + @Test(timeout = 60000) + public void testRollbackAndDoubleExecution() throws Exception { + final TableName tableName = TableName.valueOf(name.getMethodName()); + final ProcedureExecutor procExec = getMasterProcedureExecutor(); + + HRegionInfo [] regions = MasterProcedureTestingUtility.createTable( + procExec, tableName, null, ColumnFamilyName1, ColumnFamilyName2); + insertData(tableName); + int splitRowNum = startRowNum + rowCount / 2; + byte[] splitKey = Bytes.toBytes("" + splitRowNum); + + assertTrue("not able to find a splittable region", regions != null); + assertTrue("not able to find a splittable region", regions.length == 1); + ProcedureTestingUtility.waitNoProcedureRunning(procExec); + ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, true); + + // Split region of the table + long procId = procExec.submitProcedure( + new SplitTableRegionProcedure(procExec.getEnvironment(), regions[0], splitKey)); + + // Failing before SPLIT_TABLE_REGION_UPDATE_META we should trigger the + // rollback + // NOTE: the 3 (number before SPLIT_TABLE_REGION_UPDATE_META step) is + // hardcoded, so you have to look at this test at least once when you add a new step. + int numberOfSteps = 3; + MasterProcedureTestingUtility.testRollbackAndDoubleExecution(procExec, procId, numberOfSteps); + // check that we have only 1 region + assertEquals(1, UTIL.getHBaseAdmin().getTableRegions(tableName).size()); + List daughters = UTIL.getMiniHBaseCluster().getRegions(tableName); + assertEquals(1, daughters.size()); + verifyData(daughters.get(0), startRowNum, rowCount, + Bytes.toBytes(ColumnFamilyName1), Bytes.toBytes(ColumnFamilyName2)); + } + + @Test(timeout=60000) + public void testRecoveryAndDoubleExecution() throws Exception { + final TableName tableName = TableName.valueOf(name.getMethodName()); + final ProcedureExecutor procExec = getMasterProcedureExecutor(); + + HRegionInfo [] regions = MasterProcedureTestingUtility.createTable( + procExec, tableName, null, ColumnFamilyName1, ColumnFamilyName2); + insertData(tableName); + int splitRowNum = startRowNum + rowCount / 2; + byte[] splitKey = Bytes.toBytes("" + splitRowNum); + + assertTrue("not able to find a splittable region", regions != null); + assertTrue("not able to find a splittable region", regions.length == 1); + ProcedureTestingUtility.waitNoProcedureRunning(procExec); + ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, true); + + // Split region of the table + long procId = procExec.submitProcedure( + new SplitTableRegionProcedure(procExec.getEnvironment(), regions[0], splitKey)); + + // Restart the executor and execute the step twice + MasterProcedureTestingUtility.testRecoveryAndDoubleExecution(procExec, procId); + ProcedureTestingUtility.assertProcNotFailed(procExec, procId); + + verify(tableName, splitRowNum); + } + + private void insertData(final TableName tableName) throws IOException, InterruptedException { + Table t = UTIL.getConnection().getTable(tableName); + Put p; + for (int i= 0; i < rowCount / 2; i++) { + p = new Put(Bytes.toBytes("" + (startRowNum + i))); + p.addColumn(Bytes.toBytes(ColumnFamilyName1), Bytes.toBytes("q1"), Bytes.toBytes(i)); + p.addColumn(Bytes.toBytes(ColumnFamilyName2), Bytes.toBytes("q2"), Bytes.toBytes(i)); + t.put(p); + p = new Put(Bytes.toBytes("" + (startRowNum + rowCount - i - 1))); + p.addColumn(Bytes.toBytes(ColumnFamilyName1), Bytes.toBytes("q1"), Bytes.toBytes(i)); + p.addColumn(Bytes.toBytes(ColumnFamilyName2), Bytes.toBytes("q2"), Bytes.toBytes(i)); + t.put(p); + if (i % 5 == 0) { + UTIL.getAdmin().flush(tableName); + } + } + } + + private void deleteData( + final TableName tableName, + final int startDeleteRowNum) throws IOException, InterruptedException { + Table t = UTIL.getConnection().getTable(tableName); + final int numRows = rowCount + startRowNum - startDeleteRowNum; + Delete d; + for (int i= startDeleteRowNum; i <= numRows + startDeleteRowNum; i++) { + d = new Delete(Bytes.toBytes("" + i)); + t.delete(d); + if (i % 5 == 0) { + UTIL.getAdmin().flush(tableName); + } + } + } + + private void verify(final TableName tableName, final int splitRowNum) throws IOException { + List daughters = UTIL.getMiniHBaseCluster().getRegions(tableName); + assertTrue(daughters.size() == 2); + LOG.info("Row Count = " + UTIL.countRows(tableName)); + assertTrue(UTIL.countRows(tableName) == rowCount); + int startRow; + int numRows; + for (int i = 0; i < daughters.size(); i++) { + if (Bytes.compareTo( + daughters.get(i).getRegionInfo().getStartKey(), HConstants.EMPTY_BYTE_ARRAY) == 0) { + startRow = startRowNum; // first region + numRows = splitRowNum - startRowNum; + } else { + startRow = splitRowNum; + numRows = rowCount + startRowNum - splitRowNum; + } + verifyData( + daughters.get(i), + startRow, + numRows, + ColumnFamilyName1.getBytes(), + ColumnFamilyName2.getBytes()); + } + } + + private void verifyData( + final HRegion newReg, + final int startRow, + final int numRows, + final byte[]... families) + throws IOException { + for (int i = startRow; i < startRow + numRows; i++) { + byte[] row = Bytes.toBytes("" + i); + Get get = new Get(row); + Result result = newReg.get(get); + Cell[] raw = result.rawCells(); + assertEquals(families.length, result.size()); + for (int j = 0; j < families.length; j++) { + assertTrue(CellUtil.matchingRow(raw[j], row)); + assertTrue(CellUtil.matchingFamily(raw[j], families[j])); + } + } + } + + private ProcedureExecutor getMasterProcedureExecutor() { + return UTIL.getHBaseCluster().getMaster().getMasterProcedureExecutor(); + } +} http://git-wip-us.apache.org/repos/asf/hbase/blob/8faab93a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureTestingUtility.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureTestingUtility.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureTestingUtility.java index ce8b0c6..99dcf1d 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureTestingUtility.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureTestingUtility.java @@ -25,6 +25,7 @@ import static org.junit.Assert.assertTrue; import java.io.IOException; import java.util.List; import java.util.TreeSet; +import java.util.concurrent.Callable; import java.util.concurrent.atomic.AtomicInteger; import org.apache.commons.logging.Log; @@ -48,7 +49,11 @@ import org.apache.hadoop.hbase.client.Put; import org.apache.hadoop.hbase.client.Result; import org.apache.hadoop.hbase.client.TableState; import org.apache.hadoop.hbase.master.HMaster; +import org.apache.hadoop.hbase.master.MasterMetaBootstrap; +import org.apache.hadoop.hbase.master.RegionState; import org.apache.hadoop.hbase.master.TableStateManager; +import org.apache.hadoop.hbase.master.assignment.AssignmentManager; +import org.apache.hadoop.hbase.monitoring.TaskMonitor; import org.apache.hadoop.hbase.procedure2.ProcedureExecutor; import org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility; import org.apache.hadoop.hbase.util.Bytes; @@ -59,7 +64,45 @@ import org.apache.hadoop.hbase.util.ModifyRegionUtils; public class MasterProcedureTestingUtility { private static final Log LOG = LogFactory.getLog(MasterProcedureTestingUtility.class); - private MasterProcedureTestingUtility() { + private MasterProcedureTestingUtility() { } + + public static void restartMasterProcedureExecutor(ProcedureExecutor procExec) + throws Exception { + final MasterProcedureEnv env = procExec.getEnvironment(); + final HMaster master = (HMaster)env.getMasterServices(); + ProcedureTestingUtility.restart(procExec, true, true, + // stop services + new Callable() { + @Override + public Void call() throws Exception { + final AssignmentManager am = env.getAssignmentManager(); + // try to simulate a master restart by removing the ServerManager states about seqIDs + for (RegionState regionState: am.getRegionStates().getRegionStates()) { + env.getMasterServices().getServerManager().removeRegion(regionState.getRegion()); + } + am.stop(); + master.setInitialized(false); + return null; + } + }, + // restart services + new Callable() { + @Override + public Void call() throws Exception { + final AssignmentManager am = env.getAssignmentManager(); + am.start(); + if (true) { + MasterMetaBootstrap metaBootstrap = new MasterMetaBootstrap(master, + TaskMonitor.get().createStatus("meta")); + metaBootstrap.splitMetaLogsBeforeAssignment(); + metaBootstrap.assignMeta(); + metaBootstrap.processDeadServers(); + } + am.joinCluster(); + master.setInitialized(true); + return null; + } + }); } // ========================================================================== @@ -295,6 +338,9 @@ public class MasterProcedureTestingUtility { return put; } + // ========================================================================== + // Procedure Helpers + // ========================================================================== public static long generateNonceGroup(final HMaster master) { return master.getClusterConnection().getNonceGenerator().getNonceGroup(); } @@ -318,13 +364,6 @@ public class MasterProcedureTestingUtility { * finish. * @see #testRecoveryAndDoubleExecution(ProcedureExecutor, long) */ - public static void testRecoveryAndDoubleExecution( - final ProcedureExecutor procExec, final long procId, - final int numSteps) throws Exception { - testRecoveryAndDoubleExecution(procExec, procId, numSteps, true); - ProcedureTestingUtility.assertProcNotFailed(procExec, procId); - } - private static void testRecoveryAndDoubleExecution( final ProcedureExecutor procExec, final long procId, final int numSteps, final boolean expectExecRunning) throws Exception { @@ -336,9 +375,9 @@ public class MasterProcedureTestingUtility { // restart executor/store // execute step N - save on store for (int i = 0; i < numSteps; ++i) { - LOG.info("Restart " + i + " exec state: " + procExec.getProcedure(procId)); + LOG.info("Restart " + i + " exec state=" + procExec.getProcedure(procId)); ProcedureTestingUtility.assertProcNotYetCompleted(procExec, procId); - ProcedureTestingUtility.restart(procExec); + restartMasterProcedureExecutor(procExec); ProcedureTestingUtility.waitProcedure(procExec, procId); } @@ -366,8 +405,8 @@ public class MasterProcedureTestingUtility { ProcedureTestingUtility.waitProcedure(procExec, procId); assertEquals(false, procExec.isRunning()); for (int i = 0; !procExec.isFinished(procId); ++i) { - LOG.info("Restart " + i + " exec state: " + procExec.getProcedure(procId)); - ProcedureTestingUtility.restart(procExec); + LOG.info("Restart " + i + " exec state=" + procExec.getProcedure(procId)); + restartMasterProcedureExecutor(procExec); ProcedureTestingUtility.waitProcedure(procExec, procId); } assertEquals(true, procExec.isRunning()); @@ -399,7 +438,7 @@ public class MasterProcedureTestingUtility { for (int i = 0; !procExec.isFinished(procId); ++i) { LOG.info("Restart " + i + " rollback state: " + procExec.getProcedure(procId)); ProcedureTestingUtility.assertProcNotYetCompleted(procExec, procId); - ProcedureTestingUtility.restart(procExec); + restartMasterProcedureExecutor(procExec); ProcedureTestingUtility.waitProcedure(procExec, procId); } } finally { @@ -444,7 +483,7 @@ public class MasterProcedureTestingUtility { try { ProcedureTestingUtility.assertProcNotYetCompleted(procExec, procId); LOG.info("Restart and rollback procId=" + procId); - ProcedureTestingUtility.restart(procExec); + restartMasterProcedureExecutor(procExec); ProcedureTestingUtility.waitProcedure(procExec, procId); } finally { assertTrue(procExec.unregisterListener(abortListener));