Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id 900EF200D4A for ; Tue, 28 Nov 2017 16:17:53 +0100 (CET) Received: by cust-asf.ponee.io (Postfix) id 8EB3D160C18; Tue, 28 Nov 2017 15:17:53 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id 1828E160C19 for ; Tue, 28 Nov 2017 16:17:50 +0100 (CET) Received: (qmail 88691 invoked by uid 500); 28 Nov 2017 15:17:47 -0000 Mailing-List: contact commits-help@hbase.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@hbase.apache.org Delivered-To: mailing list commits@hbase.apache.org Received: (qmail 87663 invoked by uid 99); 28 Nov 2017 15:17:47 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 28 Nov 2017 15:17:47 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 5A981F600B; Tue, 28 Nov 2017 15:17:46 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: git-site-role@apache.org To: commits@hbase.apache.org Date: Tue, 28 Nov 2017 15:18:07 -0000 Message-Id: <006aebfef2a2436181a63731de5e0ffb@git.apache.org> In-Reply-To: <51959ba9a227453daad1ae44fbacf0fb@git.apache.org> References: <51959ba9a227453daad1ae44fbacf0fb@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [23/51] [partial] hbase-site git commit: Published site at . archived-at: Tue, 28 Nov 2017 15:17:53 -0000 http://git-wip-us.apache.org/repos/asf/hbase-site/blob/b9722a17/devapidocs/src-html/org/apache/hadoop/hbase/master/assignment/AssignmentManager.RegionInTransitionStat.html ---------------------------------------------------------------------- diff --git a/devapidocs/src-html/org/apache/hadoop/hbase/master/assignment/AssignmentManager.RegionInTransitionStat.html b/devapidocs/src-html/org/apache/hadoop/hbase/master/assignment/AssignmentManager.RegionInTransitionStat.html index 40cd159..2da0903 100644 --- a/devapidocs/src-html/org/apache/hadoop/hbase/master/assignment/AssignmentManager.RegionInTransitionStat.html +++ b/devapidocs/src-html/org/apache/hadoop/hbase/master/assignment/AssignmentManager.RegionInTransitionStat.html @@ -260,7 +260,7 @@ 252 253 // Update meta events (for testing) 254 if (hasProcExecutor) { -255 getProcedureScheduler().suspendEvent(metaLoadEvent); +255 metaLoadEvent.suspend(); 256 setFailoverCleanupDone(false); 257 for (RegionInfo hri: getMetaRegionSet()) { 258 setMetaInitialized(hri, false); @@ -421,1455 +421,1454 @@ 413 } 414 415 public boolean waitMetaInitialized(final Procedure proc, final RegionInfo regionInfo) { -416 return getProcedureScheduler().waitEvent( -417 getMetaInitializedEvent(getMetaForRegion(regionInfo)), proc); -418 } -419 -420 private void setMetaInitialized(final RegionInfo metaRegionInfo, final boolean isInitialized) { -421 assert isMetaRegion(metaRegionInfo) : "unexpected non-meta region " + metaRegionInfo; -422 final ProcedureEvent metaInitEvent = getMetaInitializedEvent(metaRegionInfo); -423 if (isInitialized) { -424 getProcedureScheduler().wakeEvent(metaInitEvent); -425 } else { -426 getProcedureScheduler().suspendEvent(metaInitEvent); -427 } -428 } -429 -430 private ProcedureEvent getMetaInitializedEvent(final RegionInfo metaRegionInfo) { -431 assert isMetaRegion(metaRegionInfo) : "unexpected non-meta region " + metaRegionInfo; -432 // TODO: handle multiple meta. -433 return metaInitializedEvent; -434 } -435 -436 public boolean waitMetaLoaded(final Procedure proc) { -437 return getProcedureScheduler().waitEvent(metaLoadEvent, proc); -438 } -439 -440 protected void wakeMetaLoadedEvent() { -441 getProcedureScheduler().wakeEvent(metaLoadEvent); -442 assert isMetaLoaded() : "expected meta to be loaded"; -443 } -444 -445 public boolean isMetaLoaded() { -446 return metaLoadEvent.isReady(); -447 } -448 -449 // ============================================================================================ -450 // TODO: Sync helpers -451 // ============================================================================================ -452 public void assignMeta(final RegionInfo metaRegionInfo) throws IOException { -453 assignMeta(metaRegionInfo, null); -454 } -455 -456 public void assignMeta(final RegionInfo metaRegionInfo, final ServerName serverName) -457 throws IOException { -458 assert isMetaRegion(metaRegionInfo) : "unexpected non-meta region " + metaRegionInfo; -459 AssignProcedure proc; -460 if (serverName != null) { -461 LOG.debug("Try assigning Meta " + metaRegionInfo + " to " + serverName); -462 proc = createAssignProcedure(metaRegionInfo, serverName); -463 } else { -464 LOG.debug("Assigning " + metaRegionInfo.getRegionNameAsString()); -465 proc = createAssignProcedure(metaRegionInfo, false); -466 } -467 ProcedureSyncWait.submitAndWaitProcedure(master.getMasterProcedureExecutor(), proc); -468 } -469 -470 /** -471 * Start a new thread to check if there are region servers whose versions are higher than others. -472 * If so, move all system table regions to RS with the highest version to keep compatibility. -473 * The reason is, RS in new version may not be able to access RS in old version when there are -474 * some incompatible changes. -475 */ -476 public void checkIfShouldMoveSystemRegionAsync() { -477 new Thread(() -> { -478 try { -479 synchronized (checkIfShouldMoveSystemRegionLock) { -480 List<RegionPlan> plans = new ArrayList<>(); -481 for (ServerName server : getExcludedServersForSystemTable()) { -482 if (master.getServerManager().isServerDead(server)) { -483 // TODO: See HBASE-18494 and HBASE-18495. Though getExcludedServersForSystemTable() -484 // considers only online servers, the server could be queued for dead server -485 // processing. As region assignments for crashed server is handled by -486 // ServerCrashProcedure, do NOT handle them here. The goal is to handle this through -487 // regular flow of LoadBalancer as a favored node and not to have this special -488 // handling. -489 continue; -490 } -491 List<RegionInfo> regionsShouldMove = getCarryingSystemTables(server); -492 if (!regionsShouldMove.isEmpty()) { -493 for (RegionInfo regionInfo : regionsShouldMove) { -494 // null value for dest forces destination server to be selected by balancer -495 RegionPlan plan = new RegionPlan(regionInfo, server, null); -496 if (regionInfo.isMetaRegion()) { -497 // Must move meta region first. -498 moveAsync(plan); -499 } else { -500 plans.add(plan); -501 } -502 } -503 } -504 for (RegionPlan plan : plans) { -505 moveAsync(plan); -506 } -507 } -508 } -509 } catch (Throwable t) { -510 LOG.error(t); -511 } -512 }).start(); -513 } -514 -515 private List<RegionInfo> getCarryingSystemTables(ServerName serverName) { -516 Set<RegionStateNode> regions = this.getRegionStates().getServerNode(serverName).getRegions(); -517 if (regions == null) { -518 return new ArrayList<>(); -519 } -520 return regions.stream() -521 .map(RegionStateNode::getRegionInfo) -522 .filter(r -> r.getTable().isSystemTable()) -523 .collect(Collectors.toList()); -524 } -525 -526 public void assign(final RegionInfo regionInfo) throws IOException { -527 assign(regionInfo, true); -528 } -529 -530 public void assign(final RegionInfo regionInfo, final boolean forceNewPlan) throws IOException { -531 AssignProcedure proc = createAssignProcedure(regionInfo, forceNewPlan); -532 ProcedureSyncWait.submitAndWaitProcedure(master.getMasterProcedureExecutor(), proc); -533 } -534 -535 public void unassign(final RegionInfo regionInfo) throws IOException { -536 unassign(regionInfo, false); -537 } -538 -539 public void unassign(final RegionInfo regionInfo, final boolean forceNewPlan) -540 throws IOException { -541 // TODO: rename this reassign -542 RegionStateNode node = this.regionStates.getRegionNode(regionInfo); -543 ServerName destinationServer = node.getRegionLocation(); -544 if (destinationServer == null) { -545 throw new UnexpectedStateException("DestinationServer is null; Assigned? " + node.toString()); -546 } -547 assert destinationServer != null; node.toString(); -548 UnassignProcedure proc = createUnassignProcedure(regionInfo, destinationServer, forceNewPlan); -549 ProcedureSyncWait.submitAndWaitProcedure(master.getMasterProcedureExecutor(), proc); -550 } -551 -552 public void move(final RegionInfo regionInfo) throws IOException { -553 RegionStateNode node = this.regionStates.getRegionNode(regionInfo); -554 ServerName sourceServer = node.getRegionLocation(); -555 RegionPlan plan = new RegionPlan(regionInfo, sourceServer, null); -556 MoveRegionProcedure proc = createMoveRegionProcedure(plan); -557 ProcedureSyncWait.submitAndWaitProcedure(master.getMasterProcedureExecutor(), proc); -558 } -559 -560 public Future<byte[]> moveAsync(final RegionPlan regionPlan) { -561 MoveRegionProcedure proc = createMoveRegionProcedure(regionPlan); -562 return ProcedureSyncWait.submitProcedure(master.getMasterProcedureExecutor(), proc); -563 } -564 -565 @VisibleForTesting -566 public boolean waitForAssignment(final RegionInfo regionInfo) throws IOException { -567 return waitForAssignment(regionInfo, Long.MAX_VALUE); -568 } -569 -570 @VisibleForTesting -571 // TODO: Remove this? -572 public boolean waitForAssignment(final RegionInfo regionInfo, final long timeout) -573 throws IOException { -574 RegionStateNode node = null; -575 // This method can be called before the regionInfo has made it into the regionStateMap -576 // so wait around here a while. -577 long startTime = System.currentTimeMillis(); -578 // Something badly wrong if takes ten seconds to register a region. -579 long endTime = startTime + 10000; -580 while ((node = regionStates.getRegionNode(regionInfo)) == null && isRunning() && -581 System.currentTimeMillis() < endTime) { -582 // Presume it not yet added but will be added soon. Let it spew a lot so we can tell if -583 // we are waiting here alot. -584 LOG.debug("Waiting on " + regionInfo + " to be added to regionStateMap"); -585 Threads.sleep(10); -586 } -587 if (node == null) { -588 if (!isRunning()) return false; -589 throw new RegionException(regionInfo.getRegionNameAsString() + " never registered with Assigment."); -590 } -591 -592 RegionTransitionProcedure proc = node.getProcedure(); -593 if (proc == null) { -594 throw new NoSuchProcedureException(node.toString()); -595 } -596 -597 ProcedureSyncWait.waitForProcedureToCompleteIOE( -598 master.getMasterProcedureExecutor(), proc, timeout); -599 return true; -600 } -601 -602 // ============================================================================================ -603 // RegionTransition procedures helpers -604 // ============================================================================================ -605 -606 public AssignProcedure[] createAssignProcedures(final Collection<RegionInfo> regionInfo) { -607 return createAssignProcedures(regionInfo, false); -608 } -609 -610 public AssignProcedure[] createAssignProcedures(final Collection<RegionInfo> regionInfo, -611 final boolean forceNewPlan) { -612 if (regionInfo.isEmpty()) return null; -613 final AssignProcedure[] procs = new AssignProcedure[regionInfo.size()]; -614 int index = 0; -615 for (RegionInfo hri: regionInfo) { -616 procs[index++] = createAssignProcedure(hri, forceNewPlan); -617 } -618 return procs; -619 } -620 -621 // Needed for the following method so it can type the created Array we return -622 private static final UnassignProcedure [] UNASSIGNED_PROCEDURE_FOR_TYPE_INFO = -623 new UnassignProcedure[0]; -624 -625 UnassignProcedure[] createUnassignProcedures(final Collection<RegionStateNode> nodes) { -626 if (nodes.isEmpty()) return null; -627 final List<UnassignProcedure> procs = new ArrayList<UnassignProcedure>(nodes.size()); -628 for (RegionStateNode node: nodes) { -629 if (!this.regionStates.include(node, false)) continue; -630 // Look for regions that are offline/closed; i.e. already unassigned. -631 if (this.regionStates.isRegionOffline(node.getRegionInfo())) continue; -632 assert node.getRegionLocation() != null: node.toString(); -633 procs.add(createUnassignProcedure(node.getRegionInfo(), node.getRegionLocation(), false)); -634 } -635 return procs.toArray(UNASSIGNED_PROCEDURE_FOR_TYPE_INFO); -636 } -637 -638 public MoveRegionProcedure[] createReopenProcedures(final Collection<RegionInfo> regionInfo) { -639 final MoveRegionProcedure[] procs = new MoveRegionProcedure[regionInfo.size()]; -640 int index = 0; -641 for (RegionInfo hri: regionInfo) { -642 final ServerName serverName = regionStates.getRegionServerOfRegion(hri); -643 final RegionPlan plan = new RegionPlan(hri, serverName, serverName); -644 procs[index++] = createMoveRegionProcedure(plan); -645 } -646 return procs; -647 } -648 -649 /** -650 * Called by things like EnableTableProcedure to get a list of AssignProcedure -651 * to assign the regions of the table. -652 */ -653 public AssignProcedure[] createAssignProcedures(final TableName tableName) { -654 return createAssignProcedures(regionStates.getRegionsOfTable(tableName)); -655 } -656 -657 /** -658 * Called by things like DisableTableProcedure to get a list of UnassignProcedure -659 * to unassign the regions of the table. -660 */ -661 public UnassignProcedure[] createUnassignProcedures(final TableName tableName) { -662 return createUnassignProcedures(regionStates.getTableRegionStateNodes(tableName)); -663 } -664 -665 /** -666 * Called by things like ModifyColumnFamilyProcedure to get a list of MoveRegionProcedure -667 * to reopen the regions of the table. -668 */ -669 public MoveRegionProcedure[] createReopenProcedures(final TableName tableName) { -670 return createReopenProcedures(regionStates.getRegionsOfTable(tableName)); -671 } -672 -673 public AssignProcedure createAssignProcedure(final RegionInfo regionInfo, -674 final boolean forceNewPlan) { -675 AssignProcedure proc = new AssignProcedure(regionInfo, forceNewPlan); -676 proc.setOwner(getProcedureEnvironment().getRequestUser().getShortName()); -677 return proc; -678 } -679 -680 public AssignProcedure createAssignProcedure(final RegionInfo regionInfo, -681 final ServerName targetServer) { -682 AssignProcedure proc = new AssignProcedure(regionInfo, targetServer); -683 proc.setOwner(getProcedureEnvironment().getRequestUser().getShortName()); -684 return proc; -685 } -686 -687 public UnassignProcedure createUnassignProcedure(final RegionInfo regionInfo, -688 final ServerName destinationServer, final boolean force) { -689 // If destinationServer is null, figure it. -690 ServerName sn = destinationServer != null? destinationServer: -691 getRegionStates().getRegionState(regionInfo).getServerName(); -692 assert sn != null; -693 UnassignProcedure proc = new UnassignProcedure(regionInfo, sn, force); -694 proc.setOwner(getProcedureEnvironment().getRequestUser().getShortName()); -695 return proc; -696 } -697 -698 public MoveRegionProcedure createMoveRegionProcedure(final RegionPlan plan) { -699 if (plan.getRegionInfo().getTable().isSystemTable()) { -700 List<ServerName> exclude = getExcludedServersForSystemTable(); -701 if (plan.getDestination() != null && exclude.contains(plan.getDestination())) { -702 try { -703 LOG.info("Can not move " + plan.getRegionInfo() + " to " + plan.getDestination() -704 + " because the server is not with highest version"); -705 plan.setDestination(getBalancer().randomAssignment(plan.getRegionInfo(), -706 this.master.getServerManager().createDestinationServersList(exclude))); -707 } catch (HBaseIOException e) { -708 LOG.warn(e); -709 } -710 } -711 } -712 return new MoveRegionProcedure(getProcedureEnvironment(), plan); -713 } +416 return getMetaInitializedEvent(getMetaForRegion(regionInfo)).suspendIfNotReady(proc); +417 } +418 +419 private void setMetaInitialized(final RegionInfo metaRegionInfo, final boolean isInitialized) { +420 assert isMetaRegion(metaRegionInfo) : "unexpected non-meta region " + metaRegionInfo; +421 final ProcedureEvent metaInitEvent = getMetaInitializedEvent(metaRegionInfo); +422 if (isInitialized) { +423 metaInitEvent.wake(getProcedureScheduler()); +424 } else { +425 metaInitEvent.suspend(); +426 } +427 } +428 +429 private ProcedureEvent getMetaInitializedEvent(final RegionInfo metaRegionInfo) { +430 assert isMetaRegion(metaRegionInfo) : "unexpected non-meta region " + metaRegionInfo; +431 // TODO: handle multiple meta. +432 return metaInitializedEvent; +433 } +434 +435 public boolean waitMetaLoaded(final Procedure proc) { +436 return metaLoadEvent.suspendIfNotReady(proc); +437 } +438 +439 protected void wakeMetaLoadedEvent() { +440 metaLoadEvent.wake(getProcedureScheduler()); +441 assert isMetaLoaded() : "expected meta to be loaded"; +442 } +443 +444 public boolean isMetaLoaded() { +445 return metaLoadEvent.isReady(); +446 } +447 +448 // ============================================================================================ +449 // TODO: Sync helpers +450 // ============================================================================================ +451 public void assignMeta(final RegionInfo metaRegionInfo) throws IOException { +452 assignMeta(metaRegionInfo, null); +453 } +454 +455 public void assignMeta(final RegionInfo metaRegionInfo, final ServerName serverName) +456 throws IOException { +457 assert isMetaRegion(metaRegionInfo) : "unexpected non-meta region " + metaRegionInfo; +458 AssignProcedure proc; +459 if (serverName != null) { +460 LOG.debug("Try assigning Meta " + metaRegionInfo + " to " + serverName); +461 proc = createAssignProcedure(metaRegionInfo, serverName); +462 } else { +463 LOG.debug("Assigning " + metaRegionInfo.getRegionNameAsString()); +464 proc = createAssignProcedure(metaRegionInfo, false); +465 } +466 ProcedureSyncWait.submitAndWaitProcedure(master.getMasterProcedureExecutor(), proc); +467 } +468 +469 /** +470 * Start a new thread to check if there are region servers whose versions are higher than others. +471 * If so, move all system table regions to RS with the highest version to keep compatibility. +472 * The reason is, RS in new version may not be able to access RS in old version when there are +473 * some incompatible changes. +474 */ +475 public void checkIfShouldMoveSystemRegionAsync() { +476 new Thread(() -> { +477 try { +478 synchronized (checkIfShouldMoveSystemRegionLock) { +479 List<RegionPlan> plans = new ArrayList<>(); +480 for (ServerName server : getExcludedServersForSystemTable()) { +481 if (master.getServerManager().isServerDead(server)) { +482 // TODO: See HBASE-18494 and HBASE-18495. Though getExcludedServersForSystemTable() +483 // considers only online servers, the server could be queued for dead server +484 // processing. As region assignments for crashed server is handled by +485 // ServerCrashProcedure, do NOT handle them here. The goal is to handle this through +486 // regular flow of LoadBalancer as a favored node and not to have this special +487 // handling. +488 continue; +489 } +490 List<RegionInfo> regionsShouldMove = getCarryingSystemTables(server); +491 if (!regionsShouldMove.isEmpty()) { +492 for (RegionInfo regionInfo : regionsShouldMove) { +493 // null value for dest forces destination server to be selected by balancer +494 RegionPlan plan = new RegionPlan(regionInfo, server, null); +495 if (regionInfo.isMetaRegion()) { +496 // Must move meta region first. +497 moveAsync(plan); +498 } else { +499 plans.add(plan); +500 } +501 } +502 } +503 for (RegionPlan plan : plans) { +504 moveAsync(plan); +505 } +506 } +507 } +508 } catch (Throwable t) { +509 LOG.error(t); +510 } +511 }).start(); +512 } +513 +514 private List<RegionInfo> getCarryingSystemTables(ServerName serverName) { +515 Set<RegionStateNode> regions = this.getRegionStates().getServerNode(serverName).getRegions(); +516 if (regions == null) { +517 return new ArrayList<>(); +518 } +519 return regions.stream() +520 .map(RegionStateNode::getRegionInfo) +521 .filter(r -> r.getTable().isSystemTable()) +522 .collect(Collectors.toList()); +523 } +524 +525 public void assign(final RegionInfo regionInfo) throws IOException { +526 assign(regionInfo, true); +527 } +528 +529 public void assign(final RegionInfo regionInfo, final boolean forceNewPlan) throws IOException { +530 AssignProcedure proc = createAssignProcedure(regionInfo, forceNewPlan); +531 ProcedureSyncWait.submitAndWaitProcedure(master.getMasterProcedureExecutor(), proc); +532 } +533 +534 public void unassign(final RegionInfo regionInfo) throws IOException { +535 unassign(regionInfo, false); +536 } +537 +538 public void unassign(final RegionInfo regionInfo, final boolean forceNewPlan) +539 throws IOException { +540 // TODO: rename this reassign +541 RegionStateNode node = this.regionStates.getRegionNode(regionInfo); +542 ServerName destinationServer = node.getRegionLocation(); +543 if (destinationServer == null) { +544 throw new UnexpectedStateException("DestinationServer is null; Assigned? " + node.toString()); +545 } +546 assert destinationServer != null; node.toString(); +547 UnassignProcedure proc = createUnassignProcedure(regionInfo, destinationServer, forceNewPlan); +548 ProcedureSyncWait.submitAndWaitProcedure(master.getMasterProcedureExecutor(), proc); +549 } +550 +551 public void move(final RegionInfo regionInfo) throws IOException { +552 RegionStateNode node = this.regionStates.getRegionNode(regionInfo); +553 ServerName sourceServer = node.getRegionLocation(); +554 RegionPlan plan = new RegionPlan(regionInfo, sourceServer, null); +555 MoveRegionProcedure proc = createMoveRegionProcedure(plan); +556 ProcedureSyncWait.submitAndWaitProcedure(master.getMasterProcedureExecutor(), proc); +557 } +558 +559 public Future<byte[]> moveAsync(final RegionPlan regionPlan) { +560 MoveRegionProcedure proc = createMoveRegionProcedure(regionPlan); +561 return ProcedureSyncWait.submitProcedure(master.getMasterProcedureExecutor(), proc); +562 } +563 +564 @VisibleForTesting +565 public boolean waitForAssignment(final RegionInfo regionInfo) throws IOException { +566 return waitForAssignment(regionInfo, Long.MAX_VALUE); +567 } +568 +569 @VisibleForTesting +570 // TODO: Remove this? +571 public boolean waitForAssignment(final RegionInfo regionInfo, final long timeout) +572 throws IOException { +573 RegionStateNode node = null; +574 // This method can be called before the regionInfo has made it into the regionStateMap +575 // so wait around here a while. +576 long startTime = System.currentTimeMillis(); +577 // Something badly wrong if takes ten seconds to register a region. +578 long endTime = startTime + 10000; +579 while ((node = regionStates.getRegionNode(regionInfo)) == null && isRunning() && +580 System.currentTimeMillis() < endTime) { +581 // Presume it not yet added but will be added soon. Let it spew a lot so we can tell if +582 // we are waiting here alot. +583 LOG.debug("Waiting on " + regionInfo + " to be added to regionStateMap"); +584 Threads.sleep(10); +585 } +586 if (node == null) { +587 if (!isRunning()) return false; +588 throw new RegionException(regionInfo.getRegionNameAsString() + " never registered with Assigment."); +589 } +590 +591 RegionTransitionProcedure proc = node.getProcedure(); +592 if (proc == null) { +593 throw new NoSuchProcedureException(node.toString()); +594 } +595 +596 ProcedureSyncWait.waitForProcedureToCompleteIOE( +597 master.getMasterProcedureExecutor(), proc, timeout); +598 return true; +599 } +600 +601 // ============================================================================================ +602 // RegionTransition procedures helpers +603 // ============================================================================================ +604 +605 public AssignProcedure[] createAssignProcedures(final Collection<RegionInfo> regionInfo) { +606 return createAssignProcedures(regionInfo, false); +607 } +608 +609 public AssignProcedure[] createAssignProcedures(final Collection<RegionInfo> regionInfo, +610 final boolean forceNewPlan) { +611 if (regionInfo.isEmpty()) return null; +612 final AssignProcedure[] procs = new AssignProcedure[regionInfo.size()]; +613 int index = 0; +614 for (RegionInfo hri: regionInfo) { +615 procs[index++] = createAssignProcedure(hri, forceNewPlan); +616 } +617 return procs; +618 } +619 +620 // Needed for the following method so it can type the created Array we return +621 private static final UnassignProcedure [] UNASSIGNED_PROCEDURE_FOR_TYPE_INFO = +622 new UnassignProcedure[0]; +623 +624 UnassignProcedure[] createUnassignProcedures(final Collection<RegionStateNode> nodes) { +625 if (nodes.isEmpty()) return null; +626 final List<UnassignProcedure> procs = new ArrayList<UnassignProcedure>(nodes.size()); +627 for (RegionStateNode node: nodes) { +628 if (!this.regionStates.include(node, false)) continue; +629 // Look for regions that are offline/closed; i.e. already unassigned. +630 if (this.regionStates.isRegionOffline(node.getRegionInfo())) continue; +631 assert node.getRegionLocation() != null: node.toString(); +632 procs.add(createUnassignProcedure(node.getRegionInfo(), node.getRegionLocation(), false)); +633 } +634 return procs.toArray(UNASSIGNED_PROCEDURE_FOR_TYPE_INFO); +635 } +636 +637 public MoveRegionProcedure[] createReopenProcedures(final Collection<RegionInfo> regionInfo) { +638 final MoveRegionProcedure[] procs = new MoveRegionProcedure[regionInfo.size()]; +639 int index = 0; +640 for (RegionInfo hri: regionInfo) { +641 final ServerName serverName = regionStates.getRegionServerOfRegion(hri); +642 final RegionPlan plan = new RegionPlan(hri, serverName, serverName); +643 procs[index++] = createMoveRegionProcedure(plan); +644 } +645 return procs; +646 } +647 +648 /** +649 * Called by things like EnableTableProcedure to get a list of AssignProcedure +650 * to assign the regions of the table. +651 */ +652 public AssignProcedure[] createAssignProcedures(final TableName tableName) { +653 return createAssignProcedures(regionStates.getRegionsOfTable(tableName)); +654 } +655 +656 /** +657 * Called by things like DisableTableProcedure to get a list of UnassignProcedure +658 * to unassign the regions of the table. +659 */ +660 public UnassignProcedure[] createUnassignProcedures(final TableName tableName) { +661 return createUnassignProcedures(regionStates.getTableRegionStateNodes(tableName)); +662 } +663 +664 /** +665 * Called by things like ModifyColumnFamilyProcedure to get a list of MoveRegionProcedure +666 * to reopen the regions of the table. +667 */ +668 public MoveRegionProcedure[] createReopenProcedures(final TableName tableName) { +669 return createReopenProcedures(regionStates.getRegionsOfTable(tableName)); +670 } +671 +672 public AssignProcedure createAssignProcedure(final RegionInfo regionInfo, +673 final boolean forceNewPlan) { +674 AssignProcedure proc = new AssignProcedure(regionInfo, forceNewPlan); +675 proc.setOwner(getProcedureEnvironment().getRequestUser().getShortName()); +676 return proc; +677 } +678 +679 public AssignProcedure createAssignProcedure(final RegionInfo regionInfo, +680 final ServerName targetServer) { +681 AssignProcedure proc = new AssignProcedure(regionInfo, targetServer); +682 proc.setOwner(getProcedureEnvironment().getRequestUser().getShortName()); +683 return proc; +684 } +685 +686 public UnassignProcedure createUnassignProcedure(final RegionInfo regionInfo, +687 final ServerName destinationServer, final boolean force) { +688 // If destinationServer is null, figure it. +689 ServerName sn = destinationServer != null? destinationServer: +690 getRegionStates().getRegionState(regionInfo).getServerName(); +691 assert sn != null; +692 UnassignProcedure proc = new UnassignProcedure(regionInfo, sn, force); +693 proc.setOwner(getProcedureEnvironment().getRequestUser().getShortName()); +694 return proc; +695 } +696 +697 public MoveRegionProcedure createMoveRegionProcedure(final RegionPlan plan) { +698 if (plan.getRegionInfo().getTable().isSystemTable()) { +699 List<ServerName> exclude = getExcludedServersForSystemTable(); +700 if (plan.getDestination() != null && exclude.contains(plan.getDestination())) { +701 try { +702 LOG.info("Can not move " + plan.getRegionInfo() + " to " + plan.getDestination() +703 + " because the server is not with highest version"); +704 plan.setDestination(getBalancer().randomAssignment(plan.getRegionInfo(), +705 this.master.getServerManager().createDestinationServersList(exclude))); +706 } catch (HBaseIOException e) { +707 LOG.warn(e); +708 } +709 } +710 } +711 return new MoveRegionProcedure(getProcedureEnvironment(), plan); +712 } +713 714 -715 -716 public SplitTableRegionProcedure createSplitProcedure(final RegionInfo regionToSplit, -717 final byte[] splitKey) throws IOException { -718 return new SplitTableRegionProcedure(getProcedureEnvironment(), regionToSplit, splitKey); -719 } -720 -721 public MergeTableRegionsProcedure createMergeProcedure(final RegionInfo regionToMergeA, -722 final RegionInfo regionToMergeB) throws IOException { -723 return new MergeTableRegionsProcedure(getProcedureEnvironment(), regionToMergeA,regionToMergeB); -724 } -725 -726 /** -727 * Delete the region states. This is called by "DeleteTable" -728 */ -729 public void deleteTable(final TableName tableName) throws IOException { -730 final ArrayList<RegionInfo> regions = regionStates.getTableRegionsInfo(tableName); -731 regionStateStore.deleteRegions(regions); -732 for (int i = 0; i < regions.size(); ++i) { -733 final RegionInfo regionInfo = regions.get(i); -734 // we expect the region to be offline -735 regionStates.removeFromOfflineRegions(regionInfo); -736 regionStates.deleteRegion(regionInfo); -737 } -738 } -739 -740 // ============================================================================================ -741 // RS Region Transition Report helpers -742 // ============================================================================================ -743 // TODO: Move this code in MasterRpcServices and call on specific event? -744 public ReportRegionStateTransitionResponse reportRegionStateTransition( -745 final ReportRegionStateTransitionRequest req) -746 throws PleaseHoldException { -747 final ReportRegionStateTransitionResponse.Builder builder = -748 ReportRegionStateTransitionResponse.newBuilder(); -749 final ServerName serverName = ProtobufUtil.toServerName(req.getServer()); -750 try { -751 for (RegionStateTransition transition: req.getTransitionList()) { -752 switch (transition.getTransitionCode()) { -753 case OPENED: -754 case FAILED_OPEN: -755 case CLOSED: -756 assert transition.getRegionInfoCount() == 1 : transition; -757 final RegionInfo hri = ProtobufUtil.toRegionInfo(transition.getRegionInfo(0)); -758 updateRegionTransition(serverName, transition.getTransitionCode(), hri, -759 transition.hasOpenSeqNum() ? transition.getOpenSeqNum() : HConstants.NO_SEQNUM); -760 break; -761 case READY_TO_SPLIT: -762 case SPLIT: -763 case SPLIT_REVERTED: -764 assert transition.getRegionInfoCount() == 3 : transition; -765 final RegionInfo parent = ProtobufUtil.toRegionInfo(transition.getRegionInfo(0)); -766 final RegionInfo splitA = ProtobufUtil.toRegionInfo(transition.getRegionInfo(1)); -767 final RegionInfo splitB = ProtobufUtil.toRegionInfo(transition.getRegionInfo(2)); -768 updateRegionSplitTransition(serverName, transition.getTransitionCode(), -769 parent, splitA, splitB); -770 break; -771 case READY_TO_MERGE: -772 case MERGED: -773 case MERGE_REVERTED: -774 assert transition.getRegionInfoCount() == 3 : transition; -775 final RegionInfo merged = ProtobufUtil.toRegionInfo(transition.getRegionInfo(0)); -776 final RegionInfo mergeA = ProtobufUtil.toRegionInfo(transition.getRegionInfo(1)); -777 final RegionInfo mergeB = ProtobufUtil.toRegionInfo(transition.getRegionInfo(2)); -778 updateRegionMergeTransition(serverName, transition.getTransitionCode(), -779 merged, mergeA, mergeB); -780 break; -781 } -782 } -783 } catch (PleaseHoldException e) { -784 if (LOG.isTraceEnabled()) LOG.trace("Failed transition " + e.getMessage()); -785 throw e; -786 } catch (UnsupportedOperationException|IOException e) { -787 // TODO: at the moment we have a single error message and the RS will abort -788 // if the master says that one of the region transitions failed. -789 LOG.warn("Failed transition", e); -790 builder.setErrorMessage("Failed transition " + e.getMessage()); -791 } -792 return builder.build(); -793 } -794 -795 private void updateRegionTransition(final ServerName serverName, final TransitionCode state, -796 final RegionInfo regionInfo, final long seqId) -797 throws PleaseHoldException, UnexpectedStateException { -798 checkFailoverCleanupCompleted(regionInfo); -799 -800 final RegionStateNode regionNode = regionStates.getRegionNode(regionInfo); -801 if (regionNode == null) { -802 // the table/region is gone. maybe a delete, split, merge -803 throw new UnexpectedStateException(String.format( -804 "Server %s was trying to transition region %s to %s. but the region was removed.", -805 serverName, regionInfo, state)); -806 } -807 -808 if (LOG.isTraceEnabled()) { -809 LOG.trace(String.format("Update region transition serverName=%s region=%s state=%s", -810 serverName, regionNode, state)); -811 } -812 -813 final ServerStateNode serverNode = regionStates.getOrCreateServer(serverName); -814 if (!reportTransition(regionNode, serverNode, state, seqId)) { -815 LOG.warn(String.format( -816 "No procedure for %s. server=%s to transition to %s", regionNode, serverName, state)); -817 } -818 } -819 -820 // FYI: regionNode is sometimes synchronized by the caller but not always. -821 private boolean reportTransition(final RegionStateNode regionNode, -822 final ServerStateNode serverNode, final TransitionCode state, final long seqId) -823 throws UnexpectedStateException { -824 final ServerName serverName = serverNode.getServerName(); -825 synchronized (regionNode) { -826 final RegionTransitionProcedure proc = regionNode.getProcedure(); -827 if (proc == null) return false; -828 -829 // serverNode.getReportEvent().removeProcedure(proc); -830 proc.reportTransition(master.getMasterProcedureExecutor().getEnvironment(), -831 serverName, state, seqId); -832 } -833 return true; -834 } -835 -836 private void updateRegionSplitTransition(final ServerName serverName, final TransitionCode state, -837 final RegionInfo parent, final RegionInfo hriA, final RegionInfo hriB) -838 throws IOException { -839 checkFailoverCleanupCompleted(parent); -840 -841 if (state != TransitionCode.READY_TO_SPLIT) { -842 throw new UnexpectedStateException("unsupported split state=" + state + -843 " for parent region " + parent + -844 " maybe an old RS (< 2.0) had the operation in progress"); -845 } -846 -847 // sanity check on the request -848 if (!Bytes.equals(hriA.getEndKey(), hriB.getStartKey())) { -849 throw new UnsupportedOperationException( -850 "unsupported split request with bad keys: parent=" + parent + -851 " hriA=" + hriA + " hriB=" + hriB); -852 } -853 -854 // Submit the Split procedure -855 final byte[] splitKey = hriB.getStartKey(); -856 if (LOG.isDebugEnabled()) { -857 LOG.debug("Split request from " + serverName + -858 ", parent=" + parent + " splitKey=" + Bytes.toStringBinary(splitKey)); -859 } -860 master.getMasterProcedureExecutor().submitProcedure(createSplitProcedure(parent, splitKey)); -861 -862 // If the RS is < 2.0 throw an exception to abort the operation, we are handling the split -863 if (regionStates.getOrCreateServer(serverName).getVersionNumber() < 0x0200000) { -864 throw new UnsupportedOperationException(String.format( -865 "Split handled by the master: parent=%s hriA=%s hriB=%s", parent.getShortNameToLog(), hriA, hriB)); -866 } -867 } -868 -869 private void updateRegionMergeTransition(final ServerName serverName, final TransitionCode state, -870 final RegionInfo merged, final RegionInfo hriA, final RegionInfo hriB) throws IOException { -871 checkFailoverCleanupCompleted(merged); -872 -873 if (state != TransitionCode.READY_TO_MERGE) { -874 throw new UnexpectedStateException("Unsupported merge state=" + state + -875 " for regionA=" + hriA + " regionB=" + hriB + " merged=" + merged + -876 " maybe an old RS (< 2.0) had the operation in progress"); -877 } -878 -879 // Submit the Merge procedure -880 if (LOG.isDebugEnabled()) { -881 LOG.debug("Handling merge request from RS=" + merged + ", merged=" + merged); -882 } -883 master.getMasterProcedureExecutor().submitProcedure(createMergeProcedure(hriA, hriB)); -884 -885 // If the RS is < 2.0 throw an exception to abort the operation, we are handling the merge -886 if (regionStates.getOrCreateServer(serverName).getVersionNumber() < 0x0200000) { -887 throw new UnsupportedOperationException(String.format( -888 "Merge not handled yet: state=%s merged=%s hriA=%s hriB=%s", state, merged, hriA, hriB)); -889 } -890 } -891 -892 // ============================================================================================ -893 // RS Status update (report online regions) helpers -894 // ============================================================================================ -895 /** -896 * the master will call this method when the RS send the regionServerReport(). -897 * the report will contains the "hbase version" and the "online regions". -898 * this method will check the the online regions against the in-memory state of the AM, -899 * if there is a mismatch we will try to fence out the RS with the assumption -900 * that something went wrong on the RS side. -901 */ -902 public void reportOnlineRegions(final ServerName serverName, -903 final int versionNumber, final Set<byte[]> regionNames) throws YouAreDeadException { -904 if (!isRunning()) return; -905 if (LOG.isTraceEnabled()) { -906 LOG.trace("ReportOnlineRegions " + serverName + " regionCount=" + regionNames.size() + -907 ", metaLoaded=" + isMetaLoaded() + " " + -908 regionNames.stream().map(element -> Bytes.toStringBinary(element)). -909 collect(Collectors.toList())); -910 } -911 -912 final ServerStateNode serverNode = regionStates.getOrCreateServer(serverName); -913 -914 // update the server version number. This will be used for live upgrades. -915 synchronized (serverNode) { -916 serverNode.setVersionNumber(versionNumber); -917 if (serverNode.isInState(ServerState.SPLITTING, ServerState.OFFLINE)) { -918 LOG.warn("Got a report from a server result in state " + serverNode.getState()); -919 return; -920 } -921 } -922 -923 if (regionNames.isEmpty()) { -924 // nothing to do if we don't have regions -925 LOG.trace("no online region found on " + serverName); -926 } else if (!isMetaLoaded()) { -927 // if we are still on startup, discard the report unless is from someone holding meta -928 checkOnlineRegionsReportForMeta(serverNode, regionNames); -929 } else { -930 // The Heartbeat updates us of what regions are only. check and verify the state. -931 checkOnlineRegionsReport(serverNode, regionNames); -932 } -933 -934 // wake report event -935 wakeServerReportEvent(serverNode); -936 } -937 -938 void checkOnlineRegionsReportForMeta(final ServerStateNode serverNode, -939 final Set<byte[]> regionNames) { -940 try { -941 for (byte[] regionName: regionNames) { -942 final RegionInfo hri = getMetaRegionFromName(regionName); -943 if (hri == null) { -944 if (LOG.isTraceEnabled()) { -945 LOG.trace("Skip online report for region=" + Bytes.toStringBinary(regionName) + -946 " while meta is loading"); -947 } -948 continue; -949 } -950 -951 final RegionStateNode regionNode = regionStates.getOrCreateRegionNode(hri