Return-Path: X-Original-To: apmail-hbase-commits-archive@www.apache.org Delivered-To: apmail-hbase-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 4CF2AD78A for ; Wed, 18 Jul 2012 22:01:36 +0000 (UTC) Received: (qmail 29950 invoked by uid 500); 18 Jul 2012 22:01:36 -0000 Delivered-To: apmail-hbase-commits-archive@hbase.apache.org Received: (qmail 29914 invoked by uid 500); 18 Jul 2012 22:01:36 -0000 Mailing-List: contact commits-help@hbase.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@hbase.apache.org Delivered-To: mailing list commits@hbase.apache.org Received: (qmail 29900 invoked by uid 99); 18 Jul 2012 22:01:36 -0000 Received: from athena.apache.org (HELO athena.apache.org) (140.211.11.136) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 18 Jul 2012 22:01:36 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=5.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 18 Jul 2012 22:01:34 +0000 Received: from eris.apache.org (localhost [127.0.0.1]) by eris.apache.org (Postfix) with ESMTP id 61EC6238897F for ; Wed, 18 Jul 2012 22:01:15 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r1363139 - in /hbase/branches/0.89-fb/src: main/java/org/apache/hadoop/hbase/master/ test/java/org/apache/hadoop/hbase/master/ Date: Wed, 18 Jul 2012 22:01:14 -0000 To: commits@hbase.apache.org From: mbautin@apache.org X-Mailer: svnmailer-1.0.8-patched Message-Id: <20120718220115.61EC6238897F@eris.apache.org> X-Virus-Checked: Checked by ClamAV on apache.org Author: mbautin Date: Wed Jul 18 22:01:14 2012 New Revision: 1363139 URL: http://svn.apache.org/viewvc?rev=1363139&view=rev Log: [master] do not process ops from dead servers Author: pkhemani Summary: if the master has already processed death of a region server then it should not process any other messages from that server. The shutdown processing takes care of (hopefully) all the region specific processing. Test Plan: running existing unit tests Reviewers: kranganathan, mbautin Reviewed By: mbautin CC: hbase-eng@ Differential Revision: https://phabricator.fb.com/D522392 Task ID: 1177836 Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/master/HMaster.java hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/master/ProcessRegionClose.java hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/master/ProcessRegionOpen.java hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/master/ProcessRegionStatusChange.java hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/master/ProcessServerShutdown.java hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/master/RegionServerOperation.java hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/master/RegionServerOperationQueue.java hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/master/TestRegionServerOperationQueue.java Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/master/HMaster.java URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/master/HMaster.java?rev=1363139&r1=1363138&r2=1363139&view=diff ============================================================================== --- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/master/HMaster.java (original) +++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/master/HMaster.java Wed Jul 18 22:01:14 2012 @@ -404,13 +404,15 @@ public class HMaster extends HasThread i detectClusterStartup(); isActiveMaster = true; - this.regionServerOperationQueue = - new RegionServerOperationQueue(this.conf, getClosedStatus()); - synchronized(this) { serverManager = new ServerManager(this); } + this.regionServerOperationQueue = + new RegionServerOperationQueue(this.conf, serverManager, + getClosedStatus()); + + // Start the unassigned watcher - which will create the unassigned region // in ZK. This is needed before RegionManager() constructor tries to assign // the root region. Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/master/ProcessRegionClose.java URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/master/ProcessRegionClose.java?rev=1363139&r1=1363138&r2=1363139&view=diff ============================================================================== --- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/master/ProcessRegionClose.java (original) +++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/master/ProcessRegionClose.java Wed Jul 18 22:01:14 2012 @@ -42,10 +42,11 @@ public class ProcessRegionClose extends * @param offlineRegion if true, set the region to offline in meta * @param reassignRegion if true, region is to be reassigned */ - public ProcessRegionClose(HMaster master, HRegionInfo regionInfo, + public ProcessRegionClose(HMaster master, String serverName, + HRegionInfo regionInfo, boolean offlineRegion, boolean reassignRegion) { - super(master, regionInfo); + super(master, serverName, regionInfo); this.offlineRegion = offlineRegion; this.reassignRegion = reassignRegion; } @@ -53,7 +54,8 @@ public class ProcessRegionClose extends @Override public String toString() { return "ProcessRegionClose of " + this.regionInfo.getRegionNameAsString() + - ", " + this.offlineRegion + ", reassign: " + this.reassignRegion; + ", " + this.offlineRegion + ", reassign: " + this.reassignRegion + + " from server " + this.serverName; } @Override Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/master/ProcessRegionOpen.java URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/master/ProcessRegionOpen.java?rev=1363139&r1=1363138&r2=1363139&view=diff ============================================================================== --- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/master/ProcessRegionOpen.java (original) +++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/master/ProcessRegionOpen.java Wed Jul 18 22:01:14 2012 @@ -45,7 +45,7 @@ public class ProcessRegionOpen extends P */ public ProcessRegionOpen(HMaster master, HServerInfo info, HRegionInfo regionInfo) { - super(master, regionInfo); + super(master, info.getServerName(), regionInfo); if (info == null) { throw new NullPointerException("HServerInfo cannot be null; " + "hbase-958 debugging"); @@ -55,7 +55,8 @@ public class ProcessRegionOpen extends P @Override public String toString() { - return "PendingOpenOperation from " + serverInfo.getServerName(); + return "PendingOpenOperation from " + serverInfo.getServerName() + + " for region " + this.regionInfo.getRegionNameAsString(); } @Override Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/master/ProcessRegionStatusChange.java URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/master/ProcessRegionStatusChange.java?rev=1363139&r1=1363138&r2=1363139&view=diff ============================================================================== --- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/master/ProcessRegionStatusChange.java (original) +++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/master/ProcessRegionStatusChange.java Wed Jul 18 22:01:14 2012 @@ -36,8 +36,9 @@ abstract class ProcessRegionStatusChange * @param master the master * @param regionInfo region info */ - public ProcessRegionStatusChange(HMaster master, HRegionInfo regionInfo) { - super(master); + public ProcessRegionStatusChange(HMaster master, String serverName, + HRegionInfo regionInfo) { + super(master, serverName); this.regionInfo = regionInfo; this.isMetaTable = regionInfo.isMetaTable(); } Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/master/ProcessServerShutdown.java URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/master/ProcessServerShutdown.java?rev=1363139&r1=1363138&r2=1363139&view=diff ============================================================================== --- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/master/ProcessServerShutdown.java (original) +++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/master/ProcessServerShutdown.java Wed Jul 18 22:01:14 2012 @@ -77,7 +77,7 @@ class ProcessServerShutdown extends Regi * @param serverInfo */ public ProcessServerShutdown(HMaster master, HServerInfo serverInfo) { - super(master); + super(master, serverInfo.getServerName()); this.deadServer = serverInfo.getServerName(); this.deadServerAddress = serverInfo.getServerAddress(); this.rootRescanned = false; Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/master/RegionServerOperation.java URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/master/RegionServerOperation.java?rev=1363139&r1=1363138&r2=1363139&view=diff ============================================================================== --- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/master/RegionServerOperation.java (original) +++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/master/RegionServerOperation.java Wed Jul 18 22:01:14 2012 @@ -41,12 +41,14 @@ abstract class RegionServerOperation imp private long expire; protected final HMaster master; + protected final String serverName; // server sent the message or task /* How long we stay on queue. */ private int delay; - protected RegionServerOperation(HMaster master) { + protected RegionServerOperation(HMaster master, String serverName) { this.master = master; + this.serverName = serverName; this.delay = this.master.getConfiguration(). getInt("hbase.server.thread.wakefrequency", 10 * 1000); // Set the future time at which we expect to be released from the Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/master/RegionServerOperationQueue.java URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/master/RegionServerOperationQueue.java?rev=1363139&r1=1363138&r2=1363139&view=diff ============================================================================== --- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/master/RegionServerOperationQueue.java (original) +++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/master/RegionServerOperationQueue.java Wed Jul 18 22:01:14 2012 @@ -94,9 +94,13 @@ public class RegionServerOperationQueue new CopyOnWriteArraySet(); private final int threadWakeFrequency; private final Sleeper sleeper; + private final ServerManager smgr; - RegionServerOperationQueue(final Configuration c, StopStatus stop) { - this.threadWakeFrequency = c.getInt(HConstants.THREAD_WAKE_FREQUENCY, 10 * 1000); + RegionServerOperationQueue(final Configuration c, ServerManager smgr, + StopStatus stop) { + this.threadWakeFrequency = c.getInt(HConstants.THREAD_WAKE_FREQUENCY, + 10 * 1000); + this.smgr = smgr; this.stop = stop; this.sleeper = new Sleeper(this.threadWakeFrequency, stop); } @@ -146,6 +150,12 @@ public class RegionServerOperationQueue return ProcessingResultCode.NOOP; } + if (!(op instanceof ProcessServerShutdown)) { + if (smgr.getServerInfo(op.serverName) == null) { + LOG.debug("Discarding RegionServerOperation because server is dead " + + op); + } + } try { if (LOG.isDebugEnabled()) { LOG.debug("Processing todo: " + op.toString()); Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java?rev=1363139&r1=1363138&r2=1363139&view=diff ============================================================================== --- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java (original) +++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java Wed Jul 18 22:01:14 2012 @@ -804,7 +804,8 @@ public class ServerManager { // the region's state. this.master.getRegionManager().setClosed(region.getRegionNameAsString()); RegionServerOperation op = - new ProcessRegionClose(master, region, offlineRegion, reassignRegion); + new ProcessRegionClose(master, serverInfo.getServerName(), + region, offlineRegion, reassignRegion); this.master.getRegionServerOperationQueue().put(op); if (reassignRegion) { Modified: hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/master/TestRegionServerOperationQueue.java URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/master/TestRegionServerOperationQueue.java?rev=1363139&r1=1363138&r2=1363139&view=diff ============================================================================== --- hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/master/TestRegionServerOperationQueue.java (original) +++ hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/master/TestRegionServerOperationQueue.java Wed Jul 18 22:01:14 2012 @@ -45,7 +45,7 @@ public class TestRegionServerOperationQu @Before public void setUp() throws Exception { this.conf = new Configuration(); - this.queue = new RegionServerOperationQueue(this.conf, stopStatus); + this.queue = new RegionServerOperationQueue(this.conf, null, stopStatus); } @After