Return-Path: X-Original-To: apmail-hbase-commits-archive@www.apache.org Delivered-To: apmail-hbase-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 0701E1817E for ; Mon, 7 Dec 2015 16:55:40 +0000 (UTC) Received: (qmail 15755 invoked by uid 500); 7 Dec 2015 16:55:39 -0000 Delivered-To: apmail-hbase-commits-archive@hbase.apache.org Received: (qmail 15657 invoked by uid 500); 7 Dec 2015 16:55:39 -0000 Mailing-List: contact commits-help@hbase.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@hbase.apache.org Delivered-To: mailing list commits@hbase.apache.org Received: (qmail 14525 invoked by uid 99); 7 Dec 2015 16:55:38 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Mon, 07 Dec 2015 16:55:38 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 46B6CE0663; Mon, 7 Dec 2015 16:55:38 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: syuanjiang@apache.org To: commits@hbase.apache.org Date: Mon, 07 Dec 2015 16:55:44 -0000 Message-Id: <38a96a437fc54eaf89e17643c4347b1f@git.apache.org> In-Reply-To: <1d30bbd516af4c64b652d7945355f8fa@git.apache.org> References: <1d30bbd516af4c64b652d7945355f8fa@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [07/50] [abbrv] hbase git commit: HBASE-14223 Meta WALs are not cleared if meta region was closed and RS aborts HBASE-14223 Meta WALs are not cleared if meta region was closed and RS aborts Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/756e6de2 Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/756e6de2 Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/756e6de2 Branch: refs/heads/hbase-12439 Commit: 756e6de2a0948c39a4fb3ab880dd9d182d645dbc Parents: 999ae69 Author: Enis Soztutar Authored: Mon Nov 30 16:48:52 2015 -0800 Committer: Enis Soztutar Committed: Mon Nov 30 16:48:52 2015 -0800 ---------------------------------------------------------------------- .../hbase/regionserver/HRegionServer.java | 20 ++++++++++++++++++++ .../hadoop/hbase/regionserver/LogRoller.java | 4 ++++ .../hbase/regionserver/RSRpcServices.java | 2 +- .../regionserver/RegionServerServices.java | 6 ++++++ .../regionserver/handler/CloseMetaHandler.java | 8 ++++++++ .../handler/CloseRegionHandler.java | 15 ++++++++++++--- .../org/apache/hadoop/hbase/wal/WALFactory.java | 15 +++++++++++++++ .../hadoop/hbase/MockRegionServerServices.java | 4 ++++ .../hadoop/hbase/master/MockRegionServer.java | 10 +++++++--- 9 files changed, 77 insertions(+), 7 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hbase/blob/756e6de2/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java index 2ce2193..5be240d 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java @@ -1867,6 +1867,26 @@ public class HRegionServer extends HasThread implements } @Override + public void releaseWAL(HRegionInfo regionInfo, WAL wal) throws IOException { + if (regionInfo != null && regionInfo.isMetaTable() && + regionInfo.getReplicaId() == HRegionInfo.DEFAULT_REPLICA_ID) { + + walFactory.closeMetaWAL(regionInfo.getEncodedNameAsBytes()); + + LogRoller roller; + if (regionInfo != null && regionInfo.isMetaTable() && + regionInfo.getReplicaId() == HRegionInfo.DEFAULT_REPLICA_ID) { + roller = metawalRoller.get(); + if (roller != null) { + roller.removeWAL(wal); // only do this for meta WAL + } + + // TODO: meta wal roller is left running. Should be fine. + } + } + } + + @Override public ClusterConnection getConnection() { return this.clusterConnection; } http://git-wip-us.apache.org/repos/asf/hbase/blob/756e6de2/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/LogRoller.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/LogRoller.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/LogRoller.java index 40edc05..12cd060 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/LogRoller.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/LogRoller.java @@ -78,6 +78,10 @@ public class LogRoller extends HasThread { } } + public void removeWAL(final WAL wal) { + walNeedsRoll.remove(wal); + } + public void requestRollAll() { for (WAL wal : walNeedsRoll.keySet()) { walNeedsRoll.put(wal, Boolean.TRUE); http://git-wip-us.apache.org/repos/asf/hbase/blob/756e6de2/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java index d94e11c..6fadd0c 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java @@ -1669,7 +1669,7 @@ public class RSRpcServices implements HBaseRPCErrorHandler, return response; } - HRegion.warmupHRegion(region, htd, regionServer.getWAL(region), + HRegion.warmupHRegion(region, htd, null, regionServer.getConfiguration(), regionServer, null); } catch (IOException ie) { http://git-wip-us.apache.org/repos/asf/hbase/blob/756e6de2/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerServices.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerServices.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerServices.java index cd4816c..29e6a95 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerServices.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerServices.java @@ -55,6 +55,12 @@ public interface RegionServerServices extends OnlineRegions, FavoredNodesForRegi WAL getWAL(HRegionInfo regionInfo) throws IOException; /** + * Releases the dependency of this region to the WAL previously obtained from + * {@link #getWAL(HRegionInfo)}. + */ + void releaseWAL(HRegionInfo regionInfo, WAL wal) throws IOException; + + /** * @return Implementation of {@link CompactionRequestor} or null. */ CompactionRequestor getCompactionRequester(); http://git-wip-us.apache.org/repos/asf/hbase/blob/756e6de2/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/handler/CloseMetaHandler.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/handler/CloseMetaHandler.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/handler/CloseMetaHandler.java index 70e5283..237eb5a 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/handler/CloseMetaHandler.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/handler/CloseMetaHandler.java @@ -19,6 +19,9 @@ package org.apache.hadoop.hbase.regionserver.handler; import org.apache.hadoop.hbase.classification.InterfaceAudience; + +import java.io.IOException; + import org.apache.hadoop.hbase.HRegionInfo; import org.apache.hadoop.hbase.Server; import org.apache.hadoop.hbase.executor.EventType; @@ -37,4 +40,9 @@ public class CloseMetaHandler extends CloseRegionHandler { final boolean abort) { super(server, rsServices, regionInfo, abort, EventType.M_RS_CLOSE_META, null); } + + @Override + protected void releaseWALIfNeeded() throws IOException { + rsServices.releaseWAL(regionInfo, region.getWAL()); + } } http://git-wip-us.apache.org/repos/asf/hbase/blob/756e6de2/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/handler/CloseRegionHandler.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/handler/CloseRegionHandler.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/handler/CloseRegionHandler.java index 26e824d..b5bab27 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/handler/CloseRegionHandler.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/handler/CloseRegionHandler.java @@ -44,8 +44,9 @@ public class CloseRegionHandler extends EventHandler { // have a running queue of user regions to close? private static final Log LOG = LogFactory.getLog(CloseRegionHandler.class); - private final RegionServerServices rsServices; - private final HRegionInfo regionInfo; + protected final RegionServerServices rsServices; + protected final HRegionInfo regionInfo; + protected HRegion region; // If true, the hosting server is aborting. Region close process is different // when we are aborting. @@ -90,7 +91,7 @@ public class CloseRegionHandler extends EventHandler { LOG.debug("Processing close of " + name); String encodedRegionName = regionInfo.getEncodedName(); // Check that this region is being served here - HRegion region = (HRegion)rsServices.getFromOnlineRegions(encodedRegionName); + region = (HRegion)rsServices.getFromOnlineRegions(encodedRegionName); if (region == null) { LOG.warn("Received CLOSE for region " + name + " but currently not serving - ignoring"); // TODO: do better than a simple warning @@ -106,6 +107,10 @@ public class CloseRegionHandler extends EventHandler { regionInfo.getRegionNameAsString()); return; } + + if (!abort) { + releaseWALIfNeeded(); + } } catch (IOException ioe) { // An IOException here indicates that we couldn't successfully flush the // memstore before closing. So, we need to abort the server and allow @@ -125,4 +130,8 @@ public class CloseRegionHandler extends EventHandler { remove(this.regionInfo.getEncodedNameAsBytes(), Boolean.FALSE); } } + + protected void releaseWALIfNeeded() throws IOException { + // release the WAL if needed. Only meta does this for now. + } } http://git-wip-us.apache.org/repos/asf/hbase/blob/756e6de2/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALFactory.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALFactory.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALFactory.java index 0317b66..7426979 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALFactory.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALFactory.java @@ -250,6 +250,21 @@ public class WALFactory { return metaProvider.getWAL(identifier, null); } + /** + * Closes the meta WAL and meta WALProvider + */ + public void closeMetaWAL(final byte[] identifier) throws IOException { + // NOTE: this assumes single META region. The close of WAL does not do ref-counting for the + // number of regions depending on the meta WAL + WALProvider metaProvider = this.metaProvider.get(); + if (metaProvider != null) { + if (this.metaProvider.compareAndSet(metaProvider, null)) { + // close the metaProvider + metaProvider.close(); + } + } + } + public Reader createReader(final FileSystem fs, final Path path) throws IOException { return createReader(fs, path, (CancelableProgressable)null); } http://git-wip-us.apache.org/repos/asf/hbase/blob/756e6de2/hbase-server/src/test/java/org/apache/hadoop/hbase/MockRegionServerServices.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/MockRegionServerServices.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/MockRegionServerServices.java index c126b19..250d3c1 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/MockRegionServerServices.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/MockRegionServerServices.java @@ -306,4 +306,8 @@ public class MockRegionServerServices implements RegionServerServices { public double getCompactionPressure() { return 0; } + + @Override + public void releaseWAL(HRegionInfo regionInfo, WAL wal) throws IOException { + } } http://git-wip-us.apache.org/repos/asf/hbase/blob/756e6de2/hbase-server/src/test/java/org/apache/hadoop/hbase/master/MockRegionServer.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/MockRegionServer.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/MockRegionServer.java index eb8f803..e89ce00 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/MockRegionServer.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/MockRegionServer.java @@ -66,8 +66,6 @@ import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.MergeRegionsReques import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.MergeRegionsResponse; import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.OpenRegionRequest; import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.OpenRegionResponse; -import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.WarmupRegionRequest; -import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.WarmupRegionResponse; import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.ReplicateWALEntryRequest; import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.ReplicateWALEntryResponse; import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.RollWALWriterRequest; @@ -80,6 +78,8 @@ import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.UpdateConfiguratio import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.UpdateConfigurationResponse; import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.UpdateFavoredNodesRequest; import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.UpdateFavoredNodesResponse; +import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.WarmupRegionRequest; +import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.WarmupRegionResponse; import org.apache.hadoop.hbase.protobuf.generated.ClientProtos; import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.BulkLoadHFileRequest; import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.BulkLoadHFileResponse; @@ -651,4 +651,8 @@ ClientProtos.ClientService.BlockingInterface, RegionServerServices { public double getCompactionPressure() { return 0; } -} \ No newline at end of file + + @Override + public void releaseWAL(HRegionInfo regionInfo, WAL wal) throws IOException { + } +}