Return-Path: X-Original-To: apmail-hbase-commits-archive@www.apache.org Delivered-To: apmail-hbase-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id A2929182F9 for ; Tue, 1 Dec 2015 01:12:11 +0000 (UTC) Received: (qmail 46680 invoked by uid 500); 1 Dec 2015 01:12:11 -0000 Delivered-To: apmail-hbase-commits-archive@hbase.apache.org Received: (qmail 46642 invoked by uid 500); 1 Dec 2015 01:12:11 -0000 Mailing-List: contact commits-help@hbase.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@hbase.apache.org Delivered-To: mailing list commits@hbase.apache.org Received: (qmail 46633 invoked by uid 99); 1 Dec 2015 01:12:11 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 01 Dec 2015 01:12:11 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 470CAE2C45; Tue, 1 Dec 2015 01:12:11 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: enis@apache.org To: commits@hbase.apache.org Message-Id: X-Mailer: ASF-Git Admin Mailer Subject: hbase git commit: HBASE-14223 Meta WALs are not cleared if meta region was closed and RS aborts Date: Tue, 1 Dec 2015 01:12:11 +0000 (UTC) Repository: hbase Updated Branches: refs/heads/branch-1.2 a7530c48d -> 89aa2f1a8 HBASE-14223 Meta WALs are not cleared if meta region was closed and RS aborts Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/89aa2f1a Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/89aa2f1a Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/89aa2f1a Branch: refs/heads/branch-1.2 Commit: 89aa2f1a800469adb3f43177bdb68e3159cbaeca Parents: a7530c4 Author: Enis Soztutar Authored: Mon Nov 30 17:07:54 2015 -0800 Committer: Enis Soztutar Committed: Mon Nov 30 17:08:23 2015 -0800 ---------------------------------------------------------------------- .../hbase/chaos/actions/MoveMetaAction.java | 36 ++++ .../chaos/factories/NoKillMonkeyFactory.java | 4 +- .../SlowDeterministicMonkeyFactory.java | 4 +- .../StressAssignmentManagerMonkeyFactory.java | 4 +- .../hbase/regionserver/HRegionServer.java | 20 +++ .../hadoop/hbase/regionserver/LogRoller.java | 4 + .../hbase/regionserver/RSRpcServices.java | 5 +- .../regionserver/RegionServerServices.java | 6 + .../regionserver/handler/CloseMetaHandler.java | 8 + .../handler/CloseRegionHandler.java | 15 +- .../org/apache/hadoop/hbase/wal/WALFactory.java | 15 ++ .../hadoop/hbase/MockRegionServerServices.java | 8 +- .../hadoop/hbase/master/MockRegionServer.java | 10 +- .../hadoop/hbase/wal/TestMetaWALsAreClosed.java | 166 +++++++++++++++++++ 14 files changed, 291 insertions(+), 14 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hbase/blob/89aa2f1a/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/MoveMetaAction.java ---------------------------------------------------------------------- diff --git a/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/MoveMetaAction.java b/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/MoveMetaAction.java new file mode 100644 index 0000000..e38ed21 --- /dev/null +++ b/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/MoveMetaAction.java @@ -0,0 +1,36 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.chaos.actions; + +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.chaos.factories.MonkeyConstants; + +/** + * Action that moves the meta table region(s). + */ +public class MoveMetaAction extends MoveRegionsOfTableAction { + + public MoveMetaAction() { + this(-1, MonkeyConstants.DEFAULT_MOVE_REGIONS_MAX_TIME); + } + + public MoveMetaAction(long sleepTime, long maxSleepTime) { + super(sleepTime, maxSleepTime, TableName.META_TABLE_NAME); + } +} http://git-wip-us.apache.org/repos/asf/hbase/blob/89aa2f1a/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/factories/NoKillMonkeyFactory.java ---------------------------------------------------------------------- diff --git a/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/factories/NoKillMonkeyFactory.java b/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/factories/NoKillMonkeyFactory.java index def9df0..12b2163 100644 --- a/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/factories/NoKillMonkeyFactory.java +++ b/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/factories/NoKillMonkeyFactory.java @@ -30,6 +30,7 @@ import org.apache.hadoop.hbase.chaos.actions.DumpClusterStatusAction; import org.apache.hadoop.hbase.chaos.actions.FlushRandomRegionOfTableAction; import org.apache.hadoop.hbase.chaos.actions.FlushTableAction; import org.apache.hadoop.hbase.chaos.actions.MergeRandomAdjacentRegionsOfTableAction; +import org.apache.hadoop.hbase.chaos.actions.MoveMetaAction; import org.apache.hadoop.hbase.chaos.actions.MoveRandomRegionOfTableAction; import org.apache.hadoop.hbase.chaos.actions.MoveRegionsOfTableAction; import org.apache.hadoop.hbase.chaos.actions.RemoveColumnAction; @@ -52,7 +53,8 @@ public class NoKillMonkeyFactory extends MonkeyFactory { MonkeyConstants.DEFAULT_COMPACT_RANDOM_REGION_RATIO), new FlushTableAction(tableName), new FlushRandomRegionOfTableAction(tableName), - new MoveRandomRegionOfTableAction(tableName) + new MoveRandomRegionOfTableAction(tableName), + new MoveMetaAction() }; Action[] actions2 = new Action[] { http://git-wip-us.apache.org/repos/asf/hbase/blob/89aa2f1a/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/factories/SlowDeterministicMonkeyFactory.java ---------------------------------------------------------------------- diff --git a/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/factories/SlowDeterministicMonkeyFactory.java b/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/factories/SlowDeterministicMonkeyFactory.java index 093df2a..bcf653c 100644 --- a/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/factories/SlowDeterministicMonkeyFactory.java +++ b/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/factories/SlowDeterministicMonkeyFactory.java @@ -57,7 +57,8 @@ public class SlowDeterministicMonkeyFactory extends MonkeyFactory { new CompactRandomRegionOfTableAction(tableName, compactRandomRegionRatio), new FlushTableAction(tableName), new FlushRandomRegionOfTableAction(tableName), - new MoveRandomRegionOfTableAction(tableName) + new MoveRandomRegionOfTableAction(tableName), + new MoveMetaAction() }; // Actions such as split/merge/snapshot. @@ -89,6 +90,7 @@ public class SlowDeterministicMonkeyFactory extends MonkeyFactory { new RestartRsHoldingMetaAction(restartRsHoldingMetaSleepTime), new DecreaseMaxHFileSizeAction(decreaseHFileSizeSleepTime, tableName), new SplitAllRegionOfTableAction(tableName), + new MoveMetaAction() }; // Action to log more info for debugging http://git-wip-us.apache.org/repos/asf/hbase/blob/89aa2f1a/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/factories/StressAssignmentManagerMonkeyFactory.java ---------------------------------------------------------------------- diff --git a/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/factories/StressAssignmentManagerMonkeyFactory.java b/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/factories/StressAssignmentManagerMonkeyFactory.java index 03471ab..a76fb3c 100644 --- a/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/factories/StressAssignmentManagerMonkeyFactory.java +++ b/hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/factories/StressAssignmentManagerMonkeyFactory.java @@ -34,7 +34,8 @@ public class StressAssignmentManagerMonkeyFactory extends MonkeyFactory { new CompactTableAction(tableName, 0.5f), new CompactRandomRegionOfTableAction(tableName, 0.6f), new FlushTableAction(tableName), - new FlushRandomRegionOfTableAction(tableName) + new FlushRandomRegionOfTableAction(tableName), + new MoveMetaAction() }; Action[] actions2 = new Action[]{ @@ -55,6 +56,7 @@ public class StressAssignmentManagerMonkeyFactory extends MonkeyFactory { new SplitAllRegionOfTableAction(tableName), new DecreaseMaxHFileSizeAction(MonkeyConstants.DEFAULT_DECREASE_HFILE_SIZE_SLEEP_TIME, tableName), + new MoveMetaAction() }; // Action to log more info for debugging http://git-wip-us.apache.org/repos/asf/hbase/blob/89aa2f1a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java index fe56c2d..5fc4d76 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java @@ -1858,6 +1858,26 @@ public class HRegionServer extends HasThread implements } @Override + public void releaseWAL(HRegionInfo regionInfo, WAL wal) throws IOException { + if (regionInfo != null && regionInfo.isMetaTable() && + regionInfo.getReplicaId() == HRegionInfo.DEFAULT_REPLICA_ID) { + + walFactory.closeMetaWAL(regionInfo.getEncodedNameAsBytes()); + + LogRoller roller; + if (regionInfo != null && regionInfo.isMetaTable() && + regionInfo.getReplicaId() == HRegionInfo.DEFAULT_REPLICA_ID) { + roller = metawalRoller.get(); + if (roller != null) { + roller.removeWAL(wal); // only do this for meta WAL + } + + // TODO: meta wal roller is left running. Should be fine. + } + } + } + + @Override public ClusterConnection getConnection() { return this.clusterConnection; } http://git-wip-us.apache.org/repos/asf/hbase/blob/89aa2f1a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/LogRoller.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/LogRoller.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/LogRoller.java index 136e03e..39c63ac 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/LogRoller.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/LogRoller.java @@ -82,6 +82,10 @@ public class LogRoller extends HasThread { } } + public void removeWAL(final WAL wal) { + walNeedsRoll.remove(wal); + } + public void requestRollAll() { for (WAL wal : walNeedsRoll.keySet()) { walNeedsRoll.put(wal, Boolean.TRUE); http://git-wip-us.apache.org/repos/asf/hbase/blob/89aa2f1a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java index 00d20aa..66ac50b 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java @@ -165,7 +165,6 @@ import org.apache.hadoop.hbase.regionserver.Region.Operation; import org.apache.hadoop.hbase.regionserver.ScannerContext.LimitScope; import org.apache.hadoop.hbase.regionserver.handler.OpenMetaHandler; import org.apache.hadoop.hbase.regionserver.handler.OpenRegionHandler; -import org.apache.hadoop.hbase.wal.WAL; import org.apache.hadoop.hbase.regionserver.wal.WALEdit; import org.apache.hadoop.hbase.security.User; import org.apache.hadoop.hbase.util.Bytes; @@ -175,6 +174,7 @@ import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; import org.apache.hadoop.hbase.util.Pair; import org.apache.hadoop.hbase.util.ServerRegionReplicaUtil; import org.apache.hadoop.hbase.util.Strings; +import org.apache.hadoop.hbase.wal.WAL; import org.apache.hadoop.hbase.wal.WALKey; import org.apache.hadoop.hbase.wal.WALSplitter; import org.apache.hadoop.hbase.zookeeper.ZKSplitLog; @@ -1609,9 +1609,8 @@ public class RSRpcServices implements HBaseRPCErrorHandler, return response; } - HRegion.warmupHRegion(region, htd, regionServer.getWAL(region), + HRegion.warmupHRegion(region, htd, null, regionServer.getConfiguration(), regionServer, null); - } catch (IOException ie) { LOG.error("Failed warming up region " + region.getRegionNameAsString(), ie); throw new ServiceException(ie); http://git-wip-us.apache.org/repos/asf/hbase/blob/89aa2f1a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerServices.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerServices.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerServices.java index 34d7b50..2956cbb 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerServices.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerServices.java @@ -55,6 +55,12 @@ public interface RegionServerServices extends OnlineRegions, FavoredNodesForRegi WAL getWAL(HRegionInfo regionInfo) throws IOException; /** + * Releases the dependency of this region to the WAL previously obtained from + * {@link #getWAL(HRegionInfo)}. + */ + void releaseWAL(HRegionInfo regionInfo, WAL wal) throws IOException; + + /** * @return Implementation of {@link CompactionRequestor} or null. */ CompactionRequestor getCompactionRequester(); http://git-wip-us.apache.org/repos/asf/hbase/blob/89aa2f1a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/handler/CloseMetaHandler.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/handler/CloseMetaHandler.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/handler/CloseMetaHandler.java index dba9240..72d06d2 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/handler/CloseMetaHandler.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/handler/CloseMetaHandler.java @@ -19,6 +19,9 @@ package org.apache.hadoop.hbase.regionserver.handler; import org.apache.hadoop.hbase.classification.InterfaceAudience; + +import java.io.IOException; + import org.apache.hadoop.hbase.HRegionInfo; import org.apache.hadoop.hbase.Server; import org.apache.hadoop.hbase.executor.EventType; @@ -40,4 +43,9 @@ public class CloseMetaHandler extends CloseRegionHandler { super(server, rsServices, regionInfo, abort, closeRegionCoordination, crd, EventType.M_RS_CLOSE_META); } + + @Override + protected void releaseWALIfNeeded() throws IOException { + rsServices.releaseWAL(regionInfo, region.getWAL()); + } } http://git-wip-us.apache.org/repos/asf/hbase/blob/89aa2f1a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/handler/CloseRegionHandler.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/handler/CloseRegionHandler.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/handler/CloseRegionHandler.java index ccb0e72..2e0fc7f 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/handler/CloseRegionHandler.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/handler/CloseRegionHandler.java @@ -46,8 +46,9 @@ public class CloseRegionHandler extends EventHandler { // have a running queue of user regions to close? private static final Log LOG = LogFactory.getLog(CloseRegionHandler.class); - private final RegionServerServices rsServices; - private final HRegionInfo regionInfo; + protected final RegionServerServices rsServices; + protected final HRegionInfo regionInfo; + protected HRegion region; // If true, the hosting server is aborting. Region close process is different // when we are aborting. @@ -119,7 +120,7 @@ public class CloseRegionHandler extends EventHandler { LOG.debug("Processing close of " + name); String encodedRegionName = regionInfo.getEncodedName(); // Check that this region is being served here - HRegion region = (HRegion)rsServices.getFromOnlineRegions(encodedRegionName); + region = (HRegion)rsServices.getFromOnlineRegions(encodedRegionName); if (region == null) { LOG.warn("Received CLOSE for region " + name + " but currently not serving - ignoring"); // TODO: do better than a simple warning @@ -143,6 +144,10 @@ public class CloseRegionHandler extends EventHandler { regionInfo.getRegionNameAsString()); return; } + + if (!abort) { + releaseWALIfNeeded(); + } } catch (IOException ioe) { // An IOException here indicates that we couldn't successfully flush the // memstore before closing. So, we need to abort the server and allow @@ -167,4 +172,8 @@ public class CloseRegionHandler extends EventHandler { remove(this.regionInfo.getEncodedNameAsBytes()); } } + + protected void releaseWALIfNeeded() throws IOException { + // release the WAL if needed. Only meta does this for now. + } } http://git-wip-us.apache.org/repos/asf/hbase/blob/89aa2f1a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALFactory.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALFactory.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALFactory.java index e44a4d1..9dbc9b5 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALFactory.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALFactory.java @@ -250,6 +250,21 @@ public class WALFactory { return metaProvider.getWAL(identifier); } + /** + * Closes the meta WAL and meta WALProvider + */ + public void closeMetaWAL(final byte[] identifier) throws IOException { + // NOTE: this assumes single META region. The close of WAL does not do ref-counting for the + // number of regions depending on the meta WAL + WALProvider metaProvider = this.metaProvider.get(); + if (metaProvider != null) { + if (this.metaProvider.compareAndSet(metaProvider, null)) { + // close the metaProvider + metaProvider.close(); + } + } + } + public Reader createReader(final FileSystem fs, final Path path) throws IOException { return createReader(fs, path, (CancelableProgressable)null); } http://git-wip-us.apache.org/repos/asf/hbase/blob/89aa2f1a/hbase-server/src/test/java/org/apache/hadoop/hbase/MockRegionServerServices.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/MockRegionServerServices.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/MockRegionServerServices.java index 8802ea6..c3a2873 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/MockRegionServerServices.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/MockRegionServerServices.java @@ -102,7 +102,7 @@ public class MockRegionServerServices implements RegionServerServices { public List getOnlineRegions(TableName tableName) throws IOException { return null; } - + @Override public Set getOnlineTables() { return null; @@ -181,7 +181,7 @@ public class MockRegionServerServices implements RegionServerServices { public TableLockManager getTableLockManager() { return new NullTableLockManager(); } - + @Override public RegionServerQuotaManager getRegionServerQuotaManager() { return null; @@ -304,4 +304,8 @@ public class MockRegionServerServices implements RegionServerServices { public double getCompactionPressure() { return 0; } + + @Override + public void releaseWAL(HRegionInfo regionInfo, WAL wal) throws IOException { + } } http://git-wip-us.apache.org/repos/asf/hbase/blob/89aa2f1a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/MockRegionServer.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/MockRegionServer.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/MockRegionServer.java index 8aa38d6..3f36b0e 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/MockRegionServer.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/MockRegionServer.java @@ -66,8 +66,6 @@ import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.MergeRegionsReques import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.MergeRegionsResponse; import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.OpenRegionRequest; import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.OpenRegionResponse; -import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.WarmupRegionRequest; -import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.WarmupRegionResponse; import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.ReplicateWALEntryRequest; import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.ReplicateWALEntryResponse; import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.RollWALWriterRequest; @@ -80,6 +78,8 @@ import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.UpdateConfiguratio import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.UpdateConfigurationResponse; import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.UpdateFavoredNodesRequest; import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.UpdateFavoredNodesResponse; +import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.WarmupRegionRequest; +import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.WarmupRegionResponse; import org.apache.hadoop.hbase.protobuf.generated.ClientProtos; import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.BulkLoadHFileRequest; import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.BulkLoadHFileResponse; @@ -541,7 +541,7 @@ ClientProtos.ClientService.BlockingInterface, RegionServerServices { // TODO Auto-generated method stub return null; } - + @Override public Set getOnlineTables() { return null; @@ -649,4 +649,8 @@ ClientProtos.ClientService.BlockingInterface, RegionServerServices { public double getCompactionPressure() { return 0; } + + @Override + public void releaseWAL(HRegionInfo regionInfo, WAL wal) throws IOException { + } } http://git-wip-us.apache.org/repos/asf/hbase/blob/89aa2f1a/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestMetaWALsAreClosed.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestMetaWALsAreClosed.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestMetaWALsAreClosed.java new file mode 100644 index 0000000..040ce66 --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestMetaWALsAreClosed.java @@ -0,0 +1,166 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hbase.wal; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; + +import java.io.IOException; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.MiniHBaseCluster; +import org.apache.hadoop.hbase.ServerName; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.Waiter.Predicate; +import org.apache.hadoop.hbase.client.Admin; +import org.apache.hadoop.hbase.client.Connection; +import org.apache.hadoop.hbase.client.ConnectionFactory; +import org.apache.hadoop.hbase.client.RegionLocator; +import org.apache.hadoop.hbase.client.Table; +import org.apache.hadoop.hbase.testclassification.MediumTests; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.FSUtils; +import org.apache.hadoop.hbase.util.JVMClusterUtil.RegionServerThread; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +/** + * Tests the case where a meta region is opened in one regionserver and closed, there should not + * be any WALs left over. + */ +@Category({MediumTests.class}) +public class TestMetaWALsAreClosed { + protected static final Log LOG = LogFactory.getLog(TestMetaWALsAreClosed.class); + + protected static final int NUM_RS = 2; + + protected static final HBaseTestingUtility TEST_UTIL = + new HBaseTestingUtility(); + + + protected final Configuration conf = TEST_UTIL.getConfiguration(); + + @Before + public void setUp() throws Exception { + TEST_UTIL.startMiniCluster(1, NUM_RS); + } + + @After + public void tearDown() throws Exception { + TEST_UTIL.shutdownMiniCluster(); + } + + private boolean isHostingMeta(FileSystem fs, Path wals, ServerName serverName) + throws IOException { + for (FileStatus status : fs.listStatus(wals)) { + LOG.info(status.getPath()); + if (DefaultWALProvider.isMetaFile(status.getPath())) { + return true; // only 1 meta region for now + } + } + return false; + } + + private void moveMetaRegionAndWait(final ServerName target) throws Exception { + try (final Connection conn = ConnectionFactory.createConnection(TEST_UTIL.getConfiguration()); + final Admin admin = conn.getAdmin(); + final RegionLocator rl = conn.getRegionLocator(TableName.META_TABLE_NAME)) { + + LOG.info("Disabling balancer"); + admin.setBalancerRunning(false, true); + + LOG.info("Moving meta region"); + admin.move(HRegionInfo.FIRST_META_REGIONINFO.getEncodedNameAsBytes(), + Bytes.toBytes(target.toString())); + + LOG.info("Waiting for meta region to move"); + // wait for the move of meta region + TEST_UTIL.waitFor(30000, new Predicate() { + @Override + public boolean evaluate() throws Exception { + return target.equals( + rl.getRegionLocation(HConstants.EMPTY_START_ROW, true).getServerName()); + } + }); + } + } + + @Test (timeout = 60000) + public void testMetaWALsAreClosed() throws Exception { + MiniHBaseCluster cluster = TEST_UTIL.getMiniHBaseCluster(); + + FileSystem fs = TEST_UTIL.getTestFileSystem(); + + // find the region server hosting the meta table now. + ServerName metaServerName = null; + ServerName otherServerName = null; + for (RegionServerThread rs : cluster.getRegionServerThreads()) { + ServerName serverName = rs.getRegionServer().getServerName(); + + Path wals = new Path(FSUtils.getRootDir(TEST_UTIL.getConfiguration()), + DefaultWALProvider.getWALDirectoryName(serverName.toString())); + + if (isHostingMeta(fs, wals, serverName)) { + metaServerName = serverName; // only 1 meta region for now + } else { + otherServerName = serverName; + } + } + + LOG.info(metaServerName); + LOG.info(otherServerName); + assertNotNull(metaServerName); + assertNotNull(otherServerName); + + moveMetaRegionAndWait(otherServerName); + + LOG.info("Checking that old meta server does not have WALs for meta"); + // the server that used to host meta now should not have any WAL files for the meta region now + Path wals = new Path(FSUtils.getRootDir(TEST_UTIL.getConfiguration()), + DefaultWALProvider.getWALDirectoryName(metaServerName.toString())); + for (FileStatus status : fs.listStatus(wals)) { + LOG.info(status.getPath()); + assertFalse(DefaultWALProvider.isMetaFile(status.getPath())); + } + + // assign the meta server back + moveMetaRegionAndWait(metaServerName); + + // do some basic operations to ensure that nothing is failing + HTableDescriptor htd = TEST_UTIL.createTableDescriptor("foo"); + TEST_UTIL.getHBaseAdmin().createTable(htd); + try (Connection conn = ConnectionFactory.createConnection(TEST_UTIL.getConfiguration()); + Table table = conn.getTable(htd.getTableName())) { + + TEST_UTIL.loadNumericRows(table, TEST_UTIL.fam1, 0, 100); + TEST_UTIL.verifyNumericRows(table, TEST_UTIL.fam1, 0, 100, 0); + } + } +}