Return-Path: X-Original-To: apmail-hadoop-hdfs-commits-archive@minotaur.apache.org Delivered-To: apmail-hadoop-hdfs-commits-archive@minotaur.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id BC9516BA2 for ; Tue, 7 Jun 2011 22:30:49 +0000 (UTC) Received: (qmail 14597 invoked by uid 500); 7 Jun 2011 22:30:49 -0000 Delivered-To: apmail-hadoop-hdfs-commits-archive@hadoop.apache.org Received: (qmail 14572 invoked by uid 500); 7 Jun 2011 22:30:49 -0000 Mailing-List: contact hdfs-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: hdfs-dev@hadoop.apache.org Delivered-To: mailing list hdfs-commits@hadoop.apache.org Received: (qmail 14562 invoked by uid 99); 7 Jun 2011 22:30:49 -0000 Received: from athena.apache.org (HELO athena.apache.org) (140.211.11.136) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 07 Jun 2011 22:30:49 +0000 X-ASF-Spam-Status: No, hits=-1996.5 required=5.0 tests=ALL_TRUSTED,URIBL_BLACK X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 07 Jun 2011 22:30:46 +0000 Received: by eris.apache.org (Postfix, from userid 65534) id A37AA23889B2; Tue, 7 Jun 2011 22:30:26 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r1133183 - in /hadoop/hdfs/branches/HDFS-1073: ./ src/java/ src/java/org/apache/hadoop/hdfs/ src/java/org/apache/hadoop/hdfs/server/namenode/ src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/ Date: Tue, 07 Jun 2011 22:30:26 -0000 To: hdfs-commits@hadoop.apache.org From: todd@apache.org X-Mailer: svnmailer-1.0.8 Message-Id: <20110607223026.A37AA23889B2@eris.apache.org> Author: todd Date: Tue Jun 7 22:30:25 2011 New Revision: 1133183 URL: http://svn.apache.org/viewvc?rev=1133183&view=rev Log: HDFS-2016. Add infrastructure to remove or archive old and unneeded storage files within the name directories. Contributed by Todd Lipcon. Added: hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/namenode/NNStorageArchivalManager.java hadoop/hdfs/branches/HDFS-1073/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestNNStorageArchivalManager.java Modified: hadoop/hdfs/branches/HDFS-1073/CHANGES.HDFS-1073.txt hadoop/hdfs/branches/HDFS-1073/src/java/hdfs-default.xml hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/DFSConfigKeys.java hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/namenode/FSImageTransactionalStorageInspector.java hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/namenode/GetImageServlet.java hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/namenode/NNStorage.java hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java hadoop/hdfs/branches/HDFS-1073/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestCheckpoint.java hadoop/hdfs/branches/HDFS-1073/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestFSImageStorageInspector.java Modified: hadoop/hdfs/branches/HDFS-1073/CHANGES.HDFS-1073.txt URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1073/CHANGES.HDFS-1073.txt?rev=1133183&r1=1133182&r2=1133183&view=diff ============================================================================== --- hadoop/hdfs/branches/HDFS-1073/CHANGES.HDFS-1073.txt (original) +++ hadoop/hdfs/branches/HDFS-1073/CHANGES.HDFS-1073.txt Tue Jun 7 22:30:25 2011 @@ -43,3 +43,5 @@ HDFS-1994. Fix race conditions when runn HDFS-2001. Remove use of previous.checkpoint and lastcheckpoint.tmp directories (todd) HDFS-2015. Remove checkpointTxId from VERSION file. (todd) +HDFS-2016. Add infrastructure to remove or archive old and unneeded storage + files within the name directories. (todd) Modified: hadoop/hdfs/branches/HDFS-1073/src/java/hdfs-default.xml URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1073/src/java/hdfs-default.xml?rev=1133183&r1=1133182&r2=1133183&view=diff ============================================================================== --- hadoop/hdfs/branches/HDFS-1073/src/java/hdfs-default.xml (original) +++ hadoop/hdfs/branches/HDFS-1073/src/java/hdfs-default.xml Tue Jun 7 22:30:25 2011 @@ -590,6 +590,16 @@ creations/deletions), or "all". + dfs.namenode.num.checkpoints.retained + 2 + The number of image checkpoint files that will be retained by + the NameNode and Secondary NameNode in their storage directories. All edit + logs necessary to recover an up-to-date namespace from the oldest retained + checkpoint will also be retained. + + + + dfs.namenode.delegation.key.update-interval 86400000 The update interval for master key for delegation tokens Modified: hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/DFSConfigKeys.java URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/DFSConfigKeys.java?rev=1133183&r1=1133182&r2=1133183&view=diff ============================================================================== --- hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/DFSConfigKeys.java (original) +++ hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/DFSConfigKeys.java Tue Jun 7 22:30:25 2011 @@ -106,6 +106,9 @@ public class DFSConfigKeys extends Commo public static final boolean DFS_NAMENODE_NAME_DIR_RESTORE_DEFAULT = false; public static final String DFS_NAMENODE_SUPPORT_ALLOW_FORMAT_KEY = "dfs.namenode.support.allow.format"; public static final boolean DFS_NAMENODE_SUPPORT_ALLOW_FORMAT_DEFAULT = true; + public static final String DFS_NAMENODE_NUM_CHECKPOINTS_RETAINED_KEY = "dfs.namenode.num.checkpoints.retained"; + public static final int DFS_NAMENODE_NUM_CHECKPOINTS_RETAINED_DEFAULT = 2; + public static final String DFS_LIST_LIMIT = "dfs.ls.limit"; public static final int DFS_LIST_LIMIT_DEFAULT = 1000; public static final String DFS_DATANODE_FAILED_VOLUMES_TOLERATED_KEY = "dfs.datanode.failed.volumes.tolerated"; Modified: hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java?rev=1133183&r1=1133182&r2=1133183&view=diff ============================================================================== --- hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java (original) +++ hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java Tue Jun 7 22:30:25 2011 @@ -567,51 +567,6 @@ public class FSImage implements Closeabl storage.writeTransactionIdFileToStorage(editLog.getCurSegmentTxId()); }; - private FSImageStorageInspector inspectStorageDirs() throws IOException { - int minLayoutVersion = Integer.MAX_VALUE; // the newest - int maxLayoutVersion = Integer.MIN_VALUE; // the oldest - - // First determine what range of layout versions we're going to inspect - for (Iterator it = storage.dirIterator(); - it.hasNext();) { - StorageDirectory sd = it.next(); - if (!sd.getVersionFile().exists()) { - LOG.warn("Storage directory " + sd + " contains no VERSION file. Skipping..."); - continue; - } - sd.read(); // sets layoutVersion - minLayoutVersion = Math.min(minLayoutVersion, storage.getLayoutVersion()); - maxLayoutVersion = Math.max(maxLayoutVersion, storage.getLayoutVersion()); - } - - if (minLayoutVersion > maxLayoutVersion) { - throw new IOException("No storage directories contained VERSION information"); - } - assert minLayoutVersion <= maxLayoutVersion; - - // If we have any storage directories with the new layout version - // (ie edits_) then use the new inspector, which will ignore - // the old format dirs. - FSImageStorageInspector inspector; - if (LayoutVersion.supports(Feature.TXID_BASED_LAYOUT, minLayoutVersion)) { - inspector = new FSImageTransactionalStorageInspector(); - if (!LayoutVersion.supports(Feature.TXID_BASED_LAYOUT, maxLayoutVersion)) { - LOG.warn("Ignoring one or more storage directories with old layouts"); - } - } else { - inspector = new FSImageOldStorageInspector(); - } - - // Process each of the storage directories to find the pair of - // newest image file and edit file - for (Iterator it = storage.dirIterator(); it.hasNext();) { - StorageDirectory sd = it.next(); - inspector.inspectDirectory(sd); - } - - return inspector; - } - /** * Choose latest image from one of the directories, * load it and merge with the edits from that directory. @@ -628,7 +583,7 @@ public class FSImage implements Closeabl * @throws IOException */ boolean loadFSImage() throws IOException { - FSImageStorageInspector inspector = inspectStorageDirs(); + FSImageStorageInspector inspector = storage.readAndInspectDirs(); isUpgradeFinalized = inspector.isUpgradeFinalized(); @@ -854,6 +809,10 @@ public class FSImage implements Closeabl // TODO Double-check for regressions against HDFS-1505 and HDFS-1921. renameCheckpoint(txid); + + // Since we now have a new checkpoint, we can clean up some + // old edit logs and checkpoints. + storage.archiveOldStorage(); } /** Modified: hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/namenode/FSImageTransactionalStorageInspector.java URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/namenode/FSImageTransactionalStorageInspector.java?rev=1133183&r1=1133182&r2=1133183&view=diff ============================================================================== --- hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/namenode/FSImageTransactionalStorageInspector.java (original) +++ hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/namenode/FSImageTransactionalStorageInspector.java Tue Jun 7 22:30:25 2011 @@ -41,6 +41,7 @@ import org.apache.hadoop.hdfs.server.nam import org.apache.hadoop.hdfs.server.protocol.RemoteEditLog; import org.apache.hadoop.hdfs.server.protocol.RemoteEditLogManifest; +import com.google.common.collect.ImmutableList; import com.google.common.collect.Lists; class FSImageTransactionalStorageInspector extends FSImageStorageInspector { @@ -180,6 +181,14 @@ class FSImageTransactionalStorageInspect } return ret; } + + public List getFoundImages() { + return ImmutableList.copyOf(foundImages); + } + + public List getFoundEditLogs() { + return ImmutableList.copyOf(foundEditLogs); + } @Override public LoadPlan createLoadPlan() throws IOException { @@ -442,6 +451,11 @@ class FSImageTransactionalStorageInspect public long getTxId() { return txId; } + + @Override + public String toString() { + return file.toString(); + } } /** @@ -513,6 +527,11 @@ class FSImageTransactionalStorageInspect } file = dst; } + + @Override + public String toString() { + return file.toString(); + } } static class TransactionalLoadPlan extends LoadPlan { Modified: hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/namenode/GetImageServlet.java URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/namenode/GetImageServlet.java?rev=1133183&r1=1133182&r2=1133183&view=diff ============================================================================== --- hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/namenode/GetImageServlet.java (original) +++ hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/namenode/GetImageServlet.java Tue Jun 7 22:30:25 2011 @@ -130,6 +130,10 @@ public class GetImageServlet extends Htt } }); nnImage.saveDigestAndRenameCheckpointImage(txid, downloadImageDigest); + + // Now that we have a new checkpoint, we might be able to + // remove some old ones. + nnImage.getStorage().archiveOldStorage(); } finally { currentlyDownloadingCheckpoints.remove(txid); } Modified: hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/namenode/NNStorage.java URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/namenode/NNStorage.java?rev=1133183&r1=1133182&r2=1133183&view=diff ============================================================================== --- hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/namenode/NNStorage.java (original) +++ hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/namenode/NNStorage.java Tue Jun 7 22:30:25 2011 @@ -121,6 +121,8 @@ public class NNStorage extends Storage i private UpgradeManager upgradeManager = null; protected String blockpoolID = ""; // id of the block pool + + private final NNStorageArchivalManager archivalManager; /** * flag that controls if we try to restore failed storages @@ -158,18 +160,8 @@ public class NNStorage extends Storage i storageDirs = new CopyOnWriteArrayList(); setStorageDirectories(imageDirs, editsDirs); - } - - /** - * Construct the NNStorage. - * @param storageInfo storage information - * @param bpid block pool Id - */ - public NNStorage(StorageInfo storageInfo, String bpid) { - super(NodeType.NAME_NODE, storageInfo); - - storageDirs = new CopyOnWriteArrayList(); - this.blockpoolID = bpid; + + archivalManager = new NNStorageArchivalManager(conf, this); } @Override // Storage @@ -542,6 +534,19 @@ public class NNStorage extends Storage i } /** + * Archive any files in the storage directories that are no longer + * necessary. + */ + public void archiveOldStorage() { + try { + archivalManager.archiveOldStorage(); + } catch (Exception e) { + LOG.warn("Unable to archive old storage", e); + } + } + + + /** * Generate new namespaceID. * * namespaceID is a persistent attribute of the namespace. @@ -942,4 +947,67 @@ public class NNStorage extends Storage i public String getBlockPoolID() { return blockpoolID; } + + /** + * Iterate over all current storage directories, inspecting them + * with the given inspector. + */ + void inspectStorageDirs(FSImageStorageInspector inspector) + throws IOException { + + // Process each of the storage directories to find the pair of + // newest image file and edit file + for (Iterator it = dirIterator(); it.hasNext();) { + StorageDirectory sd = it.next(); + inspector.inspectDirectory(sd); + } + } + + /** + * Iterate over all of the storage dirs, reading their contents to determine + * their layout versions. Returns an FSImageStorageInspector which has + * inspected each directory. + * + * Note: this can mutate the storage info fields (ctime, version, etc). + * @throws IOException if no valid storage dirs are found + */ + FSImageStorageInspector readAndInspectDirs() + throws IOException { + int minLayoutVersion = Integer.MAX_VALUE; // the newest + int maxLayoutVersion = Integer.MIN_VALUE; // the oldest + + // First determine what range of layout versions we're going to inspect + for (Iterator it = dirIterator(); + it.hasNext();) { + StorageDirectory sd = it.next(); + if (!sd.getVersionFile().exists()) { + FSImage.LOG.warn("Storage directory " + sd + " contains no VERSION file. Skipping..."); + continue; + } + sd.read(); // sets layoutVersion + minLayoutVersion = Math.min(minLayoutVersion, getLayoutVersion()); + maxLayoutVersion = Math.max(maxLayoutVersion, getLayoutVersion()); + } + + if (minLayoutVersion > maxLayoutVersion) { + throw new IOException("No storage directories contained VERSION information"); + } + assert minLayoutVersion <= maxLayoutVersion; + + // If we have any storage directories with the new layout version + // (ie edits_) then use the new inspector, which will ignore + // the old format dirs. + FSImageStorageInspector inspector; + if (LayoutVersion.supports(Feature.TXID_BASED_LAYOUT, minLayoutVersion)) { + inspector = new FSImageTransactionalStorageInspector(); + if (!LayoutVersion.supports(Feature.TXID_BASED_LAYOUT, maxLayoutVersion)) { + FSImage.LOG.warn("Ignoring one or more storage directories with old layouts"); + } + } else { + inspector = new FSImageOldStorageInspector(); + } + + inspectStorageDirs(inspector); + return inspector; + } } Added: hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/namenode/NNStorageArchivalManager.java URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/namenode/NNStorageArchivalManager.java?rev=1133183&view=auto ============================================================================== --- hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/namenode/NNStorageArchivalManager.java (added) +++ hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/namenode/NNStorageArchivalManager.java Tue Jun 7 22:30:25 2011 @@ -0,0 +1,145 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdfs.server.namenode; + +import java.io.IOException; +import java.util.Collections; +import java.util.List; +import java.util.TreeSet; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hdfs.DFSConfigKeys; +import org.apache.hadoop.hdfs.server.namenode.FSImageTransactionalStorageInspector.FoundEditLog; +import org.apache.hadoop.hdfs.server.namenode.FSImageTransactionalStorageInspector.FoundFSImage; +import org.apache.hadoop.hdfs.util.MD5FileUtils; + +import com.google.common.collect.Lists; +import com.google.common.collect.Sets; + +/** + * The NNStorageArchivalManager is responsible for inspecting the storage + * directories of the NN and enforcing a retention policy on checkpoints + * and edit logs. + * + * It delegates the actual removal of files to a {@link #StorageArchiver} + * implementation, which might delete the files or instead copy them to + * a filer or HDFS for later analysis. + */ +public class NNStorageArchivalManager { + + private final int numCheckpointsToRetain; + private static final Log LOG = LogFactory.getLog(NNStorageArchivalManager.class); + private final NNStorage storage; + private final StorageArchiver archiver; + + public NNStorageArchivalManager( + Configuration conf, + NNStorage storage, + StorageArchiver archiver) { + this.numCheckpointsToRetain = conf.getInt( + DFSConfigKeys.DFS_NAMENODE_NUM_CHECKPOINTS_RETAINED_KEY, + DFSConfigKeys.DFS_NAMENODE_NUM_CHECKPOINTS_RETAINED_DEFAULT); + this.storage = storage; + this.archiver = archiver; + } + + public NNStorageArchivalManager(Configuration conf, NNStorage storage) { + this(conf, storage, new DeletionStorageArchiver()); + } + + public void archiveOldStorage() throws IOException { + FSImageTransactionalStorageInspector inspector = + new FSImageTransactionalStorageInspector(); + storage.inspectStorageDirs(inspector); + + long minImageTxId = getImageTxIdToRetain(inspector); + archiveCheckpointsOlderThan(inspector, minImageTxId); + archiveLogsOlderThan(inspector, minImageTxId); + } + + private void archiveLogsOlderThan( + FSImageTransactionalStorageInspector inspector, + long minImageTxId) { + for (FoundEditLog log : inspector.getFoundEditLogs()) { + if (log.getStartTxId() < minImageTxId) { + LOG.info("Purging old edit log " + log); + archiver.archiveLog(log); + } + } + } + + private void archiveCheckpointsOlderThan( + FSImageTransactionalStorageInspector inspector, + long minTxId) { + for (FoundFSImage image : inspector.getFoundImages()) { + if (image.getTxId() < minTxId) { + LOG.info("Purging old image " + image); + archiver.archiveImage(image); + } + } + } + + /** + * @param inspector inspector that has already inspected all storage dirs + * @return the transaction ID corresponding to the oldest checkpoint + * that should be retained. + */ + private long getImageTxIdToRetain(FSImageTransactionalStorageInspector inspector) { + + List images = inspector.getFoundImages(); + TreeSet imageTxIds = Sets.newTreeSet(); + for (FoundFSImage image : images) { + imageTxIds.add(image.getTxId()); + } + + List imageTxIdsList = Lists.newArrayList(imageTxIds); + if (imageTxIdsList.isEmpty()) { + return 0; + } + + Collections.reverse(imageTxIdsList); + int toRetain = Math.min(numCheckpointsToRetain, imageTxIdsList.size()); + long minTxId = imageTxIdsList.get(toRetain - 1); + LOG.info("Going to retain " + toRetain + " images with txid >= " + + minTxId); + return minTxId; + } + + /** + * Interface responsible for archiving old checkpoints and edit logs. + */ + static interface StorageArchiver { + void archiveLog(FoundEditLog log); + void archiveImage(FoundFSImage image); + } + + static class DeletionStorageArchiver implements StorageArchiver { + @Override + public void archiveLog(FoundEditLog log) { + log.getFile().delete(); + } + + @Override + public void archiveImage(FoundFSImage image) { + image.getFile().delete(); + MD5FileUtils.getDigestFileForFile(image.getFile()).delete(); + } + } +} Modified: hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java?rev=1133183&r1=1133182&r2=1133183&view=diff ============================================================================== --- hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java (original) +++ hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java Tue Jun 7 22:30:25 2011 @@ -43,6 +43,7 @@ import org.apache.hadoop.hdfs.server.com import org.apache.hadoop.hdfs.server.common.JspHelper; import org.apache.hadoop.hdfs.server.common.Storage.StorageDirectory; import org.apache.hadoop.hdfs.server.common.Storage.StorageState; +import org.apache.hadoop.hdfs.server.namenode.NNStorageArchivalManager.StorageArchiver; import org.apache.hadoop.hdfs.server.protocol.NamenodeProtocol; import org.apache.hadoop.hdfs.server.protocol.RemoteEditLog; import org.apache.hadoop.hdfs.server.protocol.RemoteEditLogManifest; @@ -492,6 +493,10 @@ public class SecondaryNameNode implement LOG.warn("Checkpoint done. New Image Size: " + checkpointImage.getStorage().getFsImageName(txid).length()); + // Since we've successfully checkpointed, we can remove some old + // image files + checkpointImage.getStorage().archiveOldStorage(); + return loadImage; } Modified: hadoop/hdfs/branches/HDFS-1073/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestCheckpoint.java URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1073/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestCheckpoint.java?rev=1133183&r1=1133182&r2=1133183&view=diff ============================================================================== --- hadoop/hdfs/branches/HDFS-1073/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestCheckpoint.java (original) +++ hadoop/hdfs/branches/HDFS-1073/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestCheckpoint.java Tue Jun 7 22:30:25 2011 @@ -1117,6 +1117,9 @@ public class TestCheckpoint extends Test // a checkpoint with a lower txid finished most recently) NNStorage storage = cluster.getNameNode().getFSImage().getStorage(); assertEquals(4, storage.getMostRecentCheckpointTxId()); + + // Should have accepted both checkpoints + assertNNHasCheckpoints(cluster, ImmutableList.of(2,4)); // Now have second one checkpoint one more time just to make sure that // the NN isn't left in a broken state @@ -1134,9 +1137,10 @@ public class TestCheckpoint extends Test // Validate invariant that files named the same are the same. assertParallelFilesInvariant(cluster, ImmutableList.of(secondary1, secondary2)); - // Validate that the NN received checkpoints at expected txids - // (i.e that both checkpoints went through) - assertNNHasCheckpoints(cluster, ImmutableList.of(2,4,6)); + + // NN should have removed the checkpoint at txid 2 at this point, but has + // one at txid 6 + assertNNHasCheckpoints(cluster, ImmutableList.of(4,6)); } @@ -1226,6 +1230,9 @@ public class TestCheckpoint extends Test secondary2.doCheckpoint(); assertEquals(6, storage.getMostRecentCheckpointTxId()); + // Should have accepted both checkpoints + assertNNHasCheckpoints(cluster, ImmutableList.of(4,6)); + // Let the first one also go again on its own to make sure it can // continue at next checkpoint secondary1.setNameNode(origNN); @@ -1245,7 +1252,7 @@ public class TestCheckpoint extends Test assertParallelFilesInvariant(cluster, ImmutableList.of(secondary1, secondary2)); // Validate that the NN received checkpoints at expected txids // (i.e that both checkpoints went through) - assertNNHasCheckpoints(cluster, ImmutableList.of(4,6,8)); + assertNNHasCheckpoints(cluster, ImmutableList.of(6,8)); } Modified: hadoop/hdfs/branches/HDFS-1073/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestFSImageStorageInspector.java URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1073/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestFSImageStorageInspector.java?rev=1133183&r1=1133182&r2=1133183&view=diff ============================================================================== --- hadoop/hdfs/branches/HDFS-1073/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestFSImageStorageInspector.java (original) +++ hadoop/hdfs/branches/HDFS-1073/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestFSImageStorageInspector.java Tue Jun 7 22:30:25 2011 @@ -358,7 +358,7 @@ public class TestFSImageStorageInspector * @param previousExists should we mock that the previous/ dir exists? * @param fileNames the names of files contained in current/ */ - private StorageDirectory mockDirectory( + static StorageDirectory mockDirectory( StorageDirType type, boolean previousExists, String... fileNames) { Added: hadoop/hdfs/branches/HDFS-1073/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestNNStorageArchivalManager.java URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1073/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestNNStorageArchivalManager.java?rev=1133183&view=auto ============================================================================== --- hadoop/hdfs/branches/HDFS-1073/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestNNStorageArchivalManager.java (added) +++ hadoop/hdfs/branches/HDFS-1073/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestNNStorageArchivalManager.java Tue Jun 7 22:30:25 2011 @@ -0,0 +1,219 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdfs.server.namenode; + +import java.io.IOException; +import java.util.Set; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hdfs.server.common.Storage.StorageDirectory; +import org.apache.hadoop.hdfs.server.namenode.FSImageTransactionalStorageInspector.FoundEditLog; +import org.apache.hadoop.hdfs.server.namenode.FSImageTransactionalStorageInspector.FoundFSImage; +import org.apache.hadoop.hdfs.server.namenode.NNStorage.NameNodeDirType; +import org.apache.hadoop.hdfs.server.namenode.NNStorageArchivalManager.StorageArchiver; +import org.junit.Assert; +import org.junit.Test; +import org.mockito.ArgumentCaptor; +import org.mockito.Mockito; +import org.mockito.invocation.InvocationOnMock; +import org.mockito.stubbing.Answer; + +import com.google.common.base.Joiner; +import com.google.common.collect.Sets; + + +public class TestNNStorageArchivalManager { + /** + * Test the "easy case" where we have more images in the + * directory than we need to keep. Should archive the + * old ones. + */ + @Test + public void testArchiveEasyCase() throws IOException { + TestCaseDescription tc = new TestCaseDescription(); + tc.addImage("/foo1/current/fsimage_100", true); + tc.addImage("/foo1/current/fsimage_200", true); + tc.addImage("/foo1/current/fsimage_300", false); + tc.addImage("/foo1/current/fsimage_400", false); + tc.addLog("/foo1/current/edits_101-200", true); + tc.addLog("/foo1/current/edits_201-300", true); + tc.addLog("/foo1/current/edits_301-400", false); + tc.addLog("/foo1/current/edits_inprogress_401", false); + + // Test that other files don't get archived + tc.addLog("/foo1/current/VERSION", false); + runTest(tc); + } + + /** + * Same as above, but across multiple directories + */ + @Test + public void testArchiveMultipleDirs() throws IOException { + TestCaseDescription tc = new TestCaseDescription(); + tc.addImage("/foo1/current/fsimage_100", true); + tc.addImage("/foo1/current/fsimage_200", true); + tc.addImage("/foo2/current/fsimage_200", true); + tc.addImage("/foo1/current/fsimage_300", false); + tc.addImage("/foo1/current/fsimage_400", false); + tc.addLog("/foo1/current/edits_101-200", true); + tc.addLog("/foo1/current/edits_201-300", true); + tc.addLog("/foo2/current/edits_201-300", true); + tc.addLog("/foo1/current/edits_301-400", false); + tc.addLog("/foo2/current/edits_301-400", false); + tc.addLog("/foo1/current/edits_inprogress_401", false); + runTest(tc); + } + + /** + * Test that if we have fewer fsimages than the configured + * retention, we don't archive any of them + */ + @Test + public void testArchiveLessThanRetention() throws IOException { + TestCaseDescription tc = new TestCaseDescription(); + tc.addImage("/foo1/current/fsimage_100", false); + tc.addLog("/foo1/current/edits_101-200", false); + tc.addLog("/foo1/current/edits_201-300", false); + tc.addLog("/foo1/current/edits_301-400", false); + tc.addLog("/foo1/current/edits_inprogress_401", false); + runTest(tc); + } + + /** + * Check for edge case with no logs present at all. + */ + @Test + public void testNoLogs() throws IOException { + TestCaseDescription tc = new TestCaseDescription(); + tc.addImage("/foo1/current/fsimage_100", true); + tc.addImage("/foo1/current/fsimage_200", true); + tc.addImage("/foo1/current/fsimage_300", false); + tc.addImage("/foo1/current/fsimage_400", false); + runTest(tc); + } + + /** + * Check for edge case with no logs or images present at all. + */ + @Test + public void testEmptyDir() throws IOException { + TestCaseDescription tc = new TestCaseDescription(); + runTest(tc); + } + + /** + * Test that old in-progress logs are properly archived + */ + @Test + public void testOldInProgress() throws IOException { + TestCaseDescription tc = new TestCaseDescription(); + tc.addImage("/foo1/current/fsimage_100", true); + tc.addImage("/foo1/current/fsimage_200", true); + tc.addImage("/foo1/current/fsimage_300", false); + tc.addImage("/foo1/current/fsimage_400", false); + tc.addLog("/foo1/current/edits_inprogress_101", true); + runTest(tc); + } + + private void runTest(TestCaseDescription tc) throws IOException { + Configuration conf = new Configuration(); + + StorageArchiver mockArchiver = + Mockito.mock(NNStorageArchivalManager.StorageArchiver.class); + ArgumentCaptor imagesArchivedCaptor = + ArgumentCaptor.forClass(FoundFSImage.class); + ArgumentCaptor logsArchivedCaptor = + ArgumentCaptor.forClass(FoundEditLog.class); + + // Ask the manager to archive files we don't need any more + new NNStorageArchivalManager(conf, tc.mockStorage(), mockArchiver) + .archiveOldStorage(); + + // Verify that it asked the archiver to remove the correct files + Mockito.verify(mockArchiver, Mockito.atLeast(0)) + .archiveImage(imagesArchivedCaptor.capture()); + Mockito.verify(mockArchiver, Mockito.atLeast(0)) + .archiveLog(logsArchivedCaptor.capture()); + + // Check images + Set archivedPaths = Sets.newHashSet(); + for (FoundFSImage archived : imagesArchivedCaptor.getAllValues()) { + archivedPaths.add(archived.getFile().toString()); + } + Assert.assertEquals(Joiner.on(",").join(tc.expectedArchivedImages), + Joiner.on(",").join(archivedPaths)); + + // Check images + archivedPaths.clear(); + for (FoundEditLog archived : logsArchivedCaptor.getAllValues()) { + archivedPaths.add(archived.getFile().toString()); + } + Assert.assertEquals(Joiner.on(",").join(tc.expectedArchivedLogs), + Joiner.on(",").join(archivedPaths)); + } + + private static class TestCaseDescription { + private Set files = Sets.newHashSet(); + private Set expectedArchivedLogs = Sets.newHashSet(); + private Set expectedArchivedImages = Sets.newHashSet(); + + void addLog(String path, boolean expectArchive) { + files.add(path); + if (expectArchive) { + expectedArchivedLogs.add(path); + } + } + + private String[] getPaths() { + return files.toArray(new String[0]); + } + + void addImage(String path, boolean expectArchive) { + files.add(path); + if (expectArchive) { + expectedArchivedImages.add(path); + } + } + + NNStorage mockStorage() throws IOException { + String[] paths = getPaths(); + StorageDirectory mockDir = TestFSImageStorageInspector.mockDirectory( + NameNodeDirType.IMAGE_AND_EDITS, false, paths); + return mockStorageForDirs(mockDir); + } + } + + private static NNStorage mockStorageForDirs(final StorageDirectory ... mockDirs) + throws IOException { + NNStorage mockStorage = Mockito.mock(NNStorage.class); + Mockito.doAnswer(new Answer() { + @Override + public Void answer(InvocationOnMock invocation) throws Throwable { + FSImageStorageInspector inspector = + (FSImageStorageInspector) invocation.getArguments()[0]; + for (StorageDirectory sd : mockDirs) { + inspector.inspectDirectory(sd); + } + return null; + } + }).when(mockStorage).inspectStorageDirs( + Mockito.anyObject()); + return mockStorage; + } +}