hadoop-hdfs-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From t...@apache.org
Subject svn commit: r1133183 - in /hadoop/hdfs/branches/HDFS-1073: ./ src/java/ src/java/org/apache/hadoop/hdfs/ src/java/org/apache/hadoop/hdfs/server/namenode/ src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/
Date Tue, 07 Jun 2011 22:30:26 GMT
Author: todd
Date: Tue Jun  7 22:30:25 2011
New Revision: 1133183

URL: http://svn.apache.org/viewvc?rev=1133183&view=rev
Log:
HDFS-2016. Add infrastructure to remove or archive old and unneeded storage files within the
name directories. Contributed by Todd Lipcon.

Added:
    hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/namenode/NNStorageArchivalManager.java
    hadoop/hdfs/branches/HDFS-1073/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestNNStorageArchivalManager.java
Modified:
    hadoop/hdfs/branches/HDFS-1073/CHANGES.HDFS-1073.txt
    hadoop/hdfs/branches/HDFS-1073/src/java/hdfs-default.xml
    hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
    hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java
    hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/namenode/FSImageTransactionalStorageInspector.java
    hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/namenode/GetImageServlet.java
    hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/namenode/NNStorage.java
    hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java
    hadoop/hdfs/branches/HDFS-1073/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestCheckpoint.java
    hadoop/hdfs/branches/HDFS-1073/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestFSImageStorageInspector.java

Modified: hadoop/hdfs/branches/HDFS-1073/CHANGES.HDFS-1073.txt
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1073/CHANGES.HDFS-1073.txt?rev=1133183&r1=1133182&r2=1133183&view=diff
==============================================================================
--- hadoop/hdfs/branches/HDFS-1073/CHANGES.HDFS-1073.txt (original)
+++ hadoop/hdfs/branches/HDFS-1073/CHANGES.HDFS-1073.txt Tue Jun  7 22:30:25 2011
@@ -43,3 +43,5 @@ HDFS-1994. Fix race conditions when runn
 HDFS-2001. Remove use of previous.checkpoint and lastcheckpoint.tmp directories
            (todd)
 HDFS-2015. Remove checkpointTxId from VERSION file. (todd)
+HDFS-2016. Add infrastructure to remove or archive old and unneeded storage
+           files within the name directories. (todd)

Modified: hadoop/hdfs/branches/HDFS-1073/src/java/hdfs-default.xml
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1073/src/java/hdfs-default.xml?rev=1133183&r1=1133182&r2=1133183&view=diff
==============================================================================
--- hadoop/hdfs/branches/HDFS-1073/src/java/hdfs-default.xml (original)
+++ hadoop/hdfs/branches/HDFS-1073/src/java/hdfs-default.xml Tue Jun  7 22:30:25 2011
@@ -590,6 +590,16 @@ creations/deletions), or "all".</descrip
 </property>
 
 <property>
+  <name>dfs.namenode.num.checkpoints.retained</name>
+  <value>2</value>
+  <description>The number of image checkpoint files that will be retained by
+  the NameNode and Secondary NameNode in their storage directories. All edit
+  logs necessary to recover an up-to-date namespace from the oldest retained
+  checkpoint will also be retained.
+  </description>
+</property>
+
+<property>
   <name>dfs.namenode.delegation.key.update-interval</name>
   <value>86400000</value>
   <description>The update interval for master key for delegation tokens 

Modified: hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/DFSConfigKeys.java?rev=1133183&r1=1133182&r2=1133183&view=diff
==============================================================================
--- hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/DFSConfigKeys.java (original)
+++ hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/DFSConfigKeys.java Tue
Jun  7 22:30:25 2011
@@ -106,6 +106,9 @@ public class DFSConfigKeys extends Commo
   public static final boolean DFS_NAMENODE_NAME_DIR_RESTORE_DEFAULT = false;
   public static final String  DFS_NAMENODE_SUPPORT_ALLOW_FORMAT_KEY = "dfs.namenode.support.allow.format";
   public static final boolean DFS_NAMENODE_SUPPORT_ALLOW_FORMAT_DEFAULT = true;
+  public static final String  DFS_NAMENODE_NUM_CHECKPOINTS_RETAINED_KEY = "dfs.namenode.num.checkpoints.retained";
+  public static final int     DFS_NAMENODE_NUM_CHECKPOINTS_RETAINED_DEFAULT = 2;
+  
   public static final String  DFS_LIST_LIMIT = "dfs.ls.limit";
   public static final int     DFS_LIST_LIMIT_DEFAULT = 1000;
   public static final String  DFS_DATANODE_FAILED_VOLUMES_TOLERATED_KEY = "dfs.datanode.failed.volumes.tolerated";

Modified: hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java?rev=1133183&r1=1133182&r2=1133183&view=diff
==============================================================================
--- hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java
(original)
+++ hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java
Tue Jun  7 22:30:25 2011
@@ -567,51 +567,6 @@ public class FSImage implements Closeabl
     storage.writeTransactionIdFileToStorage(editLog.getCurSegmentTxId());
   };
 
-  private FSImageStorageInspector inspectStorageDirs() throws IOException {
-    int minLayoutVersion = Integer.MAX_VALUE; // the newest
-    int maxLayoutVersion = Integer.MIN_VALUE; // the oldest
-    
-    // First determine what range of layout versions we're going to inspect
-    for (Iterator<StorageDirectory> it = storage.dirIterator();
-         it.hasNext();) {
-      StorageDirectory sd = it.next();
-      if (!sd.getVersionFile().exists()) {
-        LOG.warn("Storage directory " + sd + " contains no VERSION file. Skipping...");
-        continue;
-      }
-      sd.read(); // sets layoutVersion
-      minLayoutVersion = Math.min(minLayoutVersion, storage.getLayoutVersion());
-      maxLayoutVersion = Math.max(maxLayoutVersion, storage.getLayoutVersion());
-    }
-    
-    if (minLayoutVersion > maxLayoutVersion) {
-      throw new IOException("No storage directories contained VERSION information");
-    }
-    assert minLayoutVersion <= maxLayoutVersion;
-    
-    // If we have any storage directories with the new layout version
-    // (ie edits_<txnid>) then use the new inspector, which will ignore
-    // the old format dirs.
-    FSImageStorageInspector inspector;
-    if (LayoutVersion.supports(Feature.TXID_BASED_LAYOUT, minLayoutVersion)) {
-      inspector = new FSImageTransactionalStorageInspector();
-      if (!LayoutVersion.supports(Feature.TXID_BASED_LAYOUT, maxLayoutVersion)) {
-        LOG.warn("Ignoring one or more storage directories with old layouts");
-      }
-    } else {
-      inspector = new FSImageOldStorageInspector();
-    }
-
-    // Process each of the storage directories to find the pair of
-    // newest image file and edit file
-    for (Iterator<StorageDirectory> it = storage.dirIterator(); it.hasNext();) {
-      StorageDirectory sd = it.next();
-      inspector.inspectDirectory(sd);
-    }
-
-    return inspector;
-  }
-
   /**
    * Choose latest image from one of the directories,
    * load it and merge with the edits from that directory.
@@ -628,7 +583,7 @@ public class FSImage implements Closeabl
    * @throws IOException
    */
   boolean loadFSImage() throws IOException {
-    FSImageStorageInspector inspector = inspectStorageDirs();
+    FSImageStorageInspector inspector = storage.readAndInspectDirs();
     
     isUpgradeFinalized = inspector.isUpgradeFinalized();
     
@@ -854,6 +809,10 @@ public class FSImage implements Closeabl
     // TODO Double-check for regressions against HDFS-1505 and HDFS-1921.
 
     renameCheckpoint(txid);
+    
+    // Since we now have a new checkpoint, we can clean up some
+    // old edit logs and checkpoints.
+    storage.archiveOldStorage();
   }
 
   /**

Modified: hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/namenode/FSImageTransactionalStorageInspector.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/namenode/FSImageTransactionalStorageInspector.java?rev=1133183&r1=1133182&r2=1133183&view=diff
==============================================================================
--- hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/namenode/FSImageTransactionalStorageInspector.java
(original)
+++ hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/namenode/FSImageTransactionalStorageInspector.java
Tue Jun  7 22:30:25 2011
@@ -41,6 +41,7 @@ import org.apache.hadoop.hdfs.server.nam
 import org.apache.hadoop.hdfs.server.protocol.RemoteEditLog;
 import org.apache.hadoop.hdfs.server.protocol.RemoteEditLogManifest;
 
+import com.google.common.collect.ImmutableList;
 import com.google.common.collect.Lists;
 
 class FSImageTransactionalStorageInspector extends FSImageStorageInspector {
@@ -180,6 +181,14 @@ class FSImageTransactionalStorageInspect
     }
     return ret;
   }
+  
+  public List<FoundFSImage> getFoundImages() {
+    return ImmutableList.copyOf(foundImages);
+  }
+  
+  public List<FoundEditLog> getFoundEditLogs() {
+    return ImmutableList.copyOf(foundEditLogs);
+  }
 
   @Override
   public LoadPlan createLoadPlan() throws IOException {
@@ -442,6 +451,11 @@ class FSImageTransactionalStorageInspect
     public long getTxId() {
       return txId;
     }
+    
+    @Override
+    public String toString() {
+      return file.toString();
+    }
   }
   
   /**
@@ -513,6 +527,11 @@ class FSImageTransactionalStorageInspect
       }
       file = dst;
     }
+    
+    @Override
+    public String toString() {
+      return file.toString();
+    }
   }
 
   static class TransactionalLoadPlan extends LoadPlan {

Modified: hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/namenode/GetImageServlet.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/namenode/GetImageServlet.java?rev=1133183&r1=1133182&r2=1133183&view=diff
==============================================================================
--- hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/namenode/GetImageServlet.java
(original)
+++ hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/namenode/GetImageServlet.java
Tue Jun  7 22:30:25 2011
@@ -130,6 +130,10 @@ public class GetImageServlet extends Htt
                     }
               });
               nnImage.saveDigestAndRenameCheckpointImage(txid, downloadImageDigest);
+              
+              // Now that we have a new checkpoint, we might be able to
+              // remove some old ones.
+              nnImage.getStorage().archiveOldStorage();
             } finally {
               currentlyDownloadingCheckpoints.remove(txid);
             }

Modified: hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/namenode/NNStorage.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/namenode/NNStorage.java?rev=1133183&r1=1133182&r2=1133183&view=diff
==============================================================================
--- hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/namenode/NNStorage.java
(original)
+++ hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/namenode/NNStorage.java
Tue Jun  7 22:30:25 2011
@@ -121,6 +121,8 @@ public class NNStorage extends Storage i
 
   private UpgradeManager upgradeManager = null;
   protected String blockpoolID = ""; // id of the block pool
+  
+  private final NNStorageArchivalManager archivalManager;
 
   /**
    * flag that controls if we try to restore failed storages
@@ -158,18 +160,8 @@ public class NNStorage extends Storage i
     storageDirs = new CopyOnWriteArrayList<StorageDirectory>();
     
     setStorageDirectories(imageDirs, editsDirs);
-  }
-
-  /**
-   * Construct the NNStorage.
-   * @param storageInfo storage information
-   * @param bpid block pool Id
-   */
-  public NNStorage(StorageInfo storageInfo, String bpid) {
-    super(NodeType.NAME_NODE, storageInfo);
-
-    storageDirs = new CopyOnWriteArrayList<StorageDirectory>();
-    this.blockpoolID = bpid;
+    
+    archivalManager = new NNStorageArchivalManager(conf, this);
   }
 
   @Override // Storage
@@ -542,6 +534,19 @@ public class NNStorage extends Storage i
   }
 
   /**
+   * Archive any files in the storage directories that are no longer
+   * necessary.
+   */
+  public void archiveOldStorage() {
+    try {
+      archivalManager.archiveOldStorage();
+    } catch (Exception e) {
+      LOG.warn("Unable to archive old storage", e);
+    }
+  }
+
+
+  /**
    * Generate new namespaceID.
    *
    * namespaceID is a persistent attribute of the namespace.
@@ -942,4 +947,67 @@ public class NNStorage extends Storage i
   public String getBlockPoolID() {
     return blockpoolID;
   }
+
+  /**
+   * Iterate over all current storage directories, inspecting them
+   * with the given inspector.
+   */
+  void inspectStorageDirs(FSImageStorageInspector inspector)
+      throws IOException {
+
+    // Process each of the storage directories to find the pair of
+    // newest image file and edit file
+    for (Iterator<StorageDirectory> it = dirIterator(); it.hasNext();) {
+      StorageDirectory sd = it.next();
+      inspector.inspectDirectory(sd);
+    }
+  }
+
+  /**
+   * Iterate over all of the storage dirs, reading their contents to determine
+   * their layout versions. Returns an FSImageStorageInspector which has
+   * inspected each directory.
+   * 
+   * <b>Note:</b> this can mutate the storage info fields (ctime, version, etc).
+   * @throws IOException if no valid storage dirs are found
+   */
+  FSImageStorageInspector readAndInspectDirs()
+      throws IOException {
+    int minLayoutVersion = Integer.MAX_VALUE; // the newest
+    int maxLayoutVersion = Integer.MIN_VALUE; // the oldest
+    
+    // First determine what range of layout versions we're going to inspect
+    for (Iterator<StorageDirectory> it = dirIterator();
+         it.hasNext();) {
+      StorageDirectory sd = it.next();
+      if (!sd.getVersionFile().exists()) {
+        FSImage.LOG.warn("Storage directory " + sd + " contains no VERSION file. Skipping...");
+        continue;
+      }
+      sd.read(); // sets layoutVersion
+      minLayoutVersion = Math.min(minLayoutVersion, getLayoutVersion());
+      maxLayoutVersion = Math.max(maxLayoutVersion, getLayoutVersion());
+    }
+    
+    if (minLayoutVersion > maxLayoutVersion) {
+      throw new IOException("No storage directories contained VERSION information");
+    }
+    assert minLayoutVersion <= maxLayoutVersion;
+    
+    // If we have any storage directories with the new layout version
+    // (ie edits_<txnid>) then use the new inspector, which will ignore
+    // the old format dirs.
+    FSImageStorageInspector inspector;
+    if (LayoutVersion.supports(Feature.TXID_BASED_LAYOUT, minLayoutVersion)) {
+      inspector = new FSImageTransactionalStorageInspector();
+      if (!LayoutVersion.supports(Feature.TXID_BASED_LAYOUT, maxLayoutVersion)) {
+        FSImage.LOG.warn("Ignoring one or more storage directories with old layouts");
+      }
+    } else {
+      inspector = new FSImageOldStorageInspector();
+    }
+    
+    inspectStorageDirs(inspector);
+    return inspector;
+  }
 }

Added: hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/namenode/NNStorageArchivalManager.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/namenode/NNStorageArchivalManager.java?rev=1133183&view=auto
==============================================================================
--- hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/namenode/NNStorageArchivalManager.java
(added)
+++ hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/namenode/NNStorageArchivalManager.java
Tue Jun  7 22:30:25 2011
@@ -0,0 +1,145 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.namenode;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.List;
+import java.util.TreeSet;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.hdfs.server.namenode.FSImageTransactionalStorageInspector.FoundEditLog;
+import org.apache.hadoop.hdfs.server.namenode.FSImageTransactionalStorageInspector.FoundFSImage;
+import org.apache.hadoop.hdfs.util.MD5FileUtils;
+
+import com.google.common.collect.Lists;
+import com.google.common.collect.Sets;
+
+/**
+ * The NNStorageArchivalManager is responsible for inspecting the storage
+ * directories of the NN and enforcing a retention policy on checkpoints
+ * and edit logs.
+ * 
+ * It delegates the actual removal of files to a {@link #StorageArchiver}
+ * implementation, which might delete the files or instead copy them to
+ * a filer or HDFS for later analysis.
+ */
+public class NNStorageArchivalManager {
+  
+  private final int numCheckpointsToRetain;
+  private static final Log LOG = LogFactory.getLog(NNStorageArchivalManager.class);
+  private final NNStorage storage;
+  private final StorageArchiver archiver;
+  
+  public NNStorageArchivalManager(
+      Configuration conf,
+      NNStorage storage,
+      StorageArchiver archiver) {
+    this.numCheckpointsToRetain = conf.getInt(
+        DFSConfigKeys.DFS_NAMENODE_NUM_CHECKPOINTS_RETAINED_KEY,
+        DFSConfigKeys.DFS_NAMENODE_NUM_CHECKPOINTS_RETAINED_DEFAULT);
+    this.storage = storage;
+    this.archiver = archiver;
+  }
+  
+  public NNStorageArchivalManager(Configuration conf, NNStorage storage) {
+    this(conf, storage, new DeletionStorageArchiver());
+  }
+
+  public void archiveOldStorage() throws IOException {
+    FSImageTransactionalStorageInspector inspector =
+      new FSImageTransactionalStorageInspector();
+    storage.inspectStorageDirs(inspector);
+
+    long minImageTxId = getImageTxIdToRetain(inspector);
+    archiveCheckpointsOlderThan(inspector, minImageTxId);
+    archiveLogsOlderThan(inspector, minImageTxId);
+  }
+  
+  private void archiveLogsOlderThan(
+      FSImageTransactionalStorageInspector inspector,
+      long minImageTxId) {
+    for (FoundEditLog log : inspector.getFoundEditLogs()) {
+      if (log.getStartTxId() < minImageTxId) {
+        LOG.info("Purging old edit log " + log);
+        archiver.archiveLog(log);
+      }
+    }
+  }
+
+  private void archiveCheckpointsOlderThan(
+      FSImageTransactionalStorageInspector inspector,
+      long minTxId) {
+    for (FoundFSImage image : inspector.getFoundImages()) {
+      if (image.getTxId() < minTxId) {
+        LOG.info("Purging old image " + image);
+        archiver.archiveImage(image);
+      }
+    }
+  }
+
+  /**
+   * @param inspector inspector that has already inspected all storage dirs
+   * @return the transaction ID corresponding to the oldest checkpoint
+   * that should be retained. 
+   */
+  private long getImageTxIdToRetain(FSImageTransactionalStorageInspector inspector) {
+      
+    List<FoundFSImage> images = inspector.getFoundImages();
+    TreeSet<Long> imageTxIds = Sets.newTreeSet();
+    for (FoundFSImage image : images) {
+      imageTxIds.add(image.getTxId());
+    }
+    
+    List<Long> imageTxIdsList = Lists.newArrayList(imageTxIds);
+    if (imageTxIdsList.isEmpty()) {
+      return 0;
+    }
+    
+    Collections.reverse(imageTxIdsList);
+    int toRetain = Math.min(numCheckpointsToRetain, imageTxIdsList.size());    
+    long minTxId = imageTxIdsList.get(toRetain - 1);
+    LOG.info("Going to retain " + toRetain + " images with txid >= " +
+        minTxId);
+    return minTxId;
+  }
+  
+  /**
+   * Interface responsible for archiving old checkpoints and edit logs.
+   */
+  static interface StorageArchiver {
+    void archiveLog(FoundEditLog log);
+    void archiveImage(FoundFSImage image);
+  }
+  
+  static class DeletionStorageArchiver implements StorageArchiver {
+    @Override
+    public void archiveLog(FoundEditLog log) {
+      log.getFile().delete();
+    }
+
+    @Override
+    public void archiveImage(FoundFSImage image) {
+      image.getFile().delete();
+      MD5FileUtils.getDigestFileForFile(image.getFile()).delete();
+    }
+  }
+}

Modified: hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java?rev=1133183&r1=1133182&r2=1133183&view=diff
==============================================================================
--- hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java
(original)
+++ hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java
Tue Jun  7 22:30:25 2011
@@ -43,6 +43,7 @@ import org.apache.hadoop.hdfs.server.com
 import org.apache.hadoop.hdfs.server.common.JspHelper;
 import org.apache.hadoop.hdfs.server.common.Storage.StorageDirectory;
 import org.apache.hadoop.hdfs.server.common.Storage.StorageState;
+import org.apache.hadoop.hdfs.server.namenode.NNStorageArchivalManager.StorageArchiver;
 import org.apache.hadoop.hdfs.server.protocol.NamenodeProtocol;
 import org.apache.hadoop.hdfs.server.protocol.RemoteEditLog;
 import org.apache.hadoop.hdfs.server.protocol.RemoteEditLogManifest;
@@ -492,6 +493,10 @@ public class SecondaryNameNode implement
     LOG.warn("Checkpoint done. New Image Size: " 
              + checkpointImage.getStorage().getFsImageName(txid).length());
     
+    // Since we've successfully checkpointed, we can remove some old
+    // image files
+    checkpointImage.getStorage().archiveOldStorage();
+    
     return loadImage;
   }
 

Modified: hadoop/hdfs/branches/HDFS-1073/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestCheckpoint.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1073/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestCheckpoint.java?rev=1133183&r1=1133182&r2=1133183&view=diff
==============================================================================
--- hadoop/hdfs/branches/HDFS-1073/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestCheckpoint.java
(original)
+++ hadoop/hdfs/branches/HDFS-1073/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestCheckpoint.java
Tue Jun  7 22:30:25 2011
@@ -1117,6 +1117,9 @@ public class TestCheckpoint extends Test
       // a checkpoint with a lower txid finished most recently)
       NNStorage storage = cluster.getNameNode().getFSImage().getStorage();
       assertEquals(4, storage.getMostRecentCheckpointTxId());
+
+      // Should have accepted both checkpoints
+      assertNNHasCheckpoints(cluster, ImmutableList.of(2,4));
       
       // Now have second one checkpoint one more time just to make sure that
       // the NN isn't left in a broken state
@@ -1134,9 +1137,10 @@ public class TestCheckpoint extends Test
     
     // Validate invariant that files named the same are the same.
     assertParallelFilesInvariant(cluster, ImmutableList.of(secondary1, secondary2));
-    // Validate that the NN received checkpoints at expected txids
-    // (i.e that both checkpoints went through)
-    assertNNHasCheckpoints(cluster, ImmutableList.of(2,4,6));
+
+    // NN should have removed the checkpoint at txid 2 at this point, but has
+    // one at txid 6
+    assertNNHasCheckpoints(cluster, ImmutableList.of(4,6));
   }
   
   
@@ -1226,6 +1230,9 @@ public class TestCheckpoint extends Test
       secondary2.doCheckpoint();
       assertEquals(6, storage.getMostRecentCheckpointTxId());
       
+      // Should have accepted both checkpoints
+      assertNNHasCheckpoints(cluster, ImmutableList.of(4,6));
+
       // Let the first one also go again on its own to make sure it can
       // continue at next checkpoint
       secondary1.setNameNode(origNN);
@@ -1245,7 +1252,7 @@ public class TestCheckpoint extends Test
     assertParallelFilesInvariant(cluster, ImmutableList.of(secondary1, secondary2));
     // Validate that the NN received checkpoints at expected txids
     // (i.e that both checkpoints went through)
-    assertNNHasCheckpoints(cluster, ImmutableList.of(4,6,8));
+    assertNNHasCheckpoints(cluster, ImmutableList.of(6,8));
   }
 
 

Modified: hadoop/hdfs/branches/HDFS-1073/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestFSImageStorageInspector.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1073/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestFSImageStorageInspector.java?rev=1133183&r1=1133182&r2=1133183&view=diff
==============================================================================
--- hadoop/hdfs/branches/HDFS-1073/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestFSImageStorageInspector.java
(original)
+++ hadoop/hdfs/branches/HDFS-1073/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestFSImageStorageInspector.java
Tue Jun  7 22:30:25 2011
@@ -358,7 +358,7 @@ public class TestFSImageStorageInspector
    * @param previousExists should we mock that the previous/ dir exists?
    * @param fileNames the names of files contained in current/
    */
-  private StorageDirectory mockDirectory(
+  static StorageDirectory mockDirectory(
       StorageDirType type,
       boolean previousExists,
       String...  fileNames) {

Added: hadoop/hdfs/branches/HDFS-1073/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestNNStorageArchivalManager.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1073/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestNNStorageArchivalManager.java?rev=1133183&view=auto
==============================================================================
--- hadoop/hdfs/branches/HDFS-1073/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestNNStorageArchivalManager.java
(added)
+++ hadoop/hdfs/branches/HDFS-1073/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestNNStorageArchivalManager.java
Tue Jun  7 22:30:25 2011
@@ -0,0 +1,219 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.namenode;
+
+import java.io.IOException;
+import java.util.Set;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdfs.server.common.Storage.StorageDirectory;
+import org.apache.hadoop.hdfs.server.namenode.FSImageTransactionalStorageInspector.FoundEditLog;
+import org.apache.hadoop.hdfs.server.namenode.FSImageTransactionalStorageInspector.FoundFSImage;
+import org.apache.hadoop.hdfs.server.namenode.NNStorage.NameNodeDirType;
+import org.apache.hadoop.hdfs.server.namenode.NNStorageArchivalManager.StorageArchiver;
+import org.junit.Assert;
+import org.junit.Test;
+import org.mockito.ArgumentCaptor;
+import org.mockito.Mockito;
+import org.mockito.invocation.InvocationOnMock;
+import org.mockito.stubbing.Answer;
+
+import com.google.common.base.Joiner;
+import com.google.common.collect.Sets;
+
+
+public class TestNNStorageArchivalManager {
+  /**
+   * Test the "easy case" where we have more images in the
+   * directory than we need to keep. Should archive the
+   * old ones.
+   */
+  @Test
+  public void testArchiveEasyCase() throws IOException {
+    TestCaseDescription tc = new TestCaseDescription();
+    tc.addImage("/foo1/current/fsimage_100", true);
+    tc.addImage("/foo1/current/fsimage_200", true);
+    tc.addImage("/foo1/current/fsimage_300", false);
+    tc.addImage("/foo1/current/fsimage_400", false);
+    tc.addLog("/foo1/current/edits_101-200", true);
+    tc.addLog("/foo1/current/edits_201-300", true);
+    tc.addLog("/foo1/current/edits_301-400", false);
+    tc.addLog("/foo1/current/edits_inprogress_401", false);
+    
+    // Test that other files don't get archived
+    tc.addLog("/foo1/current/VERSION", false);
+    runTest(tc);
+  }
+  
+  /**
+   * Same as above, but across multiple directories
+   */
+  @Test
+  public void testArchiveMultipleDirs() throws IOException {
+    TestCaseDescription tc = new TestCaseDescription();
+    tc.addImage("/foo1/current/fsimage_100", true);
+    tc.addImage("/foo1/current/fsimage_200", true);
+    tc.addImage("/foo2/current/fsimage_200", true);
+    tc.addImage("/foo1/current/fsimage_300", false);
+    tc.addImage("/foo1/current/fsimage_400", false);
+    tc.addLog("/foo1/current/edits_101-200", true);
+    tc.addLog("/foo1/current/edits_201-300", true);
+    tc.addLog("/foo2/current/edits_201-300", true);
+    tc.addLog("/foo1/current/edits_301-400", false);
+    tc.addLog("/foo2/current/edits_301-400", false);
+    tc.addLog("/foo1/current/edits_inprogress_401", false);
+    runTest(tc);
+  }
+  
+  /**
+   * Test that if we have fewer fsimages than the configured
+   * retention, we don't archive any of them
+   */
+  @Test
+  public void testArchiveLessThanRetention() throws IOException {
+    TestCaseDescription tc = new TestCaseDescription();
+    tc.addImage("/foo1/current/fsimage_100", false);
+    tc.addLog("/foo1/current/edits_101-200", false);
+    tc.addLog("/foo1/current/edits_201-300", false);
+    tc.addLog("/foo1/current/edits_301-400", false);
+    tc.addLog("/foo1/current/edits_inprogress_401", false);
+    runTest(tc);
+  }
+
+  /**
+   * Check for edge case with no logs present at all.
+   */
+  @Test
+  public void testNoLogs() throws IOException {
+    TestCaseDescription tc = new TestCaseDescription();
+    tc.addImage("/foo1/current/fsimage_100", true);
+    tc.addImage("/foo1/current/fsimage_200", true);
+    tc.addImage("/foo1/current/fsimage_300", false);
+    tc.addImage("/foo1/current/fsimage_400", false);
+    runTest(tc);
+  }
+  
+  /**
+   * Check for edge case with no logs or images present at all.
+   */
+  @Test
+  public void testEmptyDir() throws IOException {
+    TestCaseDescription tc = new TestCaseDescription();
+    runTest(tc);
+  }
+
+  /**
+   * Test that old in-progress logs are properly archived
+   */
+  @Test
+  public void testOldInProgress() throws IOException {
+    TestCaseDescription tc = new TestCaseDescription();
+    tc.addImage("/foo1/current/fsimage_100", true);
+    tc.addImage("/foo1/current/fsimage_200", true);
+    tc.addImage("/foo1/current/fsimage_300", false);
+    tc.addImage("/foo1/current/fsimage_400", false);
+    tc.addLog("/foo1/current/edits_inprogress_101", true);
+    runTest(tc);
+  }
+    
+  private void runTest(TestCaseDescription tc) throws IOException {
+    Configuration conf = new Configuration();
+
+    StorageArchiver mockArchiver =
+      Mockito.mock(NNStorageArchivalManager.StorageArchiver.class);
+    ArgumentCaptor<FoundFSImage> imagesArchivedCaptor =
+      ArgumentCaptor.forClass(FoundFSImage.class);    
+    ArgumentCaptor<FoundEditLog> logsArchivedCaptor =
+      ArgumentCaptor.forClass(FoundEditLog.class);    
+
+    // Ask the manager to archive files we don't need any more
+    new NNStorageArchivalManager(conf, tc.mockStorage(), mockArchiver)
+      .archiveOldStorage();
+    
+    // Verify that it asked the archiver to remove the correct files
+    Mockito.verify(mockArchiver, Mockito.atLeast(0))
+      .archiveImage(imagesArchivedCaptor.capture());
+    Mockito.verify(mockArchiver, Mockito.atLeast(0))
+      .archiveLog(logsArchivedCaptor.capture());
+
+    // Check images
+    Set<String> archivedPaths = Sets.newHashSet();
+    for (FoundFSImage archived : imagesArchivedCaptor.getAllValues()) {
+      archivedPaths.add(archived.getFile().toString());
+    }    
+    Assert.assertEquals(Joiner.on(",").join(tc.expectedArchivedImages),
+        Joiner.on(",").join(archivedPaths));
+
+    // Check images
+    archivedPaths.clear();
+    for (FoundEditLog archived : logsArchivedCaptor.getAllValues()) {
+      archivedPaths.add(archived.getFile().toString());
+    }    
+    Assert.assertEquals(Joiner.on(",").join(tc.expectedArchivedLogs),
+        Joiner.on(",").join(archivedPaths));
+  }
+  
+  private static class TestCaseDescription {
+    private Set<String> files = Sets.newHashSet();
+    private Set<String> expectedArchivedLogs = Sets.newHashSet();
+    private Set<String> expectedArchivedImages = Sets.newHashSet();
+
+    void addLog(String path, boolean expectArchive) {
+      files.add(path);
+      if (expectArchive) {
+        expectedArchivedLogs.add(path);
+      }
+    }
+    
+    private String[] getPaths() {
+      return files.toArray(new String[0]);
+    }
+    
+    void addImage(String path, boolean expectArchive) {
+      files.add(path);
+      if (expectArchive) {
+        expectedArchivedImages.add(path);
+      }
+    }
+    
+    NNStorage mockStorage() throws IOException {
+      String[] paths = getPaths();
+      StorageDirectory mockDir = TestFSImageStorageInspector.mockDirectory(
+            NameNodeDirType.IMAGE_AND_EDITS, false, paths);
+      return mockStorageForDirs(mockDir);
+    }
+  }
+
+  private static NNStorage mockStorageForDirs(final StorageDirectory ... mockDirs)
+      throws IOException {
+    NNStorage mockStorage = Mockito.mock(NNStorage.class);
+    Mockito.doAnswer(new Answer<Void>() {
+      @Override
+      public Void answer(InvocationOnMock invocation) throws Throwable {
+        FSImageStorageInspector inspector =
+          (FSImageStorageInspector) invocation.getArguments()[0];
+        for (StorageDirectory sd : mockDirs) {
+          inspector.inspectDirectory(sd);
+        }
+        return null;
+      }
+    }).when(mockStorage).inspectStorageDirs(
+        Mockito.<FSImageStorageInspector>anyObject());
+    return mockStorage;
+  }
+}



Mime
View raw message