hadoop-hdfs-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From t...@apache.org
Subject svn commit: r1128032 - in /hadoop/hdfs/branches/HDFS-1073: ./ src/java/org/apache/hadoop/hdfs/server/namenode/ src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/
Date Thu, 26 May 2011 18:55:45 GMT
Author: todd
Date: Thu May 26 18:55:44 2011
New Revision: 1128032

URL: http://svn.apache.org/viewvc?rev=1128032&view=rev
Log:
HDFS-1991. Some refactoring of Secondary NameNode to be able to share more code with the BackupNode
or CheckpointNode. Contributed by Todd Lipcon.

Modified:
    hadoop/hdfs/branches/HDFS-1073/CHANGES.HDFS-1073.txt
    hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java
    hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java
    hadoop/hdfs/branches/HDFS-1073/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestCheckpoint.java

Modified: hadoop/hdfs/branches/HDFS-1073/CHANGES.HDFS-1073.txt
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1073/CHANGES.HDFS-1073.txt?rev=1128032&r1=1128031&r2=1128032&view=diff
==============================================================================
--- hadoop/hdfs/branches/HDFS-1073/CHANGES.HDFS-1073.txt (original)
+++ hadoop/hdfs/branches/HDFS-1073/CHANGES.HDFS-1073.txt Thu May 26 18:55:44 2011
@@ -36,3 +36,5 @@ HDFS-1987. Re-enable TestCheckpoint.test
            running previously. (todd)
 HDFS-1993. TestCheckpoint needs to clean up between cases (todd)
 HDFS-1992. Remove vestiges of NNStorageListener. (todd)
+HDFS-1991. Some refactoring of Secondary NameNode to be able to share more
+           code with the BackupNode or CheckpointNode. (todd)

Modified: hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java?rev=1128032&r1=1128031&r2=1128032&view=diff
==============================================================================
--- hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java
(original)
+++ hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java
Thu May 26 18:55:44 2011
@@ -641,10 +641,9 @@ public class FSImage implements Closeabl
     // storage directory to read properties from
     sdForProperties.read();
     File imageFile = loadPlan.getImageFile();
-    MD5Hash expectedMD5 = MD5FileUtils.readStoredMd5ForFile(imageFile);
 
     try {
-      loadFSImage(imageFile, expectedMD5);
+      loadFSImage(imageFile);
     } catch (IOException ioe) {
       throw new IOException("Failed to load image from " + loadPlan.getImageFile(), ioe);
     }
@@ -692,6 +691,15 @@ public class FSImage implements Closeabl
 
 
   /**
+   * Load the image namespace from the given image file, verifying
+   * it against the MD5 sum stored in its associated .md5 file.
+   */
+  void loadFSImage(File imageFile) throws IOException {
+    MD5Hash expectedMD5 = MD5FileUtils.readStoredMd5ForFile(imageFile);
+    loadFSImage(imageFile, expectedMD5);
+  }
+  
+  /**
    * Load in the filesystem image from file. It's a big list of
    * filenames and blocks.  Return whether we should
    * "re-save" and consolidate the edit-logs

Modified: hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java?rev=1128032&r1=1128031&r2=1128032&view=diff
==============================================================================
--- hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java
(original)
+++ hadoop/hdfs/branches/HDFS-1073/src/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java
Thu May 26 18:55:44 2011
@@ -43,11 +43,9 @@ import org.apache.hadoop.hdfs.server.com
 import org.apache.hadoop.hdfs.server.common.JspHelper;
 import org.apache.hadoop.hdfs.server.common.Storage.StorageDirectory;
 import org.apache.hadoop.hdfs.server.common.Storage.StorageState;
-import org.apache.hadoop.hdfs.server.namenode.NNStorage.NameNodeDirType;
 import org.apache.hadoop.hdfs.server.protocol.NamenodeProtocol;
 import org.apache.hadoop.hdfs.server.protocol.RemoteEditLog;
 import org.apache.hadoop.hdfs.server.protocol.RemoteEditLogManifest;
-import org.apache.hadoop.hdfs.util.MD5FileUtils;
 import org.apache.hadoop.http.HttpServer;
 import org.apache.hadoop.io.MD5Hash;
 import org.apache.hadoop.ipc.RPC;
@@ -104,7 +102,6 @@ public class SecondaryNameNode implement
   private int imagePort;
   private String infoBindAddress;
 
-  private FSNamesystem namesystem;
   private Collection<URI> checkpointDirs;
   private Collection<URI> checkpointEditsDirs;
   private long checkpointPeriod;    // in seconds
@@ -345,7 +342,7 @@ public class SecondaryNameNode implement
    * @return true if a new image has been downloaded and needs to be loaded
    * @throws IOException
    */
-  private static boolean downloadCheckpointFiles(
+  static boolean downloadCheckpointFiles(
       final String nnHostPort,
       final FSImage dstImage,
       final CheckpointSignature sig,
@@ -455,8 +452,8 @@ public class SecondaryNameNode implement
 
     boolean loadImage = downloadCheckpointFiles(
         fsName, checkpointImage, sig, manifest);   // Fetch fsimage and edits
-    doMerge(sig, manifest, loadImage);                   // Do the merge
-  
+    doMerge(conf, sig, manifest, loadImage, checkpointImage);
+    
     //
     // Upload the new image into the NameNode. Then tell the Namenode
     // to make this new uploaded image as the most current image.
@@ -483,21 +480,6 @@ public class SecondaryNameNode implement
   }
 
   /**
-   * Merge downloaded image and edits and write the new image into
-   * current storage directory.
-   * @param manifest 
-   */
-  private void doMerge(
-      CheckpointSignature sig, RemoteEditLogManifest manifest, boolean loadImage)
-  throws IOException {
-    if (loadImage) {
-      namesystem = new FSNamesystem(checkpointImage, conf);
-    }
-    assert namesystem.dir.fsImage == checkpointImage;
-    checkpointImage.doMerge(sig, manifest, loadImage);
-  }
-
-  /**
    * @param argv The parameters passed to this program.
    * @exception Exception if the filesystem does not exist.
    * @return 0 on success, non zero on error.
@@ -701,57 +683,42 @@ public class SecondaryNameNode implement
         storage.moveLastCheckpoint(sd);
       }
     }
-
-    /**
-     * Merge image and edits, and verify consistency with the signature.
-     * @param manifest 
-     */
-    private void doMerge(CheckpointSignature sig,
-                         RemoteEditLogManifest manifest,
-                         boolean loadImage)
-    throws IOException {
-      StorageDirectory sdName = null;
-      StorageDirectory sdEdits = null;
-      Iterator<StorageDirectory> it = null;
-      if (loadImage) {
-        it = getStorage().dirIterator(NameNodeDirType.IMAGE);
-        if (it.hasNext())
-          sdName = it.next();
-        if (sdName == null) {
-          throw new IOException("Could not locate checkpoint fsimage");
-        }
-      }
-      it = getStorage().dirIterator(NameNodeDirType.EDITS);
-      if (it.hasNext())
-        sdEdits = it.next();
-      if (sdEdits == null)
-        throw new IOException("Could not locate checkpoint edits");
-      
-      this.getStorage().setStorageInfo(sig);
-      if (loadImage) {
-        File file = getStorage().findImageFile(sig.mostRecentCheckpointTxId);
-        MD5Hash hash = MD5FileUtils.readStoredMd5ForFile(file);
-        LOG.debug("2NN loading image from " + file);
-        loadFSImage(file, hash);
-      }
-      List<File> editsFiles = Lists.newArrayList();
-      for (RemoteEditLog log : manifest.getLogs()) {
-        File f = getStorage().findFinalizedEditsFile(
-            log.getStartTxId(), log.getEndTxId());
-        editsFiles.add(f);
-      }
-      LOG.info("SecondaryNameNode about to load edits from " +
-          editsFiles.size() + " file(s).");
-      loadEdits(editsFiles);
-      
-      storage.setClusterID(sig.getClusterID());
-      storage.setBlockPoolID(sig.getBlockpoolID());
-      
-      // TODO I think the below validation might be too strict -- it would disallow
-      // multiple concurrent checkpointers with big fsimages. Test this.
-      sig.validateStorageInfo(this);
-      saveFSImageInAllDirs(editLog.getLastWrittenTxId());
-      getStorage().writeAll();
-    }
+  }
+  
+  static void doMerge(Configuration conf, 
+      CheckpointSignature sig, RemoteEditLogManifest manifest,
+      boolean loadImage, FSImage dstImage) throws IOException {   
+    NNStorage dstStorage = dstImage.getStorage();
+    
+    dstStorage.setStorageInfo(sig);
+    if (loadImage) {
+      // TODO: dstImage.namesystem.close(); ??
+      dstImage.namesystem = new FSNamesystem(dstImage, conf);
+      dstImage.editLog = new FSEditLog(dstStorage);
+
+      File file = dstStorage.findImageFile(sig.mostRecentCheckpointTxId);
+      LOG.debug("2NN loading image from " + file);
+      dstImage.loadFSImage(file);
+    }
+    List<File> editsFiles = Lists.newArrayList();
+    for (RemoteEditLog log : manifest.getLogs()) {
+      File f = dstStorage.findFinalizedEditsFile(
+          log.getStartTxId(), log.getEndTxId());
+      editsFiles.add(f);
+    }
+    LOG.info("SecondaryNameNode about to load edits from " +
+        editsFiles.size() + " file(s).");
+    dstImage.loadEdits(editsFiles);
+    
+    // TODO: why do we need the following two lines? We shouldn't have even
+    // been able to download an image from a NN that had a different
+    // cluster ID or blockpool ID! this should only be done for the
+    // very first checkpoint.
+    dstStorage.setClusterID(sig.getClusterID());
+    dstStorage.setBlockPoolID(sig.getBlockpoolID());
+    
+    sig.validateStorageInfo(dstImage);
+    dstImage.saveFSImageInAllDirs(dstImage.getEditLog().getLastWrittenTxId());
+    dstStorage.writeAll();
   }
 }

Modified: hadoop/hdfs/branches/HDFS-1073/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestCheckpoint.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/HDFS-1073/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestCheckpoint.java?rev=1128032&r1=1128031&r2=1128032&view=diff
==============================================================================
--- hadoop/hdfs/branches/HDFS-1073/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestCheckpoint.java
(original)
+++ hadoop/hdfs/branches/HDFS-1073/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/TestCheckpoint.java
Thu May 26 18:55:44 2011
@@ -463,7 +463,7 @@ public class TestCheckpoint extends Test
 
       // start new instance of secondary and verify that 
       // a new rollEditLog suceedes inspite of the fact that 
-      // edits.new already exists.
+      // edits.new already exists. TODO update this comment!
       //
       secondary = startSecondaryNameNode(conf);
       secondary.doCheckpoint();  // this should work correctly
@@ -988,7 +988,8 @@ public class TestCheckpoint extends Test
       File secondaryFsImageAfter = new File(secondaryCurrent,
           "fsimage_" + (expectedTxIdToDownload + 2));
       
-      assertFalse("Secondary should start with empty current/ dir",
+      assertFalse("Secondary should start with empty current/ dir " +
+          "but " + secondaryFsImageBefore + " exists",
           secondaryFsImageBefore.exists());
 
       assertTrue("Secondary should have loaded an image",



Mime
View raw message