hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From s..@apache.org
Subject svn commit: r775338 - in /hadoop/core/trunk: CHANGES.txt src/hdfs/org/apache/hadoop/hdfs/server/common/Storage.java src/hdfs/org/apache/hadoop/hdfs/server/namenode/FSImage.java
Date Fri, 15 May 2009 21:03:01 GMT
Author: shv
Date: Fri May 15 21:03:00 2009
New Revision: 775338

URL: http://svn.apache.org/viewvc?rev=775338&view=rev
Log:
HADOOP-5314. Prevent unnecessary saving of the file system image during name-node startup.
Contributed by Jakob Homan.

Modified:
    hadoop/core/trunk/CHANGES.txt
    hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/server/common/Storage.java
    hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/server/namenode/FSImage.java

Modified: hadoop/core/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/CHANGES.txt?rev=775338&r1=775337&r2=775338&view=diff
==============================================================================
--- hadoop/core/trunk/CHANGES.txt (original)
+++ hadoop/core/trunk/CHANGES.txt Fri May 15 21:03:00 2009
@@ -599,6 +599,9 @@
     HADOOP-5845. Build successful despite test failure on test-core target.
     (sharad)
 
+    HADOOP-5314. Prevent unnecessary saving of the file system image during 
+    name-node startup. (Jakob Homan via shv)
+
 Release 0.20.1 - Unreleased
 
   INCOMPATIBLE CHANGES

Modified: hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/server/common/Storage.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/server/common/Storage.java?rev=775338&r1=775337&r2=775338&view=diff
==============================================================================
--- hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/server/common/Storage.java (original)
+++ hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/server/common/Storage.java Fri May 15
21:03:00 2009
@@ -159,7 +159,7 @@
   
   /**
    * Return default iterator
-   * This iterator returns all entires of storageDirs
+   * This iterator returns all entries in storageDirs
    */
   public Iterator<StorageDirectory> dirIterator() {
     return dirIterator(null);
@@ -167,7 +167,7 @@
   
   /**
    * Return iterator based on Storage Directory Type
-   * This iterator selects entires of storageDirs of type dirType and returns
+   * This iterator selects entries in storageDirs of type dirType and returns
    * them via the Iterator
    */
   public Iterator<StorageDirectory> dirIterator(StorageDirType dirType) {

Modified: hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/server/namenode/FSImage.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/server/namenode/FSImage.java?rev=775338&r1=775337&r2=775338&view=diff
==============================================================================
--- hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/server/namenode/FSImage.java (original)
+++ hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/server/namenode/FSImage.java Fri May
15 21:03:00 2009
@@ -28,42 +28,41 @@
 import java.io.FileOutputStream;
 import java.io.IOException;
 import java.io.RandomAccessFile;
+import java.nio.ByteBuffer;
 import java.text.SimpleDateFormat;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Date;
+import java.util.HashMap;
+import java.util.HashSet;
 import java.util.Iterator;
 import java.util.List;
+import java.util.Map;
 import java.util.Properties;
 import java.util.Random;
-import java.util.Map;
-import java.util.HashMap;
-import java.lang.Math;
-import java.nio.ByteBuffer;
+import java.util.Set;
 
+import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.permission.PermissionStatus;
 import org.apache.hadoop.fs.permission.FsPermission;
-import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.permission.PermissionStatus;
 import org.apache.hadoop.hdfs.protocol.Block;
 import org.apache.hadoop.hdfs.protocol.DatanodeID;
 import org.apache.hadoop.hdfs.protocol.FSConstants;
+import org.apache.hadoop.hdfs.server.common.InconsistentFSStateException;
+import org.apache.hadoop.hdfs.server.common.Storage;
+import org.apache.hadoop.hdfs.server.common.StorageInfo;
+import org.apache.hadoop.hdfs.server.common.UpgradeManager;
 import org.apache.hadoop.hdfs.server.common.HdfsConstants.NamenodeRole;
 import org.apache.hadoop.hdfs.server.common.HdfsConstants.NodeType;
 import org.apache.hadoop.hdfs.server.common.HdfsConstants.StartupOption;
-import org.apache.hadoop.io.DeprecatedUTF8;
-import org.apache.hadoop.io.Writable;
-import org.apache.hadoop.hdfs.server.namenode.NameNode;
 import org.apache.hadoop.hdfs.server.namenode.BlocksMap.BlockInfo;
-import org.apache.hadoop.hdfs.server.namenode.EditLogFileInputStream;
 import org.apache.hadoop.hdfs.server.protocol.CheckpointCommand;
 import org.apache.hadoop.hdfs.server.protocol.NamenodeCommand;
 import org.apache.hadoop.hdfs.server.protocol.NamenodeProtocol;
 import org.apache.hadoop.hdfs.server.protocol.NamenodeRegistration;
-import org.apache.hadoop.hdfs.server.common.InconsistentFSStateException;
-import org.apache.hadoop.hdfs.server.common.Storage;
-import org.apache.hadoop.hdfs.server.common.StorageInfo;
-import org.apache.hadoop.hdfs.server.common.UpgradeManager;
+import org.apache.hadoop.io.DeprecatedUTF8;
+import org.apache.hadoop.io.Writable;
 
 /**
  * FSImage handles checkpointing and logging of the namespace edits.
@@ -362,7 +361,7 @@
         case NON_EXISTENT:
           // name-node fails if any of the configured storage dirs are missing
           throw new InconsistentFSStateException(sd.getRoot(),
-                                                 "storage directory does not exist or is
not accessible.");
+                      "storage directory does not exist or is not accessible.");
         case NOT_FORMATTED:
           break;
         case NORMAL:
@@ -396,9 +395,9 @@
           && layoutVersion < LAST_PRE_UPGRADE_LAYOUT_VERSION
           && layoutVersion != FSConstants.LAYOUT_VERSION)
         throw new IOException(
-                          "\nFile system image contains an old layout version " + layoutVersion
-                          + ".\nAn upgrade to version " + FSConstants.LAYOUT_VERSION
-                          + " is required.\nPlease restart NameNode with -upgrade option.");
+           "\nFile system image contains an old layout version " + layoutVersion
+         + ".\nAn upgrade to version " + FSConstants.LAYOUT_VERSION
+         + " is required.\nPlease restart NameNode with -upgrade option.");
     // check whether distributed upgrade is reguired and/or should be continued
     verifyDistributedUpgradeProgress(startOpt);
 
@@ -628,6 +627,13 @@
     this.checkpointTime = readCheckpointTime(sd);
   }
 
+  /**
+   * Determine the checkpoint time of the specified StorageDirectory
+   * 
+   * @param sd StorageDirectory to check
+   * @return If file exists and can be read, last checkpoint time. If not, 0L.
+   * @throws IOException On errors processing file pointed to by sd
+   */
   long readCheckpointTime(StorageDirectory sd) throws IOException {
     File timeFile = getImageFile(sd, NameNodeFile.TIME);
     long timeStamp = 0L;
@@ -844,51 +850,65 @@
    * @throws IOException
    */
   boolean loadFSImage() throws IOException {
-    // Now check all curFiles and see which is the newest
     long latestNameCheckpointTime = Long.MIN_VALUE;
     long latestEditsCheckpointTime = Long.MIN_VALUE;
-    StorageDirectory latestNameSD = null;
-    StorageDirectory latestEditsSD = null;
     boolean needToSave = false;
     isUpgradeFinalized = true;
+    
+    StorageDirectory latestNameSD = null;
+    StorageDirectory latestEditsSD = null;
+    
     Collection<String> imageDirs = new ArrayList<String>();
     Collection<String> editsDirs = new ArrayList<String>();
+    
+    // Set to determine if all of storageDirectories share the same checkpoint
+    Set<Long> checkpointTimes = new HashSet<Long>();
+
+    // Process each of the storage directories to find the pair of
+    // newest image file and edit file
     for (Iterator<StorageDirectory> it = dirIterator(); it.hasNext();) {
       StorageDirectory sd = it.next();
+
+      // Was the file just formatted?
       if (!sd.getVersionFile().exists()) {
         needToSave |= true;
-        continue; // some of them might have just been formatted
+        continue;
       }
-      boolean imageExists = false, editsExists = false;
+      
+      boolean imageExists = false;
+      boolean editsExists = false;
+      
+      // Determine if sd is image, edits or both
       if (sd.getStorageDirType().isOfType(NameNodeDirType.IMAGE)) {
         imageExists = getImageFile(sd, NameNodeFile.IMAGE).exists();
         imageDirs.add(sd.getRoot().getCanonicalPath());
       }
+      
       if (sd.getStorageDirType().isOfType(NameNodeDirType.EDITS)) {
         editsExists = getImageFile(sd, NameNodeFile.EDITS).exists();
         editsDirs.add(sd.getRoot().getCanonicalPath());
       }
       
       checkpointTime = readCheckpointTime(sd);
-      if ((checkpointTime != Long.MIN_VALUE) && 
-          ((checkpointTime != latestNameCheckpointTime) || 
-           (checkpointTime != latestEditsCheckpointTime))) {
-        // Force saving of new image if checkpoint time
-        // is not same in all of the storage directories.
-        needToSave |= true;
-      }
+
+      checkpointTimes.add(checkpointTime);
+      
       if (sd.getStorageDirType().isOfType(NameNodeDirType.IMAGE) && 
          (latestNameCheckpointTime < checkpointTime) && imageExists) {
         latestNameCheckpointTime = checkpointTime;
         latestNameSD = sd;
       }
+      
       if (sd.getStorageDirType().isOfType(NameNodeDirType.EDITS) && 
            (latestEditsCheckpointTime < checkpointTime) && editsExists) {
         latestEditsCheckpointTime = checkpointTime;
         latestEditsSD = sd;
       }
+      
+      // check that we have a valid, non-default checkpointTime
       if (checkpointTime <= 0L)
         needToSave |= true;
+      
       // set finalized flag
       isUpgradeFinalized = isUpgradeFinalized && !sd.getPreviousDir().exists();
     }
@@ -901,10 +921,13 @@
 
     // Make sure we are loading image and edits from same checkpoint
     if (latestNameCheckpointTime != latestEditsCheckpointTime)
-      throw new IOException("Inconsitent storage detected, " +
+      throw new IOException("Inconsistent storage detected, " +
                             "name and edits storage do not match");
     
-    // Recover from previous interrrupted checkpoint if any
+    // If there was more than one checkpointTime recorded we should save
+    needToSave |= checkpointTimes.size() != 1;
+    
+    // Recover from previous interrupted checkpoint, if any
     needToSave |= recoverInterruptedCheckpoint(latestNameSD, latestEditsSD);
 
     long startTime = FSNamesystem.now();
@@ -920,12 +943,16 @@
     
     // Load latest edits
     needToSave |= (loadFSEdits(latestEditsSD) > 0);
+
+    assert editLog != null : "editLog must be initialized";
+    if(!editLog.isOpen())
+      editLog.open();
     
     return needToSave;
   }
 
   /**
-   * Load in the filesystem imagefrom file. It's a big list of
+   * Load in the filesystem image from file. It's a big list of
    * filenames and blocks.  Return whether we should
    * "re-save" and consolidate the edit-logs
    */
@@ -1099,16 +1126,20 @@
     int numEdits = 0;
     EditLogFileInputStream edits = 
       new EditLogFileInputStream(getImageFile(sd, NameNodeFile.EDITS));
+    
     numEdits = editLog.loadFSEdits(edits);
     edits.close();
     File editsNew = getImageFile(sd, NameNodeFile.EDITS_NEW);
+    
     if (editsNew.exists() && editsNew.length() > 0) {
       edits = new EditLogFileInputStream(editsNew);
       numEdits += editLog.loadFSEdits(edits);
       edits.close();
     }
+    
     // update the counts.
     getFSNamesystem().dir.updateCountForINodeWithQuota();    
+    
     return numEdits;
   }
 



Mime
View raw message