hadoop-hdfs-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From t...@apache.org
Subject svn commit: r1132525 - in /hadoop/hdfs/trunk: ./ src/java/org/apache/hadoop/hdfs/server/common/ src/java/org/apache/hadoop/hdfs/server/namenode/ src/test/hdfs/org/apache/hadoop/hdfs/
Date Mon, 06 Jun 2011 05:17:54 GMT
Author: todd
Date: Mon Jun  6 05:17:53 2011
New Revision: 1132525

URL: http://svn.apache.org/viewvc?rev=1132525&view=rev
Log:
HDFS-1969. Running rollback on new-version namenode destroys the namespace. Contributed by
Todd Lipcon.

Modified:
    hadoop/hdfs/trunk/CHANGES.txt
    hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/common/Storage.java
    hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java
    hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/NNStorage.java
    hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/TestDFSRollback.java

Modified: hadoop/hdfs/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/CHANGES.txt?rev=1132525&r1=1132524&r2=1132525&view=diff
==============================================================================
--- hadoop/hdfs/trunk/CHANGES.txt (original)
+++ hadoop/hdfs/trunk/CHANGES.txt Mon Jun  6 05:17:53 2011
@@ -1139,6 +1139,9 @@ Release 0.22.0 - Unreleased
     HDFS-977. DataNode.createInterDataNodeProtocolProxy() guards a log
     at the wrong level. (Harsh J Chouraria via todd)
 
+    HDFS-1969. Running rollback on new-version namenode destroys the
+    namespace. (todd)
+
 Release 0.21.1 - Unreleased
     HDFS-1466. TestFcHdfsSymlink relies on /tmp/test not existing. (eli)
 

Modified: hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/common/Storage.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/common/Storage.java?rev=1132525&r1=1132524&r2=1132525&view=diff
==============================================================================
--- hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/common/Storage.java (original)
+++ hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/common/Storage.java Mon Jun 
6 05:17:53 2011
@@ -873,7 +873,7 @@ public abstract class Storage extends St
     String property = props.getProperty(name);
     if (property == null) {
       throw new InconsistentFSStateException(sd.root, "file "
-          + STORAGE_FILE_VERSION + " has " + name + " mising.");
+          + STORAGE_FILE_VERSION + " has " + name + " missing.");
     }
     return property;
   }

Modified: hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java?rev=1132525&r1=1132524&r2=1132525&view=diff
==============================================================================
--- hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java (original)
+++ hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java Mon Jun
 6 05:17:53 2011
@@ -438,6 +438,14 @@ public class FSImage implements NNStorag
 
       // read and verify consistency of the prev dir
       sdPrev.read(sdPrev.getPreviousVersionFile());
+      if (prevState.getLayoutVersion() != FSConstants.LAYOUT_VERSION) {
+        throw new IOException(
+          "Cannot rollback to storage version " +
+          prevState.getLayoutVersion() +
+          " using this version of the NameNode, which uses storage version " +
+          FSConstants.LAYOUT_VERSION + ". " +
+          "Please use the previous version of HDFS to perform the rollback.");
+      }
       canRollback = true;
     }
     if (!canRollback)
@@ -960,6 +968,11 @@ public class FSImage implements NNStorag
    * Save current image and empty journal into {@code current} directory.
    */
   protected void saveCurrent(StorageDirectory sd) throws IOException {
+    if (storage.getLayoutVersion() != FSConstants.LAYOUT_VERSION) {
+      throw new IllegalStateException(
+        "NN with storage version " + FSConstants.LAYOUT_VERSION  +
+        "cannot save an image with version " + storage.getLayoutVersion());
+    }
     File curDir = sd.getCurrentDir();
     NameNodeDirType dirType = (NameNodeDirType)sd.getStorageDirType();
     // save new image or new edits

Modified: hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/NNStorage.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/NNStorage.java?rev=1132525&r1=1132524&r2=1132525&view=diff
==============================================================================
--- hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/NNStorage.java (original)
+++ hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/NNStorage.java Mon Jun
 6 05:17:53 2011
@@ -732,13 +732,17 @@ public class NNStorage extends Storage i
       props.setProperty("distributedUpgradeVersion",
                         Integer.toString(uVersion));
     }
-    if (imageDigest == null) {
-      imageDigest = MD5Hash.digest(
-          new FileInputStream(getStorageFile(sd, NameNodeFile.IMAGE)));
+    if (LayoutVersion.supports(Feature.FSIMAGE_CHECKSUM, layoutVersion)) {
+      // Though the current NN supports this feature, this function
+      // is called with old layoutVersions from the upgrade tests.
+      if (imageDigest == null) {
+        // May be null on the first save after an upgrade.
+        imageDigest = MD5Hash.digest(
+            new FileInputStream(getStorageFile(sd, NameNodeFile.IMAGE)));
+      }
+      props.setProperty(MESSAGE_DIGEST_PROPERTY, imageDigest.toString());
     }
 
-    props.setProperty(MESSAGE_DIGEST_PROPERTY, imageDigest.toString());
-
     writeCheckpointTime(sd);
   }
 

Modified: hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/TestDFSRollback.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/TestDFSRollback.java?rev=1132525&r1=1132524&r2=1132525&view=diff
==============================================================================
--- hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/TestDFSRollback.java (original)
+++ hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/TestDFSRollback.java Mon Jun  6
05:17:53 2011
@@ -32,6 +32,7 @@ import org.apache.hadoop.fs.FileUtil;
 import org.apache.hadoop.hdfs.server.common.StorageInfo;
 import org.apache.hadoop.hdfs.server.common.HdfsConstants.NodeType;
 import org.apache.hadoop.hdfs.server.common.HdfsConstants.StartupOption;
+import org.apache.hadoop.util.StringUtils;
 
 /**
 * This test ensures the appropriate response (successful or failure) from
@@ -89,7 +90,7 @@ public class TestDFSRollback extends Tes
    * Attempts to start a NameNode with the given operation.  Starting
    * the NameNode should throw an exception.
    */
-  void startNameNodeShouldFail(StartupOption operation) {
+  void startNameNodeShouldFail(StartupOption operation, String searchString) {
     try {
       cluster = new MiniDFSCluster.Builder(conf).numDataNodes(0)
                                                 .startupOption(operation)
@@ -99,6 +100,10 @@ public class TestDFSRollback extends Tes
                                                 .build(); // should fail
       throw new AssertionError("NameNode should have failed to start");
     } catch (Exception expected) {
+      if (!expected.getMessage().contains(searchString)) {
+        fail("Expected substring '" + searchString + "' in exception " +
+            "but got: " + StringUtils.stringifyException(expected));
+      }
       // expected
     }
   }
@@ -165,7 +170,8 @@ public class TestDFSRollback extends Tes
 
       log("NameNode rollback without existing previous dir", numDirs);
       UpgradeUtilities.createNameNodeStorageDirs(nameNodeDirs, "current");
-      startNameNodeShouldFail(StartupOption.ROLLBACK);
+      startNameNodeShouldFail(StartupOption.ROLLBACK,
+          "None of the storage directories contain previous fs state");
       UpgradeUtilities.createEmptyDirs(nameNodeDirs);
       
       log("DataNode rollback without existing previous dir", numDirs);
@@ -238,7 +244,8 @@ public class TestDFSRollback extends Tes
       for (File f : baseDirs) { 
         FileUtil.fullyDelete(new File(f,"edits"));
       }
-      startNameNodeShouldFail(StartupOption.ROLLBACK);
+      startNameNodeShouldFail(StartupOption.ROLLBACK,
+          "Edits file is not found");
       UpgradeUtilities.createEmptyDirs(nameNodeDirs);
       
       log("NameNode rollback with no image file", numDirs);
@@ -247,7 +254,8 @@ public class TestDFSRollback extends Tes
       for (File f : baseDirs) { 
         FileUtil.fullyDelete(new File(f,"fsimage")); 
       }
-      startNameNodeShouldFail(StartupOption.ROLLBACK);
+      startNameNodeShouldFail(StartupOption.ROLLBACK,
+          "Image file is not found");
       UpgradeUtilities.createEmptyDirs(nameNodeDirs);
       
       log("NameNode rollback with corrupt version file", numDirs);
@@ -256,7 +264,8 @@ public class TestDFSRollback extends Tes
       for (File f : baseDirs) { 
         UpgradeUtilities.corruptFile(new File(f,"VERSION")); 
       }
-      startNameNodeShouldFail(StartupOption.ROLLBACK);
+      startNameNodeShouldFail(StartupOption.ROLLBACK,
+          "file VERSION has layoutVersion missing");
       UpgradeUtilities.createEmptyDirs(nameNodeDirs);
       
       log("NameNode rollback with old layout version in previous", numDirs);
@@ -269,7 +278,8 @@ public class TestDFSRollback extends Tes
       
       UpgradeUtilities.createNameNodeVersionFile(conf, baseDirs,
           storageInfo, UpgradeUtilities.getCurrentBlockPoolID(cluster));
-      startNameNodeShouldFail(StartupOption.UPGRADE);
+      startNameNodeShouldFail(StartupOption.ROLLBACK,
+          "Cannot rollback to storage version 1 using this version");
       UpgradeUtilities.createEmptyDirs(nameNodeDirs);
     } // end numDir loop
   }



Mime
View raw message