hadoop-hdfs-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From e..@apache.org
Subject svn commit: r1028906 - in /hadoop/hdfs/trunk: ./ src/java/org/apache/hadoop/hdfs/server/common/ src/java/org/apache/hadoop/hdfs/server/datanode/ src/java/org/apache/hadoop/hdfs/server/namenode/ src/test/hdfs/org/apache/hadoop/hdfs/
Date Fri, 29 Oct 2010 20:54:18 GMT
Author: eli
Date: Fri Oct 29 20:54:17 2010
New Revision: 1028906

URL: http://svn.apache.org/viewvc?rev=1028906&view=rev
Log:
Revert HDFS-259, need to update LAYOUT_VERSION.

Modified:
    hadoop/hdfs/trunk/CHANGES.txt
    hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/common/Storage.java
    hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/datanode/DataStorage.java
    hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/BackupStorage.java
    hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java
    hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java
    hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/TestDFSUpgradeFromImage.java
    hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/UpgradeUtilities.java

Modified: hadoop/hdfs/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/CHANGES.txt?rev=1028906&r1=1028905&r2=1028906&view=diff
==============================================================================
--- hadoop/hdfs/trunk/CHANGES.txt (original)
+++ hadoop/hdfs/trunk/CHANGES.txt Fri Oct 29 20:54:17 2010
@@ -154,9 +154,6 @@ Trunk (unreleased changes)
     HADOOP-7007. Update the hudson-test-patch ant target to work with the
     latest test-patch.sh script (gkesavan)
 
-    HDFS-259. Remove intentionally corrupt 0.13 directory layout creation.
-    (Todd Lipcon via eli)
-
     HDFS-1462. Refactor edit log loading to a separate class from edit log writing.
     (Todd Lipcon via eli)
 

Modified: hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/common/Storage.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/common/Storage.java?rev=1028906&r1=1028905&r2=1028906&view=diff
==============================================================================
--- hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/common/Storage.java (original)
+++ hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/common/Storage.java Fri Oct 29
20:54:17 2010
@@ -64,7 +64,7 @@ public abstract class Storage extends St
 
   // Constants
   
-  // last layout version that did not support upgrades
+  // last layout version that did not suppot upgrades
   protected static final int LAST_PRE_UPGRADE_LAYOUT_VERSION = -3;
   
   // this corresponds to Hadoop-0.14.
@@ -80,7 +80,7 @@ public abstract class Storage extends St
   
   private   static final String STORAGE_FILE_LOCK     = "in_use.lock";
   protected static final String STORAGE_FILE_VERSION  = "VERSION";
-  public    static final String STORAGE_DIR_CURRENT   = "current";
+  public static final String STORAGE_DIR_CURRENT   = "current";
   private   static final String STORAGE_DIR_PREVIOUS  = "previous";
   private   static final String STORAGE_TMP_REMOVED   = "removed.tmp";
   private   static final String STORAGE_TMP_PREVIOUS  = "previous.tmp";
@@ -256,6 +256,7 @@ public abstract class Storage extends St
      * @throws IOException
      */
     public void write() throws IOException {
+      corruptPreUpgradeStorage(root);
       write(getVersionFile());
     }
 
@@ -471,7 +472,8 @@ public abstract class Storage extends St
       if (startOpt == HdfsConstants.StartupOption.FORMAT)
         return StorageState.NOT_FORMATTED;
       if (startOpt != HdfsConstants.StartupOption.IMPORT) {
-        checkOldLayoutStorage(this);
+        //make sure no conversion is required
+        checkConversionNeeded(this);
       }
 
       // check whether current directory is valid
@@ -684,22 +686,16 @@ public abstract class Storage extends St
   protected void addStorageDir(StorageDirectory sd) {
     storageDirs.add(sd);
   }
+  
+  public abstract boolean isConversionNeeded(StorageDirectory sd) throws IOException;
 
-  /**
-   * Return true if the layout of the given storage directory is from a version
-   * of Hadoop prior to the introduction of the "current" and "previous"
-   * directories which allow upgrade and rollback.
-   */
-  public abstract boolean isPreUpgradableLayout(StorageDirectory sd)
-  throws IOException;
-
-  /**
-   * Check if the given storage directory comes from a version of Hadoop
-   * prior to when the directory layout changed (ie 0.13). If this is
-   * the case, this method throws an IOException.
-   */
-  private void checkOldLayoutStorage(StorageDirectory sd) throws IOException {
-    if (isPreUpgradableLayout(sd)) {
+  /*
+   * Coversion is no longer supported. So this should throw exception if
+   * conversion is needed.
+   */
+  private void checkConversionNeeded(StorageDirectory sd) throws IOException {
+    if (isConversionNeeded(sd)) {
+      //throw an exception
       checkVersionUpgradable(0);
     }
   }
@@ -855,4 +851,20 @@ public abstract class Storage extends St
       + "-" + Integer.toString(storage.getLayoutVersion())
       + "-" + Long.toString(storage.getCTime());
   }
+
+  // Pre-upgrade version compatibility
+  protected abstract void corruptPreUpgradeStorage(File rootDir) throws IOException;
+
+  protected void writeCorruptedData(RandomAccessFile file) throws IOException {
+    final String messageForPreUpgradeVersion =
+      "\nThis file is INTENTIONALLY CORRUPTED so that versions\n"
+      + "of Hadoop prior to 0.13 (which are incompatible\n"
+      + "with this directory layout) will fail to start.\n";
+  
+    file.seek(0);
+    file.writeInt(FSConstants.LAYOUT_VERSION);
+    org.apache.hadoop.hdfs.DeprecatedUTF8.writeString(file, "");
+    file.writeBytes(messageForPreUpgradeVersion);
+    file.getFD().sync();
+  }
 }

Modified: hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/datanode/DataStorage.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/datanode/DataStorage.java?rev=1028906&r1=1028905&r2=1028906&view=diff
==============================================================================
--- hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/datanode/DataStorage.java (original)
+++ hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/datanode/DataStorage.java Fri
Oct 29 20:54:17 2010
@@ -190,8 +190,7 @@ public class DataStorage extends Storage
       storageID = ssid;
   }
 
-  @Override
-  public boolean isPreUpgradableLayout(StorageDirectory sd) throws IOException {
+  public boolean isConversionNeeded(StorageDirectory sd) throws IOException {
     File oldF = new File(sd.getRoot(), "storage");
     if (!oldF.exists())
       return false;
@@ -463,6 +462,22 @@ public class DataStorage extends Storage
                  new File(to, blockNames[i]), oldLV);
   }
 
+  protected void corruptPreUpgradeStorage(File rootDir) throws IOException {
+    File oldF = new File(rootDir, "storage");
+    if (oldF.exists())
+      return;
+    // recreate old storage file to let pre-upgrade versions fail
+    if (!oldF.createNewFile())
+      throw new IOException("Cannot create file " + oldF);
+    RandomAccessFile oldFile = new RandomAccessFile(oldF, "rws");
+    // write new version into old storage file
+    try {
+      writeCorruptedData(oldFile);
+    } finally {
+      oldFile.close();
+    }
+  }
+
   private void verifyDistributedUpgradeProgress(
                   NamespaceInfo nsInfo
                 ) throws IOException {

Modified: hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/BackupStorage.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/BackupStorage.java?rev=1028906&r1=1028905&r2=1028906&view=diff
==============================================================================
--- hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/BackupStorage.java (original)
+++ hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/BackupStorage.java Fri
Oct 29 20:54:17 2010
@@ -61,7 +61,7 @@ public class BackupStorage extends FSIma
   }
 
   @Override
-  public boolean isPreUpgradableLayout(StorageDirectory sd) throws IOException {
+  public boolean isConversionNeeded(StorageDirectory sd) {
     return false;
   }
 

Modified: hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java?rev=1028906&r1=1028905&r2=1028906&view=diff
==============================================================================
--- hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java (original)
+++ hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java Fri Oct
29 20:54:17 2010
@@ -862,9 +862,12 @@ public class FSImage extends Storage {
     return editLog;
   }
 
-  public boolean isPreUpgradableLayout(StorageDirectory sd) throws IOException {
+  public boolean isConversionNeeded(StorageDirectory sd) throws IOException {
     File oldImageDir = new File(sd.getRoot(), "image");
     if (!oldImageDir.exists()) {
+      if(sd.getVersionFile().exists())
+        throw new InconsistentFSStateException(sd.getRoot(),
+            oldImageDir + " does not exist.");
       return false;
     }
     // check the layout version inside the image file
@@ -880,7 +883,7 @@ public class FSImage extends Storage {
     }
     return true;
   }
-
+  
   //
   // Atomic move sequence, to recover from interrupted checkpoint
   //
@@ -2098,6 +2101,25 @@ public class FSImage extends Storage {
     }
   }
 
+  protected void corruptPreUpgradeStorage(File rootDir) throws IOException {
+    File oldImageDir = new File(rootDir, "image");
+    if (!oldImageDir.exists())
+      if (!oldImageDir.mkdir())
+        throw new IOException("Cannot create directory " + oldImageDir);
+    File oldImage = new File(oldImageDir, "fsimage");
+    if (!oldImage.exists())
+      // recreate old image file to let pre-upgrade versions fail
+      if (!oldImage.createNewFile())
+        throw new IOException("Cannot create file " + oldImage);
+    RandomAccessFile oldFile = new RandomAccessFile(oldImage, "rws");
+    // write new version into old image file
+    try {
+      writeCorruptedData(oldFile);
+    } finally {
+      oldFile.close();
+    }
+  }
+
   private boolean getDistributedUpgradeState() {
     FSNamesystem ns = getFSNamesystem();
     return ns == null ? false : ns.getDistributedUpgradeState();

Modified: hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java?rev=1028906&r1=1028905&r2=1028906&view=diff
==============================================================================
--- hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java
(original)
+++ hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java
Fri Oct 29 20:54:17 2010
@@ -585,6 +585,12 @@ public class SecondaryNameNode implement
       super(conf);
     }
 
+    @Override
+    public
+    boolean isConversionNeeded(StorageDirectory sd) {
+      return false;
+    }
+
     /**
      * Analyze checkpoint directories.
      * Create directories if they do not exist.

Modified: hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/TestDFSUpgradeFromImage.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/TestDFSUpgradeFromImage.java?rev=1028906&r1=1028905&r2=1028906&view=diff
==============================================================================
--- hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/TestDFSUpgradeFromImage.java (original)
+++ hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/TestDFSUpgradeFromImage.java Fri
Oct 29 20:54:17 2010
@@ -32,7 +32,6 @@ import org.apache.hadoop.fs.FileUtil;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hdfs.protocol.FSConstants;
 import org.apache.hadoop.hdfs.server.common.HdfsConstants.StartupOption;
-import org.apache.hadoop.util.StringUtils;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 
@@ -49,8 +48,6 @@ public class TestDFSUpgradeFromImage ext
   
   private static final Log LOG = LogFactory.getLog(
                     "org.apache.hadoop.hdfs.TestDFSUpgradeFromImage");
-  private static File TEST_ROOT_DIR =
-                      new File(MiniDFSCluster.getBaseDirectory());
   
   public int numDataNodes = 4;
   
@@ -206,47 +203,4 @@ public class TestDFSUpgradeFromImage ext
       if (cluster != null) { cluster.shutdown(); }
     }
   }
-
-  /**
-   * Test that sets up a fake image from Hadoop 0.3.0 and tries to start a
-   * NN, verifying that the correct error message is thrown.
-   */
-  public void testFailOnPreUpgradeImage() throws IOException {
-    Configuration conf = new HdfsConfiguration();
-
-    File namenodeStorage = new File(TEST_ROOT_DIR, "nnimage-0.3.0");
-    conf.set(DFSConfigKeys.DFS_NAMENODE_NAME_DIR_KEY, namenodeStorage.toString());
-
-    // Set up a fake NN storage that looks like an ancient Hadoop dir circa 0.3.0
-    FileUtil.fullyDelete(namenodeStorage);
-    assertTrue("Make " + namenodeStorage, namenodeStorage.mkdirs());
-    File imageDir = new File(namenodeStorage, "image");
-    assertTrue("Make " + imageDir, imageDir.mkdirs());
-
-    // Hex dump of a formatted image from Hadoop 0.3.0
-    File imageFile = new File(imageDir, "fsimage");
-    byte[] imageBytes = StringUtils.hexStringToByte(
-      "fffffffee17c0d2700000000");
-    FileOutputStream fos = new FileOutputStream(imageFile);
-    try {
-      fos.write(imageBytes);
-    } finally {
-      fos.close();
-    }
-
-    // Now try to start an NN from it
-
-    try {
-      new MiniDFSCluster.Builder(conf).numDataNodes(0)
-        .format(false)
-        .manageDataDfsDirs(false)
-        .manageNameDfsDirs(false)
-        .startupOption(StartupOption.REGULAR)
-        .build();
-      fail("Was able to start NN from 0.3.0 image");
-    } catch (IOException ioe) {
-      LOG.info("Got expected exception", ioe);
-      assertTrue(ioe.toString().contains("Old layout version is 'too old'"));
-    }
-  }
 }

Modified: hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/UpgradeUtilities.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/UpgradeUtilities.java?rev=1028906&r1=1028905&r2=1028906&view=diff
==============================================================================
--- hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/UpgradeUtilities.java (original)
+++ hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/UpgradeUtilities.java Fri Oct 29
20:54:17 2010
@@ -279,11 +279,23 @@ public class UpgradeUtilities {
         localFS.copyToLocalFile(new Path(namenodeStorage.toString(), "current"),
                                 new Path(newDir.toString()),
                                 false);
+        Path newImgDir = new Path(newDir.getParent(), "image");
+        if (!localFS.exists(newImgDir))
+          localFS.copyToLocalFile(
+              new Path(namenodeStorage.toString(), "image"),
+              newImgDir,
+              false);
         break;
       case DATA_NODE:
         localFS.copyToLocalFile(new Path(datanodeStorage.toString(), "current"),
                                 new Path(newDir.toString()),
                                 false);
+        Path newStorageFile = new Path(newDir.getParent(), "storage");
+        if (!localFS.exists(newStorageFile))
+          localFS.copyToLocalFile(
+              new Path(datanodeStorage.toString(), "storage"),
+              newStorageFile,
+              false);
         break;
       }
       retVal[i] = newDir;



Mime
View raw message