hadoop-hdfs-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From sur...@apache.org
Subject svn commit: r1097254 - in /hadoop/hdfs/branches/branch-0.22: ./ src/java/org/apache/hadoop/hdfs/protocol/ src/java/org/apache/hadoop/hdfs/server/common/ src/java/org/apache/hadoop/hdfs/server/namenode/ src/java/org/apache/hadoop/hdfs/tools/offlineImage...
Date Wed, 27 Apr 2011 21:45:29 GMT
Author: suresh
Date: Wed Apr 27 21:45:29 2011
New Revision: 1097254

URL: http://svn.apache.org/viewvc?rev=1097254&view=rev
Log:
HDFS-1822. Handle editlog opcode conflict with 0.20.203 during upgrade, by throwing an error
to indicate the editlog needs to be empty. Contributed by Suresh Srinivas.


Modified:
    hadoop/hdfs/branches/branch-0.22/CHANGES.txt
    hadoop/hdfs/branches/branch-0.22/src/java/org/apache/hadoop/hdfs/protocol/FSConstants.java
    hadoop/hdfs/branches/branch-0.22/src/java/org/apache/hadoop/hdfs/server/common/Storage.java
    hadoop/hdfs/branches/branch-0.22/src/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java
    hadoop/hdfs/branches/branch-0.22/src/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/ImageLoaderCurrent.java
    hadoop/hdfs/branches/branch-0.22/src/test/hdfs/org/apache/hadoop/hdfs/TestDFSUpgrade.java

Modified: hadoop/hdfs/branches/branch-0.22/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/branch-0.22/CHANGES.txt?rev=1097254&r1=1097253&r2=1097254&view=diff
==============================================================================
--- hadoop/hdfs/branches/branch-0.22/CHANGES.txt (original)
+++ hadoop/hdfs/branches/branch-0.22/CHANGES.txt Wed Apr 27 21:45:29 2011
@@ -476,6 +476,10 @@ Release 0.22.0 - Unreleased
     HDFS-1823. start-dfs.sh script fails if HADOOP_HOME is not set.
     (tomwhite via eli)
 
+    HDFS-1822. Handle editlog opcode conflict with 0.20.203 during upgrade,
+    by throwing an error to indicate the editlog needs to be empty.
+    (suresh)
+
 Release 0.21.1 - Unreleased
 
   IMPROVEMENTS

Modified: hadoop/hdfs/branches/branch-0.22/src/java/org/apache/hadoop/hdfs/protocol/FSConstants.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/branch-0.22/src/java/org/apache/hadoop/hdfs/protocol/FSConstants.java?rev=1097254&r1=1097253&r2=1097254&view=diff
==============================================================================
--- hadoop/hdfs/branches/branch-0.22/src/java/org/apache/hadoop/hdfs/protocol/FSConstants.java
(original)
+++ hadoop/hdfs/branches/branch-0.22/src/java/org/apache/hadoop/hdfs/protocol/FSConstants.java
Wed Apr 27 21:45:29 2011
@@ -91,7 +91,7 @@ public interface FSConstants {
   // Version is reflected in the data storage file.
   // Versions are negative.
   // Decrement LAYOUT_VERSION to define a new version.
-  public static final int LAYOUT_VERSION = -27;
-  // Current version: 
-  // -27: remove intentionally corrupt pre-0.13 image directory
+  public static final int LAYOUT_VERSION = -33;
+  // Current version: 33
+  // -31, -32 are reserved for 0.20.203, 0.20.204
 }

Modified: hadoop/hdfs/branches/branch-0.22/src/java/org/apache/hadoop/hdfs/server/common/Storage.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/branch-0.22/src/java/org/apache/hadoop/hdfs/server/common/Storage.java?rev=1097254&r1=1097253&r2=1097254&view=diff
==============================================================================
--- hadoop/hdfs/branches/branch-0.22/src/java/org/apache/hadoop/hdfs/server/common/Storage.java
(original)
+++ hadoop/hdfs/branches/branch-0.22/src/java/org/apache/hadoop/hdfs/server/common/Storage.java
Wed Apr 27 21:45:29 2011
@@ -78,6 +78,9 @@ public abstract class Storage extends St
   // last layout version that did not support persistent rbw replicas
   public static final int PRE_RBW_LAYOUT_VERSION = -19;
   
+  /** Layout versions of 0.20.203 release */
+  public static final int[] LAYOUT_VERSIONS_203 = {-19, -31};
+
   private   static final String STORAGE_FILE_LOCK     = "in_use.lock";
   protected static final String STORAGE_FILE_VERSION  = "VERSION";
   public    static final String STORAGE_DIR_CURRENT   = "current";
@@ -855,4 +858,13 @@ public abstract class Storage extends St
       + "-" + Integer.toString(storage.getLayoutVersion())
       + "-" + Long.toString(storage.getCTime());
   }
+  
+  public static boolean is203LayoutVersion(int layoutVersion) {
+    for (int lv203 : LAYOUT_VERSIONS_203) {
+      if (lv203 == layoutVersion) {
+        return true;
+      }
+    }
+    return false;
+  }
 }

Modified: hadoop/hdfs/branches/branch-0.22/src/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/branch-0.22/src/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java?rev=1097254&r1=1097253&r2=1097254&view=diff
==============================================================================
--- hadoop/hdfs/branches/branch-0.22/src/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java
(original)
+++ hadoop/hdfs/branches/branch-0.22/src/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java
Wed Apr 27 21:45:29 2011
@@ -481,6 +481,8 @@ public class FSEditLogLoader {
         }
         }
       }
+    } catch (IOException ex) {
+      check203UpgradeFailure(logVersion, ex);
     } finally {
       if(closeOnExit)
         in.close();
@@ -608,4 +610,25 @@ public class FSEditLogLoader {
     }
     return blocks;
   }
+  
+  /** 
+   * Throw appropriate exception during upgrade from 203, when editlog loading
+   * could fail due to opcode conflicts.
+   */
+  private void check203UpgradeFailure(int logVersion, IOException ex)
+      throws IOException {
+    // 0.20.203 version version has conflicting opcodes with the later releases.
+    // The editlog must be emptied by restarting the namenode, before proceeding
+    // with the upgrade.
+    if (Storage.is203LayoutVersion(logVersion)
+        && logVersion != FSConstants.LAYOUT_VERSION) {
+      String msg = "During upgrade failed to load the editlog version "
+          + logVersion + " from release 0.20.203. Please go back to the old "
+          + " release and restart the namenode. This empties the editlog "
+          + " and saves the namespace. Resume the upgrade after this step.";
+      throw new IOException(msg, ex);
+    } else {
+      throw ex;
+    }
+  }
 }

Modified: hadoop/hdfs/branches/branch-0.22/src/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/ImageLoaderCurrent.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/branch-0.22/src/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/ImageLoaderCurrent.java?rev=1097254&r1=1097253&r2=1097254&view=diff
==============================================================================
--- hadoop/hdfs/branches/branch-0.22/src/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/ImageLoaderCurrent.java
(original)
+++ hadoop/hdfs/branches/branch-0.22/src/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/ImageLoaderCurrent.java
Wed Apr 27 21:45:29 2011
@@ -121,7 +121,7 @@ class ImageLoaderCurrent implements Imag
   protected final DateFormat dateFormat = 
                                       new SimpleDateFormat("yyyy-MM-dd HH:mm");
   private static int [] versions = 
-    {-16, -17, -18, -19, -20, -21, -22, -23, -24, -25, -26, -27};
+    {-16, -17, -18, -19, -20, -21, -22, -23, -24, -25, -26, -27, -31, -32, -33};
   private int imageVersion = 0;
 
   /* (non-Javadoc)

Modified: hadoop/hdfs/branches/branch-0.22/src/test/hdfs/org/apache/hadoop/hdfs/TestDFSUpgrade.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/branch-0.22/src/test/hdfs/org/apache/hadoop/hdfs/TestDFSUpgrade.java?rev=1097254&r1=1097253&r2=1097254&view=diff
==============================================================================
--- hadoop/hdfs/branches/branch-0.22/src/test/hdfs/org/apache/hadoop/hdfs/TestDFSUpgrade.java
(original)
+++ hadoop/hdfs/branches/branch-0.22/src/test/hdfs/org/apache/hadoop/hdfs/TestDFSUpgrade.java
Wed Apr 27 21:45:29 2011
@@ -29,17 +29,20 @@ import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileUtil;
+import org.apache.hadoop.hdfs.protocol.FSConstants;
 import org.apache.hadoop.hdfs.server.common.Storage;
 import org.apache.hadoop.hdfs.server.common.StorageInfo;
 import org.apache.hadoop.hdfs.server.common.HdfsConstants.NodeType;
 import org.apache.hadoop.hdfs.server.common.HdfsConstants.StartupOption;
+import org.junit.Test;
+import static org.junit.Assert.*;
 
 /**
 * This test ensures the appropriate response (successful or failure) from
 * the system when the system is upgraded under various storage state and
 * version conditions.
 */
-public class TestDFSUpgrade extends TestCase {
+public class TestDFSUpgrade {
  
   private static final Log LOG = LogFactory.getLog(
                                                    "org.apache.hadoop.hdfs.TestDFSUpgrade");
@@ -141,6 +144,7 @@ public class TestDFSUpgrade extends Test
    * This test attempts to upgrade the NameNode and DataNode under
    * a number of valid and invalid conditions.
    */
+  @Test
   public void testUpgrade() throws Exception {
     File[] baseDirs;
     UpgradeUtilities.initialize();
@@ -256,15 +260,15 @@ public class TestDFSUpgrade extends Test
     } // end numDir loop
   }
  
-  protected void tearDown() throws Exception {
-    LOG.info("Shutting down MiniDFSCluster");
-    if (cluster != null) cluster.shutdown();
+  public void test203LayoutVersion() {
+    for (int lv : Storage.LAYOUT_VERSIONS_203) {
+      assertTrue(Storage.is203LayoutVersion(lv));
+    }
   }
-    
+  
   public static void main(String[] args) throws Exception {
     new TestDFSUpgrade().testUpgrade();
   }
-  
 }
 
 



Mime
View raw message