hadoop-hdfs-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From sur...@apache.org
Subject svn commit: r1096846 - in /hadoop/hdfs/trunk: ./ src/java/org/apache/hadoop/hdfs/protocol/ src/java/org/apache/hadoop/hdfs/server/common/ src/java/org/apache/hadoop/hdfs/server/namenode/ src/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/ src/jav...
Date Tue, 26 Apr 2011 18:45:26 GMT
Author: suresh
Date: Tue Apr 26 18:45:25 2011
New Revision: 1096846

URL: http://svn.apache.org/viewvc?rev=1096846&view=rev
Log:
HDFS-1842. Handle editlog opcode conflict with 0.20.203 during upgrade, by throwing an error
to indicate the editlog needs to be empty. Contributed by Suresh Srinivas.


Modified:
    hadoop/hdfs/trunk/CHANGES.txt
    hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/protocol/FSConstants.java
    hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/common/Storage.java
    hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java
    hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/EditsLoaderCurrent.java
    hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/ImageLoaderCurrent.java
    hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/TestDFSUpgrade.java

Modified: hadoop/hdfs/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/CHANGES.txt?rev=1096846&r1=1096845&r2=1096846&view=diff
==============================================================================
--- hadoop/hdfs/trunk/CHANGES.txt (original)
+++ hadoop/hdfs/trunk/CHANGES.txt Tue Apr 26 18:45:25 2011
@@ -210,6 +210,10 @@ Trunk (unreleased changes)
     HDFS-1594. When the disk becomes full Namenode is getting shutdown and 
     not able to recover. (Aaron T. Myers via eli)
 
+    HDFS-1842. Handle editlog opcode conflict with 0.20.203 during upgrade,
+    by throwing an error to indicate the editlog needs to be empty.
+    (suresh)
+
 Release 0.22.0 - Unreleased
 
   NEW FEATURES

Modified: hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/protocol/FSConstants.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/protocol/FSConstants.java?rev=1096846&r1=1096845&r2=1096846&view=diff
==============================================================================
--- hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/protocol/FSConstants.java (original)
+++ hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/protocol/FSConstants.java Tue Apr 26
18:45:25 2011
@@ -88,7 +88,7 @@ public interface FSConstants {
   // Version is reflected in the data storage file.
   // Versions are negative.
   // Decrement LAYOUT_VERSION to define a new version.
-  public static final int LAYOUT_VERSION = -30;
+  public static final int LAYOUT_VERSION = -34;
   // Current version: 
-  // -30: store only last component of a path in image
+  // -31, -32 and -33 are reserved for 0.20.203, 0.20.204 and 0.22.
 }

Modified: hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/common/Storage.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/common/Storage.java?rev=1096846&r1=1096845&r2=1096846&view=diff
==============================================================================
--- hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/common/Storage.java (original)
+++ hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/common/Storage.java Tue Apr 26
18:45:25 2011
@@ -78,6 +78,9 @@ public abstract class Storage extends St
   // last layout version that did not support persistent rbw replicas
   public static final int PRE_RBW_LAYOUT_VERSION = -19;
   
+  /** Layout versions of 0.20.203 release */
+  public static final int[] LAYOUT_VERSIONS_203 = {-19, -31};
+
   private   static final String STORAGE_FILE_LOCK     = "in_use.lock";
   protected static final String STORAGE_FILE_VERSION  = "VERSION";
   public    static final String STORAGE_DIR_CURRENT   = "current";
@@ -861,4 +864,13 @@ public abstract class Storage extends St
       + "-" + Integer.toString(storage.getLayoutVersion())
       + "-" + Long.toString(storage.getCTime());
   }
+  
+  public static boolean is203LayoutVersion(int layoutVersion) {
+    for (int lv203 : LAYOUT_VERSIONS_203) {
+      if (lv203 == layoutVersion) {
+        return true;
+      }
+    }
+    return false;
+  }
 }

Modified: hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java?rev=1096846&r1=1096845&r2=1096846&view=diff
==============================================================================
--- hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java
(original)
+++ hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java
Tue Apr 26 18:45:25 2011
@@ -509,6 +509,8 @@ public class FSEditLogLoader {
         }
         validateChecksum(in, checksum, numEdits);
       }
+    } catch (IOException ex) {
+      check203UpgradeFailure(logVersion, ex);
     } finally {
       if(closeOnExit)
         in.close();
@@ -652,4 +654,25 @@ public class FSEditLogLoader {
     }
     return blocks;
   }
+  
+  /** 
+   * Throw appropriate exception during upgrade from 203, when editlog loading
+   * could fail due to opcode conflicts.
+   */
+  private void check203UpgradeFailure(int logVersion, IOException ex)
+      throws IOException {
+    // 0.20.203 version version has conflicting opcodes with the later releases.
+    // The editlog must be emptied by restarting the namenode, before proceeding
+    // with the upgrade.
+    if (Storage.is203LayoutVersion(logVersion)
+        && logVersion != FSConstants.LAYOUT_VERSION) {
+      String msg = "During upgrade failed to load the editlog version "
+          + logVersion + " from release 0.20.203. Please go back to the old "
+          + " release and restart the namenode. This empties the editlog "
+          + " and saves the namespace. Resume the upgrade after this step.";
+      throw new IOException(msg, ex);
+    } else {
+      throw ex;
+    }
+  }
 }

Modified: hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/EditsLoaderCurrent.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/EditsLoaderCurrent.java?rev=1096846&r1=1096845&r2=1096846&view=diff
==============================================================================
--- hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/EditsLoaderCurrent.java
(original)
+++ hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/EditsLoaderCurrent.java
Tue Apr 26 18:45:25 2011
@@ -37,8 +37,8 @@ import static org.apache.hadoop.hdfs.too
 @InterfaceStability.Unstable
 class EditsLoaderCurrent implements EditsLoader {
 
-  private static int [] supportedVersions = {
-    -18, -19, -20, -21, -22, -23, -24, -25, -26, -27, -28, -30 };
+  private static int[] supportedVersions = { -18, -19, -20, -21, -22, -23, -24,
+      -25, -26, -27, -28, -30, -31, -32, -33, -34 };
 
   private EditsVisitor v;
   private int editsVersion = 0;

Modified: hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/ImageLoaderCurrent.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/ImageLoaderCurrent.java?rev=1096846&r1=1096845&r2=1096846&view=diff
==============================================================================
--- hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/ImageLoaderCurrent.java
(original)
+++ hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/ImageLoaderCurrent.java
Tue Apr 26 18:45:25 2011
@@ -119,8 +119,8 @@ import org.apache.hadoop.security.token.
 class ImageLoaderCurrent implements ImageLoader {
   protected final DateFormat dateFormat = 
                                       new SimpleDateFormat("yyyy-MM-dd HH:mm");
-  private static int [] versions = 
-    {-16, -17, -18, -19, -20, -21, -22, -23, -24, -25, -26, -27, -28, -30};
+  private static int[] versions = { -16, -17, -18, -19, -20, -21, -22, -23,
+      -24, -25, -26, -27, -28, -30, -31, -32, -33, -34 };
   private int imageVersion = 0;
 
   /* (non-Javadoc)

Modified: hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/TestDFSUpgrade.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/TestDFSUpgrade.java?rev=1096846&r1=1096845&r2=1096846&view=diff
==============================================================================
--- hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/TestDFSUpgrade.java (original)
+++ hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/TestDFSUpgrade.java Tue Apr 26
18:45:25 2011
@@ -29,17 +29,20 @@ import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileUtil;
+import org.apache.hadoop.hdfs.protocol.FSConstants;
 import org.apache.hadoop.hdfs.server.common.Storage;
 import org.apache.hadoop.hdfs.server.common.StorageInfo;
 import org.apache.hadoop.hdfs.server.common.HdfsConstants.NodeType;
 import org.apache.hadoop.hdfs.server.common.HdfsConstants.StartupOption;
+import org.junit.Test;
+import static org.junit.Assert.*;
 
 /**
 * This test ensures the appropriate response (successful or failure) from
 * the system when the system is upgraded under various storage state and
 * version conditions.
 */
-public class TestDFSUpgrade extends TestCase {
+public class TestDFSUpgrade {
  
   private static final Log LOG = LogFactory.getLog(
                                                    "org.apache.hadoop.hdfs.TestDFSUpgrade");
@@ -141,6 +144,7 @@ public class TestDFSUpgrade extends Test
    * This test attempts to upgrade the NameNode and DataNode under
    * a number of valid and invalid conditions.
    */
+  @Test
   public void testUpgrade() throws Exception {
     File[] baseDirs;
     UpgradeUtilities.initialize();
@@ -256,15 +260,23 @@ public class TestDFSUpgrade extends Test
     } // end numDir loop
   }
  
-  protected void tearDown() throws Exception {
-    LOG.info("Shutting down MiniDFSCluster");
-    if (cluster != null) cluster.shutdown();
+  @Test(expected=IOException.class)
+  public void testUpgradeFromPreUpgradeLVFails() throws IOException {
+    // Upgrade from versions prior to Storage#LAST_UPGRADABLE_LAYOUT_VERSION
+    // is not allowed
+    Storage.checkVersionUpgradable(Storage.LAST_PRE_UPGRADE_LAYOUT_VERSION + 1);
+    fail("Expected IOException is not thrown");
   }
-    
+  
+  public void test203LayoutVersion() {
+    for (int lv : Storage.LAYOUT_VERSIONS_203) {
+      assertTrue(Storage.is203LayoutVersion(lv));
+    }
+  }
+  
   public static void main(String[] args) throws Exception {
     new TestDFSUpgrade().testUpgrade();
   }
-  
 }
 
 



Mime
View raw message