hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From sur...@apache.org
Subject svn commit: r1096079 - in /hadoop/common/branches/branch-0.20-security: ./ src/hdfs/org/apache/hadoop/hdfs/protocol/ src/hdfs/org/apache/hadoop/hdfs/server/common/ src/hdfs/org/apache/hadoop/hdfs/server/namenode/ src/test/org/apache/hadoop/hdfs/server/...
Date Fri, 22 Apr 2011 23:10:22 GMT
Author: suresh
Date: Fri Apr 22 23:10:22 2011
New Revision: 1096079

URL: http://svn.apache.org/viewvc?rev=1096079&view=rev
Log:
HDFS-1842. Handle editlog opcode conflict with 0.20.203 during upgrade, by throwing an error
to indicate the editlog needs to be empty. Contributed by Suresh Srinivas.


Modified:
    hadoop/common/branches/branch-0.20-security/CHANGES.txt
    hadoop/common/branches/branch-0.20-security/src/hdfs/org/apache/hadoop/hdfs/protocol/FSConstants.java
    hadoop/common/branches/branch-0.20-security/src/hdfs/org/apache/hadoop/hdfs/server/common/Storage.java
    hadoop/common/branches/branch-0.20-security/src/hdfs/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
    hadoop/common/branches/branch-0.20-security/src/test/org/apache/hadoop/hdfs/server/namenode/TestEditLog.java

Modified: hadoop/common/branches/branch-0.20-security/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security/CHANGES.txt?rev=1096079&r1=1096078&r2=1096079&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security/CHANGES.txt (original)
+++ hadoop/common/branches/branch-0.20-security/CHANGES.txt Fri Apr 22 23:10:22 2011
@@ -55,6 +55,10 @@ Release 0.20.204.0 - unreleased
     MAPREDUCE-2443. Fix TaskAspect for TaskUmbilicalProtocol.ping(..).
     (Siddharth Seth via szetszwo)
 
+    HDFS-1842. Handle editlog opcode conflict with 0.20.203 during upgrade,
+    by throwing an error to indicate the editlog needs to be empty.
+    (suresh)
+
   IMPROVEMENTS
 
     HDFS-1541. Not marking datanodes dead when namenode in safemode.

Modified: hadoop/common/branches/branch-0.20-security/src/hdfs/org/apache/hadoop/hdfs/protocol/FSConstants.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security/src/hdfs/org/apache/hadoop/hdfs/protocol/FSConstants.java?rev=1096079&r1=1096078&r2=1096079&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security/src/hdfs/org/apache/hadoop/hdfs/protocol/FSConstants.java
(original)
+++ hadoop/common/branches/branch-0.20-security/src/hdfs/org/apache/hadoop/hdfs/protocol/FSConstants.java
Fri Apr 22 23:10:22 2011
@@ -77,8 +77,8 @@ public interface FSConstants {
   // Version is reflected in the data storage file.
   // Versions are negative.
   // Decrement LAYOUT_VERSION to define a new version.
-  public static final int LAYOUT_VERSION = -19;
+  public static final int LAYOUT_VERSION = -32;
   // Current version: 
-  // -19: added new OP_[GET|RENEW|CANCEL]_DELEGATION_TOKEN and
-  // OP_UPDATE_MASTER_KEY.
+  // -32: to handle editlog opcode conflicts with 0.20.203 during upgrades and
+  // to disallow upgrade to release 0.21.
 }

Modified: hadoop/common/branches/branch-0.20-security/src/hdfs/org/apache/hadoop/hdfs/server/common/Storage.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security/src/hdfs/org/apache/hadoop/hdfs/server/common/Storage.java?rev=1096079&r1=1096078&r2=1096079&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security/src/hdfs/org/apache/hadoop/hdfs/server/common/Storage.java
(original)
+++ hadoop/common/branches/branch-0.20-security/src/hdfs/org/apache/hadoop/hdfs/server/common/Storage.java
Fri Apr 22 23:10:22 2011
@@ -73,6 +73,9 @@ public abstract class Storage extends St
    * any upgrade code that uses this constant should also be removed. */
   public static final int PRE_GENERATIONSTAMP_LAYOUT_VERSION = -13;
   
+  /** Layout versions of 203 release */
+  public static final int[] LAYOUT_VERSIONS_203 = {-19, -31};
+  
   private   static final String STORAGE_FILE_LOCK     = "in_use.lock";
   protected static final String STORAGE_FILE_VERSION  = "VERSION";
   public static final String STORAGE_DIR_CURRENT   = "current";
@@ -849,4 +852,13 @@ public abstract class Storage extends St
     file.writeBytes(messageForPreUpgradeVersion);
     file.getFD().sync();
   }
+  
+  public static boolean is203LayoutVersion(int layoutVersion) {
+    for (int lv : LAYOUT_VERSIONS_203) {
+      if (lv == layoutVersion) {
+        return true;
+      }
+    }
+    return false;
+  }
 }

Modified: hadoop/common/branches/branch-0.20-security/src/hdfs/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security/src/hdfs/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java?rev=1096079&r1=1096078&r2=1096079&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security/src/hdfs/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
(original)
+++ hadoop/common/branches/branch-0.20-security/src/hdfs/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
Fri Apr 22 23:10:22 2011
@@ -70,10 +70,10 @@ public class FSEditLog {
   private static final byte OP_CLEAR_NS_QUOTA = 12; // clear namespace quota
   private static final byte OP_TIMES = 13; // sets mod & access time on a file
   private static final byte OP_SET_QUOTA = 14; // sets name and disk quotas.
-  private static final byte OP_GET_DELEGATION_TOKEN = 15; //new delegation token
-  private static final byte OP_RENEW_DELEGATION_TOKEN = 16; //renew delegation token
-  private static final byte OP_CANCEL_DELEGATION_TOKEN = 17; //cancel delegation token
-  private static final byte OP_UPDATE_MASTER_KEY = 18; //update master key
+  private static final byte OP_GET_DELEGATION_TOKEN = 18; //new delegation token
+  private static final byte OP_RENEW_DELEGATION_TOKEN = 19; //renew delegation token
+  private static final byte OP_CANCEL_DELEGATION_TOKEN = 20; //cancel delegation token
+  private static final byte OP_UPDATE_MASTER_KEY = 21; //update master key
 
   private static int sizeFlushBuffer = 512*1024;
 
@@ -832,6 +832,21 @@ public class FSEditLog {
         }
         }
       }
+    } catch (IOException ex) {
+      // Failed to load 0.20.203 version edits during upgrade. This version has
+      // conflicting opcodes with the later releases. The editlog must be 
+      // emptied by restarting the namenode, before proceeding with the upgrade.
+      if (Storage.is203LayoutVersion(logVersion) &&
+          logVersion != FSConstants.LAYOUT_VERSION) {
+        String msg = "During upgrade, failed to load the editlog version " + 
+        logVersion + " from release 0.20.203. Please go back to the old " + 
+        " release and restart the namenode. This empties the editlog " +
+        " and saves the namespace. Resume the upgrade after this step.";
+        throw new IOException(msg, ex);
+      } else {
+        throw ex;
+      }
+      
     } finally {
       in.close();
     }

Modified: hadoop/common/branches/branch-0.20-security/src/test/org/apache/hadoop/hdfs/server/namenode/TestEditLog.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security/src/test/org/apache/hadoop/hdfs/server/namenode/TestEditLog.java?rev=1096079&r1=1096078&r2=1096079&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security/src/test/org/apache/hadoop/hdfs/server/namenode/TestEditLog.java
(original)
+++ hadoop/common/branches/branch-0.20-security/src/test/org/apache/hadoop/hdfs/server/namenode/TestEditLog.java
Fri Apr 22 23:10:22 2011
@@ -21,16 +21,13 @@ import junit.framework.TestCase;
 import java.io.*;
 import java.util.Collection;
 import java.util.Iterator;
-import java.util.Random;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.permission.*;
 
 import org.apache.hadoop.hdfs.MiniDFSCluster;
-import org.apache.hadoop.io.ArrayWritable;
-import org.apache.hadoop.io.UTF8;
-import org.apache.hadoop.io.Writable;
 import org.apache.hadoop.hdfs.server.namenode.FSEditLog.EditLogFileInputStream;
+import org.apache.hadoop.hdfs.server.common.Storage;
 import org.apache.hadoop.hdfs.server.common.Storage.StorageDirectory;
 import org.apache.hadoop.hdfs.server.namenode.FSImage.NameNodeDirType;
 import org.apache.hadoop.hdfs.server.namenode.FSImage.NameNodeFile;
@@ -156,4 +153,10 @@ public class TestEditLog extends TestCas
 
     }
   }
+ 
+ public void test203LayoutVersion() {
+   for (int lv : Storage.LAYOUT_VERSIONS_203) {
+     assertTrue(Storage.is203LayoutVersion(lv));
+   }
+ }
 }



Mime
View raw message