hadoop-hdfs-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From sur...@apache.org
Subject svn commit: r1126897 - in /hadoop/hdfs/branches/yahoo-merge: ./ src/c++/libhdfs/ src/contrib/hdfsproxy/ src/java/ src/java/org/apache/hadoop/hdfs/server/common/ src/java/org/apache/hadoop/hdfs/server/datanode/ src/java/org/apache/hadoop/hdfs/server/nam...
Date Tue, 24 May 2011 07:10:01 GMT
Author: suresh
Date: Tue May 24 07:10:00 2011
New Revision: 1126897

URL: http://svn.apache.org/viewvc?rev=1126897&view=rev
Log:
Merging change r1078871 from HDFS-1052 branch

Modified:
    hadoop/hdfs/branches/yahoo-merge/   (props changed)
    hadoop/hdfs/branches/yahoo-merge/CHANGES.txt
    hadoop/hdfs/branches/yahoo-merge/build.xml   (props changed)
    hadoop/hdfs/branches/yahoo-merge/src/c++/libhdfs/   (props changed)
    hadoop/hdfs/branches/yahoo-merge/src/contrib/hdfsproxy/   (props changed)
    hadoop/hdfs/branches/yahoo-merge/src/java/   (props changed)
    hadoop/hdfs/branches/yahoo-merge/src/java/org/apache/hadoop/hdfs/server/common/Storage.java
    hadoop/hdfs/branches/yahoo-merge/src/java/org/apache/hadoop/hdfs/server/datanode/BlockPoolSliceStorage.java
    hadoop/hdfs/branches/yahoo-merge/src/java/org/apache/hadoop/hdfs/server/datanode/DataStorage.java
    hadoop/hdfs/branches/yahoo-merge/src/java/org/apache/hadoop/hdfs/server/datanode/ReplicaInfo.java
  (props changed)
    hadoop/hdfs/branches/yahoo-merge/src/java/org/apache/hadoop/hdfs/server/namenode/BackupStorage.java
    hadoop/hdfs/branches/yahoo-merge/src/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java
    hadoop/hdfs/branches/yahoo-merge/src/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java
    hadoop/hdfs/branches/yahoo-merge/src/test/hdfs/   (props changed)
    hadoop/hdfs/branches/yahoo-merge/src/test/hdfs/org/apache/hadoop/hdfs/TestDFSUpgradeFromImage.java
    hadoop/hdfs/branches/yahoo-merge/src/test/hdfs/org/apache/hadoop/hdfs/UpgradeUtilities.java
    hadoop/hdfs/branches/yahoo-merge/src/webapps/datanode/   (props changed)
    hadoop/hdfs/branches/yahoo-merge/src/webapps/hdfs/   (props changed)
    hadoop/hdfs/branches/yahoo-merge/src/webapps/secondary/   (props changed)

Propchange: hadoop/hdfs/branches/yahoo-merge/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue May 24 07:10:00 2011
@@ -1,5 +1,5 @@
 /hadoop/core/branches/branch-0.19/hdfs:713112
-/hadoop/hdfs/branches/HDFS-1052:1078924,1078943,1080331,1080391,1080402,1081603,1082326,1084245,1086788,1090419
+/hadoop/hdfs/branches/HDFS-1052:1078871,1078924,1078943,1080331,1080391,1080402,1081603,1082326,1084245,1086788,1090419
 /hadoop/hdfs/branches/HDFS-265:796829-820463
 /hadoop/hdfs/branches/branch-0.21:820487
-/hadoop/hdfs/trunk:987665-1004788,1026178-1028906,1032470-1033639,1034073,1034082-1034181,1034501-1034544,1035386,1035508,1035841-1035842,1036738,1039957,1040005,1040411-1040412,1049193,1051334,1052823,1053214,1060619,1061067,1062020,1062045,1062052,1067288,1071518,1074282,1080095,1080380,1080836,1081580,1082263,1083951,1085509,1086479,1086654,1087080,1087115,1087437,1090114,1090357,1091515,1091619,1091874,1092432,1092507,1092524,1092584,1094748,1095245,1095461,1095789,1096846,1097648,1097969,1098781,1098867,1099285,1099640-1099641,1101137,1101282,1101293,1101324,1101675,1101753,1102005,1102459,1102511,1102833,1102947,1103957-1103958,1103970,1104395,1104407,1124576
+/hadoop/hdfs/trunk:987665-1004788,1026178-1028906,1032470-1033639,1034073,1034082-1034181,1034501-1034544,1035386,1035508,1035841-1035842,1036213,1036738,1039957,1040005,1040411-1040412,1049193,1051334,1052823,1053214,1060619,1061067,1062020,1062045,1062052,1067288,1071518,1074282,1080095,1080380,1080836,1081580,1082263,1083951,1085509,1086479,1086654,1087080,1087115,1087437,1090114,1090357,1091515,1091619,1091874,1092432,1092507,1092524,1092584,1094748,1095245,1095461,1095789,1096846,1097648,1097969,1098781,1098867,1099285,1099640-1099641,1101137,1101282,1101293,1101324,1101675,1101753,1102005,1102459,1102511,1102833,1102947,1103957-1103958,1103970,1104395,1104407,1124576

Modified: hadoop/hdfs/branches/yahoo-merge/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/yahoo-merge/CHANGES.txt?rev=1126897&r1=1126896&r2=1126897&view=diff
==============================================================================
--- hadoop/hdfs/branches/yahoo-merge/CHANGES.txt (original)
+++ hadoop/hdfs/branches/yahoo-merge/CHANGES.txt Tue May 24 07:10:00 2011
@@ -618,6 +618,9 @@ Release 0.22.0 - Unreleased
     HDFS-1071. savenamespace should write the fsimage to all configured 
     fs.name.dir in parallel (Dmytro Molkov via jghoman)
 
+    HDFS-259. Remove intentionally corrupt 0.13 directory layout creation.
+    (Todd Lipcon via eli).
+
   OPTIMIZATIONS
 
     HDFS-1140. Speedup INode.getPathComponents. (Dmytro Molkov via shv)

Propchange: hadoop/hdfs/branches/yahoo-merge/build.xml
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue May 24 07:10:00 2011
@@ -1,6 +1,6 @@
 /hadoop/core/branches/branch-0.19/hdfs/build.xml:713112
 /hadoop/core/trunk/build.xml:779102
-/hadoop/hdfs/branches/HDFS-1052/build.xml:1078924,1078943,1080331,1080391,1080402,1081603,1082326,1084245,1086788,1090419
+/hadoop/hdfs/branches/HDFS-1052/build.xml:1078871,1078924,1078943,1080331,1080391,1080402,1081603,1082326,1084245,1086788,1090419
 /hadoop/hdfs/branches/HDFS-265/build.xml:796829-820463
 /hadoop/hdfs/branches/branch-0.21/build.xml:820487
-/hadoop/hdfs/trunk/build.xml:987665-1004788,1026178-1028906,1032470-1033639,1034073,1034082-1034181,1034501-1034544,1035386,1035508,1035841-1035842,1039957,1040005,1040411-1040412,1049193,1051334,1052823,1053214,1060619,1061067,1062020,1062045,1062052,1067288,1071518,1074282,1080095,1080380,1080836,1081580,1082263,1083951,1085509,1086479,1086654,1087080,1087115,1087437,1090114,1090357,1091515,1091619,1091874,1092432,1092507,1092524,1092584,1094748,1095245,1095461,1095789,1096846,1097648,1097969,1098781,1098867,1099285,1099640-1099641,1101137,1101282,1101293,1101324,1101675,1101753,1102005,1102459,1102511,1102833,1102947,1103957-1103958,1103970,1104395,1104407,1124576
+/hadoop/hdfs/trunk/build.xml:987665-1004788,1026178-1028906,1032470-1033639,1034073,1034082-1034181,1034501-1034544,1035386,1035508,1035841-1035842,1036213,1039957,1040005,1040411-1040412,1049193,1051334,1052823,1053214,1060619,1061067,1062020,1062045,1062052,1067288,1071518,1074282,1080095,1080380,1080836,1081580,1082263,1083951,1085509,1086479,1086654,1087080,1087115,1087437,1090114,1090357,1091515,1091619,1091874,1092432,1092507,1092524,1092584,1094748,1095245,1095461,1095789,1096846,1097648,1097969,1098781,1098867,1099285,1099640-1099641,1101137,1101282,1101293,1101324,1101675,1101753,1102005,1102459,1102511,1102833,1102947,1103957-1103958,1103970,1104395,1104407,1124576

Propchange: hadoop/hdfs/branches/yahoo-merge/src/c++/libhdfs/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue May 24 07:10:00 2011
@@ -1,4 +1,4 @@
 /hadoop/core/branches/branch-0.19/mapred/src/c++/libhdfs:713112
 /hadoop/core/trunk/src/c++/libhdfs:776175-784663
-/hadoop/hdfs/branches/HDFS-1052/src/c++/libhdfs:1078924,1078943,1080331,1080391,1080402,1081603,1082326,1084245,1086788,1090419
-/hadoop/hdfs/trunk/src/c++/libhdfs:987665-1004788,1026178-1028906,1032470-1033639,1034073,1034082-1034181,1034501-1034544,1035386,1035508,1035841-1035842,1039957,1040005,1040411-1040412,1049193,1051334,1052823,1053214,1060619,1061067,1062020,1062045,1062052,1067288,1071518,1074282,1080095,1080380,1080836,1081580,1082263,1083951,1085509,1086479,1086654,1087080,1087115,1087437,1090114,1090357,1091515,1091619,1091874,1092432,1092507,1092524,1092584,1094748,1095245,1095461,1095789,1096846,1097648,1097969,1098781,1098867,1099285,1099640-1099641,1101137,1101282,1101293,1101324,1101675,1101753,1102005,1102459,1102511,1102833,1102947,1103957-1103958,1103970,1104395,1104407,1124576
+/hadoop/hdfs/branches/HDFS-1052/src/c++/libhdfs:1078871,1078924,1078943,1080331,1080391,1080402,1081603,1082326,1084245,1086788,1090419
+/hadoop/hdfs/trunk/src/c++/libhdfs:987665-1004788,1026178-1028906,1032470-1033639,1034073,1034082-1034181,1034501-1034544,1035386,1035508,1035841-1035842,1036213,1039957,1040005,1040411-1040412,1049193,1051334,1052823,1053214,1060619,1061067,1062020,1062045,1062052,1067288,1071518,1074282,1080095,1080380,1080836,1081580,1082263,1083951,1085509,1086479,1086654,1087080,1087115,1087437,1090114,1090357,1091515,1091619,1091874,1092432,1092507,1092524,1092584,1094748,1095245,1095461,1095789,1096846,1097648,1097969,1098781,1098867,1099285,1099640-1099641,1101137,1101282,1101293,1101324,1101675,1101753,1102005,1102459,1102511,1102833,1102947,1103957-1103958,1103970,1104395,1104407,1124576

Propchange: hadoop/hdfs/branches/yahoo-merge/src/contrib/hdfsproxy/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue May 24 07:10:00 2011
@@ -1,6 +1,6 @@
 /hadoop/core/branches/branch-0.19/hdfs/src/contrib/hdfsproxy:713112
 /hadoop/core/trunk/src/contrib/hdfsproxy:776175-784663
-/hadoop/hdfs/branches/HDFS-1052/src/contrib/hdfsproxy:1078924,1078943,1080331,1080391,1080402,1081603,1082326,1084245,1086788,1090419
+/hadoop/hdfs/branches/HDFS-1052/src/contrib/hdfsproxy:1078871,1078924,1078943,1080331,1080391,1080402,1081603,1082326,1084245,1086788,1090419
 /hadoop/hdfs/branches/HDFS-265/src/contrib/hdfsproxy:796829-820463
 /hadoop/hdfs/branches/branch-0.21/src/contrib/hdfsproxy:820487
-/hadoop/hdfs/trunk/src/contrib/hdfsproxy:987665-1004788,1026178-1028906,1032470-1033639,1034073,1034082-1034181,1034501-1034544,1035386,1035508,1035841-1035842,1039957,1040005,1040411-1040412,1049193,1051334,1052823,1053214,1060619,1061067,1062020,1062045,1062052,1067288,1071518,1074282,1080095,1080380,1080836,1081580,1082263,1083951,1085509,1086479,1086654,1087080,1087115,1087437,1090114,1090357,1091515,1091619,1091874,1092432,1092507,1092524,1092584,1094748,1095245,1095461,1095789,1096846,1097648,1097969,1098781,1098867,1099285,1099640-1099641,1101137,1101282,1101293,1101324,1101675,1101753,1102005,1102459,1102511,1102833,1102947,1103957-1103958,1103970,1104395,1104407,1124576
+/hadoop/hdfs/trunk/src/contrib/hdfsproxy:987665-1004788,1026178-1028906,1032470-1033639,1034073,1034082-1034181,1034501-1034544,1035386,1035508,1035841-1035842,1036213,1039957,1040005,1040411-1040412,1049193,1051334,1052823,1053214,1060619,1061067,1062020,1062045,1062052,1067288,1071518,1074282,1080095,1080380,1080836,1081580,1082263,1083951,1085509,1086479,1086654,1087080,1087115,1087437,1090114,1090357,1091515,1091619,1091874,1092432,1092507,1092524,1092584,1094748,1095245,1095461,1095789,1096846,1097648,1097969,1098781,1098867,1099285,1099640-1099641,1101137,1101282,1101293,1101324,1101675,1101753,1102005,1102459,1102511,1102833,1102947,1103957-1103958,1103970,1104395,1104407,1124576

Propchange: hadoop/hdfs/branches/yahoo-merge/src/java/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue May 24 07:10:00 2011
@@ -1,6 +1,6 @@
 /hadoop/core/branches/branch-0.19/hdfs/src/java:713112
 /hadoop/core/trunk/src/hdfs:776175-785643,785929-786278
-/hadoop/hdfs/branches/HDFS-1052/src/java:1078924,1078943,1080331,1080391,1080402,1081603,1082326,1084245,1086788,1090419
+/hadoop/hdfs/branches/HDFS-1052/src/java:1078871,1078924,1078943,1080331,1080391,1080402,1081603,1082326,1084245,1086788,1090419
 /hadoop/hdfs/branches/HDFS-265/src/java:796829-820463
 /hadoop/hdfs/branches/branch-0.21/src/java:820487
-/hadoop/hdfs/trunk/src/java:987665-1004788,1026178-1028906,1032470-1033639,1034073,1034082-1034181,1034501-1034544,1035386,1035508,1035841-1035842,1039957,1040005,1040411-1040412,1049193,1051334,1052823,1053214,1060619,1061067,1062020,1062045,1062052,1067288,1071518,1074282,1080095,1080380,1080836,1081580,1082263,1083951,1085509,1086479,1086654,1087080,1087115,1087437,1090114,1090357,1091515,1091619,1091874,1092432,1092507,1092524,1092584,1094748,1095245,1095461,1095789,1096846,1097648,1097969,1098781,1098867,1099285,1099640-1099641,1101137,1101282,1101293,1101324,1101675,1101753,1102005,1102459,1102511,1102833,1102947,1103957-1103958,1103970,1104395,1104407,1124576
+/hadoop/hdfs/trunk/src/java:987665-1004788,1026178-1028906,1032470-1033639,1034073,1034082-1034181,1034501-1034544,1035386,1035508,1035841-1035842,1036213,1039957,1040005,1040411-1040412,1049193,1051334,1052823,1053214,1060619,1061067,1062020,1062045,1062052,1067288,1071518,1074282,1080095,1080380,1080836,1081580,1082263,1083951,1085509,1086479,1086654,1087080,1087115,1087437,1090114,1090357,1091515,1091619,1091874,1092432,1092507,1092524,1092584,1094748,1095245,1095461,1095789,1096846,1097648,1097969,1098781,1098867,1099285,1099640-1099641,1101137,1101282,1101293,1101324,1101675,1101753,1102005,1102459,1102511,1102833,1102947,1103957-1103958,1103970,1104395,1104407,1124576

Modified: hadoop/hdfs/branches/yahoo-merge/src/java/org/apache/hadoop/hdfs/server/common/Storage.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/yahoo-merge/src/java/org/apache/hadoop/hdfs/server/common/Storage.java?rev=1126897&r1=1126896&r2=1126897&view=diff
==============================================================================
--- hadoop/hdfs/branches/yahoo-merge/src/java/org/apache/hadoop/hdfs/server/common/Storage.java
(original)
+++ hadoop/hdfs/branches/yahoo-merge/src/java/org/apache/hadoop/hdfs/server/common/Storage.java
Tue May 24 07:10:00 2011
@@ -64,7 +64,7 @@ public abstract class Storage extends St
 
   // Constants
   
-  // last layout version that did not suppot upgrades
+  // last layout version that did not support upgrades
   public static final int LAST_PRE_UPGRADE_LAYOUT_VERSION = -3;
   
   // this corresponds to Hadoop-0.14.
@@ -86,7 +86,7 @@ public abstract class Storage extends St
 
   private   static final String STORAGE_FILE_LOCK     = "in_use.lock";
   protected static final String STORAGE_FILE_VERSION  = "VERSION";
-  public static final String STORAGE_DIR_CURRENT      = "current";
+  public    static final String STORAGE_DIR_CURRENT   = "current";
   public    static final String STORAGE_DIR_PREVIOUS  = "previous";
   public    static final String STORAGE_TMP_REMOVED   = "removed.tmp";
   public    static final String STORAGE_TMP_PREVIOUS  = "previous.tmp";
@@ -279,7 +279,6 @@ public abstract class Storage extends St
      * @throws IOException
      */
     public void write() throws IOException {
-      corruptPreUpgradeStorage(root);
       write(getVersionFile());
     }
 
@@ -495,8 +494,7 @@ public abstract class Storage extends St
       if (startOpt == HdfsConstants.StartupOption.FORMAT)
         return StorageState.NOT_FORMATTED;
       if (startOpt != HdfsConstants.StartupOption.IMPORT) {
-        //make sure no conversion is required
-        checkConversionNeeded(this);
+        checkOldLayoutStorage(this);
       }
 
       // check whether current directory is valid
@@ -708,16 +706,22 @@ public abstract class Storage extends St
   protected void addStorageDir(StorageDirectory sd) {
     storageDirs.add(sd);
   }
-  
-  public abstract boolean isConversionNeeded(StorageDirectory sd) throws IOException;
 
-  /*
-   * Coversion is no longer supported. So this should throw exception if
-   * conversion is needed.
-   */
-  private void checkConversionNeeded(StorageDirectory sd) throws IOException {
-    if (isConversionNeeded(sd)) {
-      //throw an exception
+  /**
+   * Return true if the layout of the given storage directory is from a version
+   * of Hadoop prior to the introduction of the "current" and "previous"
+   * directories which allow upgrade and rollback.
+   */
+  public abstract boolean isPreUpgradableLayout(StorageDirectory sd)
+  throws IOException;
+
+  /**
+   * Check if the given storage directory comes from a version of Hadoop
+   * prior to when the directory layout changed (ie 0.13). If this is
+   * the case, this method throws an IOException.
+   */
+  private void checkOldLayoutStorage(StorageDirectory sd) throws IOException {
+    if (isPreUpgradableLayout(sd)) {
       checkVersionUpgradable(0);
     }
   }
@@ -861,22 +865,6 @@ public abstract class Storage extends St
       + "-" + Integer.toString(storage.getLayoutVersion())
       + "-" + Long.toString(storage.getCTime());
   }
-
-  // Pre-upgrade version compatibility
-  protected abstract void corruptPreUpgradeStorage(File rootDir) throws IOException;
-
-  protected void writeCorruptedData(RandomAccessFile file) throws IOException {
-    final String messageForPreUpgradeVersion =
-      "\nThis file is INTENTIONALLY CORRUPTED so that versions\n"
-      + "of Hadoop prior to 0.13 (which are incompatible\n"
-      + "with this directory layout) will fail to start.\n";
-  
-    file.seek(0);
-    file.writeInt(FSConstants.LAYOUT_VERSION);
-    org.apache.hadoop.hdfs.DeprecatedUTF8.writeString(file, "");
-    file.writeBytes(messageForPreUpgradeVersion);
-    file.getFD().sync();
-  }
   
   String getProperty(Properties props, StorageDirectory sd,
       String name) throws InconsistentFSStateException {

Modified: hadoop/hdfs/branches/yahoo-merge/src/java/org/apache/hadoop/hdfs/server/datanode/BlockPoolSliceStorage.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/yahoo-merge/src/java/org/apache/hadoop/hdfs/server/datanode/BlockPoolSliceStorage.java?rev=1126897&r1=1126896&r2=1126897&view=diff
==============================================================================
--- hadoop/hdfs/branches/yahoo-merge/src/java/org/apache/hadoop/hdfs/server/datanode/BlockPoolSliceStorage.java
(original)
+++ hadoop/hdfs/branches/yahoo-merge/src/java/org/apache/hadoop/hdfs/server/datanode/BlockPoolSliceStorage.java
Tue May 24 07:10:00 2011
@@ -20,7 +20,6 @@ package org.apache.hadoop.hdfs.server.da
 
 import java.io.File;
 import java.io.IOException;
-import java.io.RandomAccessFile;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Iterator;
@@ -473,22 +472,6 @@ public class BlockPoolSliceStorage exten
     LOG.info( hardLink.linkStats.report() );
   }
 
-  protected void corruptPreUpgradeStorage(File rootDir) throws IOException {
-    File oldF = new File(rootDir, "storage");
-    if (oldF.exists())
-      return;
-    // recreate old storage file to let pre-upgrade versions fail
-    if (!oldF.createNewFile())
-      throw new IOException("Cannot create file " + oldF);
-    RandomAccessFile oldFile = new RandomAccessFile(oldF, "rws");
-    // write new version into old storage file
-    try {
-      writeCorruptedData(oldFile);
-    } finally {
-      oldFile.close();
-    }
-  }
-
   private void verifyDistributedUpgradeProgress(NamespaceInfo nsInfo)
       throws IOException {
     UpgradeManagerDatanode um = 
@@ -514,11 +497,6 @@ public class BlockPoolSliceStorage exten
   }
 
   @Override
-  public boolean isConversionNeeded(StorageDirectory sd) throws IOException {
-    return false;
-  }
-  
-  @Override
   public String toString() {
     return super.toString() + ";bpid=" + blockpoolID;
   }
@@ -532,4 +510,9 @@ public class BlockPoolSliceStorage exten
   public static File getBpRoot(String bpID, File dnCurDir) {
     return new File(dnCurDir, bpID);
   }
+
+  @Override
+  public boolean isPreUpgradableLayout(StorageDirectory sd) throws IOException {
+    return false;
+  }
 }

Modified: hadoop/hdfs/branches/yahoo-merge/src/java/org/apache/hadoop/hdfs/server/datanode/DataStorage.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/yahoo-merge/src/java/org/apache/hadoop/hdfs/server/datanode/DataStorage.java?rev=1126897&r1=1126896&r2=1126897&view=diff
==============================================================================
--- hadoop/hdfs/branches/yahoo-merge/src/java/org/apache/hadoop/hdfs/server/datanode/DataStorage.java
(original)
+++ hadoop/hdfs/branches/yahoo-merge/src/java/org/apache/hadoop/hdfs/server/datanode/DataStorage.java
Tue May 24 07:10:00 2011
@@ -323,7 +323,8 @@ public class DataStorage extends Storage
     }
   }
 
-  public boolean isConversionNeeded(StorageDirectory sd) throws IOException {
+  @Override
+  public boolean isPreUpgradableLayout(StorageDirectory sd) throws IOException {
     File oldF = new File(sd.getRoot(), "storage");
     if (!oldF.exists())
       return false;
@@ -720,22 +721,6 @@ public class DataStorage extends Storage
     }
   }
 
-  protected void corruptPreUpgradeStorage(File rootDir) throws IOException {
-    File oldF = new File(rootDir, "storage");
-    if (oldF.exists())
-      return;
-    // recreate old storage file to let pre-upgrade versions fail
-    if (!oldF.createNewFile())
-      throw new IOException("Cannot create file " + oldF);
-    RandomAccessFile oldFile = new RandomAccessFile(oldF, "rws");
-    // write new version into old storage file
-    try {
-      writeCorruptedData(oldFile);
-    } finally {
-      oldFile.close();
-    }
-  }
-
   private void verifyDistributedUpgradeProgress(
                   NamespaceInfo nsInfo
                 ) throws IOException {

Propchange: hadoop/hdfs/branches/yahoo-merge/src/java/org/apache/hadoop/hdfs/server/datanode/ReplicaInfo.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue May 24 07:10:00 2011
@@ -2,7 +2,7 @@
 /hadoop/core/branches/branch-0.19/hdfs/src/java/org/apache/hadoop/hdfs/server/datanode/ReplicaInfo.java:713112
 /hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/server/datanode/DatanodeBlockInfo.java:776175-785643,785929-786278
 /hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/server/datanode/ReplicaInfo.java:776175-785643,785929-786278
-/hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/datanode/ReplicaInfo.java:1078924,1078943,1080331,1080391,1080402,1081603,1082326,1084245,1086788,1090419
+/hadoop/hdfs/branches/HDFS-1052/src/java/org/apache/hadoop/hdfs/server/datanode/ReplicaInfo.java:1078871,1078924,1078943,1080331,1080391,1080402,1081603,1082326,1084245,1086788,1090419
 /hadoop/hdfs/branches/HDFS-265/src/java/org/apache/hadoop/hdfs/server/datanode/ReplicaInfo.java:796829-820463
 /hadoop/hdfs/branches/branch-0.21/src/java/org/apache/hadoop/hdfs/server/datanode/ReplicaInfo.java:820487
-/hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/datanode/ReplicaInfo.java:1026178-1028906,1032470-1033639,1034073,1034082-1034181,1034501-1034544,1035386,1035508,1035841-1035842,1039957,1040005,1040411-1040412,1049193,1051334,1052823,1053214,1060619,1061067,1062020,1062045,1062052,1067288,1071518,1074282,1080095,1080380,1080836,1081580,1082263,1083951,1085509,1086479,1086654,1087080,1087115,1087437,1090114,1090357,1091515,1091619,1091874,1092432,1092507,1092524,1092584,1094748,1095245,1095461,1095789,1096846,1097648,1097969,1098781,1098867,1099285,1099640-1099641,1101137,1101282,1101293,1101324,1101675,1101753,1102005,1102459,1102511,1102833,1102947,1103957-1103958,1103970,1104395,1104407,1124576
+/hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/datanode/ReplicaInfo.java:1026178-1028906,1032470-1033639,1034073,1034082-1034181,1034501-1034544,1035386,1035508,1035841-1035842,1036213,1039957,1040005,1040411-1040412,1049193,1051334,1052823,1053214,1060619,1061067,1062020,1062045,1062052,1067288,1071518,1074282,1080095,1080380,1080836,1081580,1082263,1083951,1085509,1086479,1086654,1087080,1087115,1087437,1090114,1090357,1091515,1091619,1091874,1092432,1092507,1092524,1092584,1094748,1095245,1095461,1095789,1096846,1097648,1097969,1098781,1098867,1099285,1099640-1099641,1101137,1101282,1101293,1101324,1101675,1101753,1102005,1102459,1102511,1102833,1102947,1103957-1103958,1103970,1104395,1104407,1124576

Modified: hadoop/hdfs/branches/yahoo-merge/src/java/org/apache/hadoop/hdfs/server/namenode/BackupStorage.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/yahoo-merge/src/java/org/apache/hadoop/hdfs/server/namenode/BackupStorage.java?rev=1126897&r1=1126896&r2=1126897&view=diff
==============================================================================
--- hadoop/hdfs/branches/yahoo-merge/src/java/org/apache/hadoop/hdfs/server/namenode/BackupStorage.java
(original)
+++ hadoop/hdfs/branches/yahoo-merge/src/java/org/apache/hadoop/hdfs/server/namenode/BackupStorage.java
Tue May 24 07:10:00 2011
@@ -61,7 +61,7 @@ public class BackupStorage extends FSIma
   }
 
   @Override
-  public boolean isConversionNeeded(StorageDirectory sd) {
+  public boolean isPreUpgradableLayout(StorageDirectory sd) throws IOException {
     return false;
   }
 

Modified: hadoop/hdfs/branches/yahoo-merge/src/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/yahoo-merge/src/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java?rev=1126897&r1=1126896&r2=1126897&view=diff
==============================================================================
--- hadoop/hdfs/branches/yahoo-merge/src/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java
(original)
+++ hadoop/hdfs/branches/yahoo-merge/src/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java
Tue May 24 07:10:00 2011
@@ -909,12 +909,9 @@ public class FSImage extends Storage {
     return editLog;
   }
 
-  public boolean isConversionNeeded(StorageDirectory sd) throws IOException {
+  public boolean isPreUpgradableLayout(StorageDirectory sd) throws IOException {
     File oldImageDir = new File(sd.getRoot(), "image");
     if (!oldImageDir.exists()) {
-      if(sd.getVersionFile().exists())
-        throw new InconsistentFSStateException(sd.getRoot(),
-            oldImageDir + " does not exist.");
       return false;
     }
     // check the layout version inside the image file
@@ -930,7 +927,7 @@ public class FSImage extends Storage {
     }
     return true;
   }
-  
+
   //
   // Atomic move sequence, to recover from interrupted checkpoint
   //
@@ -1834,25 +1831,6 @@ public class FSImage extends Storage {
     return null;
   }
 
-  protected void corruptPreUpgradeStorage(File rootDir) throws IOException {
-    File oldImageDir = new File(rootDir, "image");
-    if (!oldImageDir.exists())
-      if (!oldImageDir.mkdir())
-        throw new IOException("Cannot create directory " + oldImageDir);
-    File oldImage = new File(oldImageDir, "fsimage");
-    if (!oldImage.exists())
-      // recreate old image file to let pre-upgrade versions fail
-      if (!oldImage.createNewFile())
-        throw new IOException("Cannot create file " + oldImage);
-    RandomAccessFile oldFile = new RandomAccessFile(oldImage, "rws");
-    // write new version into old image file
-    try {
-      writeCorruptedData(oldFile);
-    } finally {
-      oldFile.close();
-    }
-  }
-
   private boolean getDistributedUpgradeState() {
     FSNamesystem ns = getFSNamesystem();
     return ns == null ? false : ns.getDistributedUpgradeState();

Modified: hadoop/hdfs/branches/yahoo-merge/src/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/yahoo-merge/src/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java?rev=1126897&r1=1126896&r2=1126897&view=diff
==============================================================================
--- hadoop/hdfs/branches/yahoo-merge/src/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java
(original)
+++ hadoop/hdfs/branches/yahoo-merge/src/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java
Tue May 24 07:10:00 2011
@@ -596,12 +596,6 @@ public class SecondaryNameNode implement
       super(conf);
     }
 
-    @Override
-    public
-    boolean isConversionNeeded(StorageDirectory sd) {
-      return false;
-    }
-
     /**
      * Analyze checkpoint directories.
      * Create directories if they do not exist.

Propchange: hadoop/hdfs/branches/yahoo-merge/src/test/hdfs/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue May 24 07:10:00 2011
@@ -1,6 +1,6 @@
 /hadoop/core/branches/branch-0.19/hdfs/src/test/hdfs:713112
 /hadoop/core/trunk/src/test/hdfs:776175-785643
-/hadoop/hdfs/branches/HDFS-1052/src/test/hdfs:1078924,1078943,1080331,1080391,1080402,1081603,1082326,1084245,1086788,1090419
+/hadoop/hdfs/branches/HDFS-1052/src/test/hdfs:1078871,1078924,1078943,1080331,1080391,1080402,1081603,1082326,1084245,1086788,1090419
 /hadoop/hdfs/branches/HDFS-265/src/test/hdfs:796829-820463
 /hadoop/hdfs/branches/branch-0.21/src/test/hdfs:820487
-/hadoop/hdfs/trunk/src/test/hdfs:987665-1004788,1026178-1028906,1032470-1033639,1034073,1034082-1034181,1034501-1034544,1035386,1035508,1035841-1035842,1039957,1040005,1040411-1040412,1049193,1051334,1052823,1053214,1060619,1061067,1062020,1062045,1062052,1067288,1071518,1074282,1080095,1080380,1080836,1081580,1082263,1083951,1085509,1086479,1086654,1087080,1087115,1087437,1090114,1090357,1091515,1091619,1091874,1092432,1092507,1092524,1092584,1094748,1095245,1095461,1095789,1096846,1097648,1097969,1098781,1098867,1099285,1099640-1099641,1101137,1101282,1101293,1101324,1101675,1101753,1102005,1102459,1102511,1102833,1102947,1103957-1103958,1103970,1104395,1104407,1124576
+/hadoop/hdfs/trunk/src/test/hdfs:987665-1004788,1026178-1028906,1032470-1033639,1034073,1034082-1034181,1034501-1034544,1035386,1035508,1035841-1035842,1036213,1039957,1040005,1040411-1040412,1049193,1051334,1052823,1053214,1060619,1061067,1062020,1062045,1062052,1067288,1071518,1074282,1080095,1080380,1080836,1081580,1082263,1083951,1085509,1086479,1086654,1087080,1087115,1087437,1090114,1090357,1091515,1091619,1091874,1092432,1092507,1092524,1092584,1094748,1095245,1095461,1095789,1096846,1097648,1097969,1098781,1098867,1099285,1099640-1099641,1101137,1101282,1101293,1101324,1101675,1101753,1102005,1102459,1102511,1102833,1102947,1103957-1103958,1103970,1104395,1104407,1124576

Modified: hadoop/hdfs/branches/yahoo-merge/src/test/hdfs/org/apache/hadoop/hdfs/TestDFSUpgradeFromImage.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/yahoo-merge/src/test/hdfs/org/apache/hadoop/hdfs/TestDFSUpgradeFromImage.java?rev=1126897&r1=1126896&r2=1126897&view=diff
==============================================================================
--- hadoop/hdfs/branches/yahoo-merge/src/test/hdfs/org/apache/hadoop/hdfs/TestDFSUpgradeFromImage.java
(original)
+++ hadoop/hdfs/branches/yahoo-merge/src/test/hdfs/org/apache/hadoop/hdfs/TestDFSUpgradeFromImage.java
Tue May 24 07:10:00 2011
@@ -32,6 +32,7 @@ import org.apache.hadoop.fs.FileUtil;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hdfs.protocol.FSConstants;
 import org.apache.hadoop.hdfs.server.common.HdfsConstants.StartupOption;
+import org.apache.hadoop.util.StringUtils;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 
@@ -48,6 +49,8 @@ public class TestDFSUpgradeFromImage ext
   
   private static final Log LOG = LogFactory.getLog(
                     "org.apache.hadoop.hdfs.TestDFSUpgradeFromImage");
+  private static File TEST_ROOT_DIR =
+                      new File(MiniDFSCluster.getBaseDirectory());
   
   public int numDataNodes = 4;
   
@@ -204,4 +207,47 @@ public class TestDFSUpgradeFromImage ext
       if (cluster != null) { cluster.shutdown(); }
     }
   }
+
+  /**
+   * Test that sets up a fake image from Hadoop 0.3.0 and tries to start a
+   * NN, verifying that the correct error message is thrown.
+   */
+  public void testFailOnPreUpgradeImage() throws IOException {
+    Configuration conf = new HdfsConfiguration();
+
+    File namenodeStorage = new File(TEST_ROOT_DIR, "nnimage-0.3.0");
+    conf.set(DFSConfigKeys.DFS_NAMENODE_NAME_DIR_KEY, namenodeStorage.toString());
+
+    // Set up a fake NN storage that looks like an ancient Hadoop dir circa 0.3.0
+    FileUtil.fullyDelete(namenodeStorage);
+    assertTrue("Make " + namenodeStorage, namenodeStorage.mkdirs());
+    File imageDir = new File(namenodeStorage, "image");
+    assertTrue("Make " + imageDir, imageDir.mkdirs());
+
+    // Hex dump of a formatted image from Hadoop 0.3.0
+    File imageFile = new File(imageDir, "fsimage");
+    byte[] imageBytes = StringUtils.hexStringToByte(
+      "fffffffee17c0d2700000000");
+    FileOutputStream fos = new FileOutputStream(imageFile);
+    try {
+      fos.write(imageBytes);
+    } finally {
+      fos.close();
+    }
+
+    // Now try to start an NN from it
+
+    try {
+      new MiniDFSCluster.Builder(conf).numDataNodes(0)
+        .format(false)
+        .manageDataDfsDirs(false)
+        .manageNameDfsDirs(false)
+        .startupOption(StartupOption.REGULAR)
+        .build();
+      fail("Was able to start NN from 0.3.0 image");
+    } catch (IOException ioe) {
+      LOG.info("Got expected exception", ioe);
+      assertTrue(ioe.toString().contains("Old layout version is 'too old'"));
+    }
+  }
 }

Modified: hadoop/hdfs/branches/yahoo-merge/src/test/hdfs/org/apache/hadoop/hdfs/UpgradeUtilities.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/yahoo-merge/src/test/hdfs/org/apache/hadoop/hdfs/UpgradeUtilities.java?rev=1126897&r1=1126896&r2=1126897&view=diff
==============================================================================
--- hadoop/hdfs/branches/yahoo-merge/src/test/hdfs/org/apache/hadoop/hdfs/UpgradeUtilities.java
(original)
+++ hadoop/hdfs/branches/yahoo-merge/src/test/hdfs/org/apache/hadoop/hdfs/UpgradeUtilities.java
Tue May 24 07:10:00 2011
@@ -326,12 +326,6 @@ public class UpgradeUtilities {
       localFS.copyToLocalFile(new Path(namenodeStorage.toString(), "current"),
                               new Path(newDir.toString()),
                               false);
-      Path newImgDir = new Path(newDir.getParent(), "image");
-      if (!localFS.exists(newImgDir))
-        localFS.copyToLocalFile(
-            new Path(namenodeStorage.toString(), "image"),
-            newImgDir,
-            false);
       retVal[i] = newDir;
     }
     return retVal;
@@ -359,12 +353,6 @@ public class UpgradeUtilities {
       localFS.copyToLocalFile(new Path(datanodeStorage.toString(), "current"),
                               new Path(newDir.toString()),
                               false);
-      Path newStorageFile = new Path(newDir.getParent(), "storage");
-      if (!localFS.exists(newStorageFile))
-        localFS.copyToLocalFile(
-            new Path(datanodeStorage.toString(), "storage"),
-            newStorageFile,
-            false);
       retVal[i] = newDir;
     }
     return retVal;

Propchange: hadoop/hdfs/branches/yahoo-merge/src/webapps/datanode/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue May 24 07:10:00 2011
@@ -1,6 +1,6 @@
 /hadoop/core/branches/branch-0.19/hdfs/src/webapps/datanode:713112
 /hadoop/core/trunk/src/webapps/datanode:776175-784663
-/hadoop/hdfs/branches/HDFS-1052/src/webapps/datanode:1078924,1078943,1080331,1080391,1080402,1081603,1082326,1084245,1086788,1090419
+/hadoop/hdfs/branches/HDFS-1052/src/webapps/datanode:1078871,1078924,1078943,1080331,1080391,1080402,1081603,1082326,1084245,1086788,1090419
 /hadoop/hdfs/branches/HDFS-265/src/webapps/datanode:796829-820463
 /hadoop/hdfs/branches/branch-0.21/src/webapps/datanode:820487
-/hadoop/hdfs/trunk/src/webapps/datanode:987665-1004788,1026178-1028906,1032470-1033639,1034073,1034082-1034181,1034501-1034544,1035386,1035508,1035841-1035842,1039957,1040005,1040411-1040412,1049193,1051334,1052823,1053214,1060619,1061067,1062020,1062045,1062052,1067288,1071518,1074282,1080095,1080380,1080836,1081580,1082263,1083951,1085509,1086479,1086654,1087080,1087115,1087437,1090114,1090357,1091515,1091619,1091874,1092432,1092507,1092524,1092584,1094748,1095245,1095461,1095789,1096846,1097648,1097969,1098781,1098867,1099285,1099640-1099641,1101137,1101282,1101293,1101324,1101675,1101753,1102005,1102459,1102511,1102833,1102947,1103957-1103958,1103970,1104395,1104407,1124576
+/hadoop/hdfs/trunk/src/webapps/datanode:987665-1004788,1026178-1028906,1032470-1033639,1034073,1034082-1034181,1034501-1034544,1035386,1035508,1035841-1035842,1036213,1039957,1040005,1040411-1040412,1049193,1051334,1052823,1053214,1060619,1061067,1062020,1062045,1062052,1067288,1071518,1074282,1080095,1080380,1080836,1081580,1082263,1083951,1085509,1086479,1086654,1087080,1087115,1087437,1090114,1090357,1091515,1091619,1091874,1092432,1092507,1092524,1092584,1094748,1095245,1095461,1095789,1096846,1097648,1097969,1098781,1098867,1099285,1099640-1099641,1101137,1101282,1101293,1101324,1101675,1101753,1102005,1102459,1102511,1102833,1102947,1103957-1103958,1103970,1104395,1104407,1124576

Propchange: hadoop/hdfs/branches/yahoo-merge/src/webapps/hdfs/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue May 24 07:10:00 2011
@@ -1,6 +1,6 @@
 /hadoop/core/branches/branch-0.19/hdfs/src/webapps/hdfs:713112
 /hadoop/core/trunk/src/webapps/hdfs:776175-784663
-/hadoop/hdfs/branches/HDFS-1052/src/webapps/hdfs:1078924,1078943,1080331,1080391,1080402,1081603,1082326,1084245,1086788,1090419
+/hadoop/hdfs/branches/HDFS-1052/src/webapps/hdfs:1078871,1078924,1078943,1080331,1080391,1080402,1081603,1082326,1084245,1086788,1090419
 /hadoop/hdfs/branches/HDFS-265/src/webapps/hdfs:796829-820463
 /hadoop/hdfs/branches/branch-0.21/src/webapps/hdfs:820487
-/hadoop/hdfs/trunk/src/webapps/hdfs:987665-1004788,1026178-1028906,1032470-1033639,1034073,1034082-1034181,1034501-1034544,1035386,1035508,1035841-1035842,1039957,1040005,1040411-1040412,1049193,1051334,1052823,1053214,1060619,1061067,1062020,1062045,1062052,1067288,1071518,1074282,1080095,1080380,1080836,1081580,1082263,1083951,1085509,1086479,1086654,1087080,1087115,1087437,1090114,1090357,1091515,1091619,1091874,1092432,1092507,1092524,1092584,1094748,1095245,1095461,1095789,1096846,1097648,1097969,1098781,1098867,1099285,1099640-1099641,1101137,1101282,1101293,1101324,1101675,1101753,1102005,1102459,1102511,1102833,1102947,1103957-1103958,1103970,1104395,1104407,1124576
+/hadoop/hdfs/trunk/src/webapps/hdfs:987665-1004788,1026178-1028906,1032470-1033639,1034073,1034082-1034181,1034501-1034544,1035386,1035508,1035841-1035842,1036213,1039957,1040005,1040411-1040412,1049193,1051334,1052823,1053214,1060619,1061067,1062020,1062045,1062052,1067288,1071518,1074282,1080095,1080380,1080836,1081580,1082263,1083951,1085509,1086479,1086654,1087080,1087115,1087437,1090114,1090357,1091515,1091619,1091874,1092432,1092507,1092524,1092584,1094748,1095245,1095461,1095789,1096846,1097648,1097969,1098781,1098867,1099285,1099640-1099641,1101137,1101282,1101293,1101324,1101675,1101753,1102005,1102459,1102511,1102833,1102947,1103957-1103958,1103970,1104395,1104407,1124576

Propchange: hadoop/hdfs/branches/yahoo-merge/src/webapps/secondary/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue May 24 07:10:00 2011
@@ -1,6 +1,6 @@
 /hadoop/core/branches/branch-0.19/hdfs/src/webapps/secondary:713112
 /hadoop/core/trunk/src/webapps/secondary:776175-784663
-/hadoop/hdfs/branches/HDFS-1052/src/webapps/secondary:1078924,1078943,1080331,1080391,1080402,1081603,1082326,1084245,1086788,1090419
+/hadoop/hdfs/branches/HDFS-1052/src/webapps/secondary:1078871,1078924,1078943,1080331,1080391,1080402,1081603,1082326,1084245,1086788,1090419
 /hadoop/hdfs/branches/HDFS-265/src/webapps/secondary:796829-820463
 /hadoop/hdfs/branches/branch-0.21/src/webapps/secondary:820487
-/hadoop/hdfs/trunk/src/webapps/secondary:987665-1004788,1026178-1028906,1032470-1033639,1034073,1034082-1034181,1034501-1034544,1035386,1035508,1035841-1035842,1039957,1040005,1040411-1040412,1049193,1051334,1052823,1053214,1060619,1061067,1062020,1062045,1062052,1067288,1071518,1074282,1080095,1080380,1080836,1081580,1082263,1083951,1085509,1086479,1086654,1087080,1087115,1087437,1090114,1090357,1091515,1091619,1091874,1092432,1092507,1092524,1092584,1094748,1095245,1095461,1095789,1096846,1097648,1097969,1098781,1098867,1099285,1099640-1099641,1101137,1101282,1101293,1101324,1101675,1101753,1102005,1102459,1102511,1102833,1102947,1103957-1103958,1103970,1104395,1104407,1124576
+/hadoop/hdfs/trunk/src/webapps/secondary:987665-1004788,1026178-1028906,1032470-1033639,1034073,1034082-1034181,1034501-1034544,1035386,1035508,1035841-1035842,1036213,1039957,1040005,1040411-1040412,1049193,1051334,1052823,1053214,1060619,1061067,1062020,1062045,1062052,1067288,1071518,1074282,1080095,1080380,1080836,1081580,1082263,1083951,1085509,1086479,1086654,1087080,1087115,1087437,1090114,1090357,1091515,1091619,1091874,1092432,1092507,1092524,1092584,1094748,1095245,1095461,1095789,1096846,1097648,1097969,1098781,1098867,1099285,1099640-1099641,1101137,1101282,1101293,1101324,1101675,1101753,1102005,1102459,1102511,1102833,1102947,1103957-1103958,1103970,1104395,1104407,1124576



Mime
View raw message