hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From rang...@apache.org
Subject svn commit: r677665 - in /hadoop/core/trunk: ./ src/hdfs/org/apache/hadoop/hdfs/server/common/ src/hdfs/org/apache/hadoop/hdfs/server/datanode/ src/hdfs/org/apache/hadoop/hdfs/server/namenode/
Date Thu, 17 Jul 2008 18:22:35 GMT
Author: rangadi
Date: Thu Jul 17 11:22:34 2008
New Revision: 677665

URL: http://svn.apache.org/viewvc?rev=677665&view=rev
Log:
HADOOP-3677. Simplify generation stamp upgrade by making is a
local upgrade on datandodes. Deleted distributed upgrade. (rangadi)

Removed:
    hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/server/common/GenerationStampStatsUpgradeCommand.java
    hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/server/datanode/GenerationStampUpgradeDatanode.java
    hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/server/namenode/GenerationStampUpgradeNamenode.java
Modified:
    hadoop/core/trunk/CHANGES.txt
    hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/server/common/Storage.java
    hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/server/common/UpgradeObjectCollection.java
    hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/server/datanode/DataStorage.java
    hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/server/datanode/FSDataset.java

Modified: hadoop/core/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/CHANGES.txt?rev=677665&r1=677664&r2=677665&view=diff
==============================================================================
--- hadoop/core/trunk/CHANGES.txt (original)
+++ hadoop/core/trunk/CHANGES.txt Thu Jul 17 11:22:34 2008
@@ -307,6 +307,10 @@
     in hadoop user guide. (shv)
 
   IMPROVEMENTS
+
+    HADOOP-3677. Simplify generation stamp upgrade by making is a 
+    local upgrade on datandodes. Deleted distributed upgrade.
+    (rangadi)
    
     HADOOP-2928. Remove deprecated FileSystem.getContentLength().
     (Lohit Vjayarenu via rangadi)

Modified: hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/server/common/Storage.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/server/common/Storage.java?rev=677665&r1=677664&r2=677665&view=diff
==============================================================================
--- hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/server/common/Storage.java (original)
+++ hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/server/common/Storage.java Thu Jul 17
11:22:34 2008
@@ -69,6 +69,10 @@
   // this corresponds to Hadoop-0.14.
   public static final int LAST_UPGRADABLE_LAYOUT_VERSION = -7;
   protected static final String LAST_UPGRADABLE_HADOOP_VERSION = "Hadoop-0.14";
+
+  /* this should be removed when LAST_UPGRADABLE_LV goes beyond -13.
+   * any upgrade code that uses this constant should also be removed. */
+  public static final int PRE_GENERATIONSTAMP_LAYOUT_VERSION = -13;
   
   private   static final String STORAGE_FILE_LOCK     = "in_use.lock";
   protected static final String STORAGE_FILE_VERSION  = "VERSION";

Modified: hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/server/common/UpgradeObjectCollection.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/server/common/UpgradeObjectCollection.java?rev=677665&r1=677664&r2=677665&view=diff
==============================================================================
--- hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/server/common/UpgradeObjectCollection.java
(original)
+++ hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/server/common/UpgradeObjectCollection.java
Thu Jul 17 11:22:34 2008
@@ -22,8 +22,6 @@
 import java.util.TreeSet;
 
 import org.apache.hadoop.hdfs.protocol.FSConstants;
-import org.apache.hadoop.hdfs.server.datanode.GenerationStampUpgradeDatanode;
-import org.apache.hadoop.hdfs.server.namenode.GenerationStampUpgradeNamenode;
 import org.apache.hadoop.util.StringUtils;
 
 /**
@@ -36,8 +34,6 @@
     initialize();
     // Registered distributed upgrade objects here
     // registerUpgrade(new UpgradeObject());
-    registerUpgrade(new GenerationStampUpgradeNamenode());
-    registerUpgrade(new GenerationStampUpgradeDatanode());
   }
 
   static class UOSignature implements Comparable<UOSignature> {

Modified: hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/server/datanode/DataStorage.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/server/datanode/DataStorage.java?rev=677665&r1=677664&r2=677665&view=diff
==============================================================================
--- hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/server/datanode/DataStorage.java (original)
+++ hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/server/datanode/DataStorage.java Thu
Jul 17 11:22:34 2008
@@ -28,7 +28,10 @@
 import java.util.ArrayList;
 import java.util.Iterator;
 import java.util.Properties;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
 
+import org.apache.hadoop.hdfs.protocol.Block;
 import org.apache.hadoop.hdfs.protocol.FSConstants;
 import org.apache.hadoop.hdfs.protocol.FSConstants.NodeType;
 import org.apache.hadoop.hdfs.protocol.FSConstants.StartupOption;
@@ -275,7 +278,7 @@
     // rename current to tmp
     rename(curDir, tmpDir);
     // hardlink blocks
-    linkBlocks(tmpDir, curDir);
+    linkBlocks(tmpDir, curDir, this.getLayoutVersion());
     // write version file
     this.layoutVersion = FSConstants.LAYOUT_VERSION;
     assert this.namespaceID == nsInfo.getNamespaceID() :
@@ -357,12 +360,19 @@
     }
   }
   
-  static void linkBlocks(File from, File to) throws IOException {
+  static void linkBlocks(File from, File to, int oldLV) throws IOException {
     if (!from.isDirectory()) {
       if (from.getName().startsWith(COPY_FILE_PREFIX)) {
         IOUtils.copyBytes(new FileInputStream(from), 
                           new FileOutputStream(to), 16*1024, true);
       } else {
+        
+        //check if we are upgrading from pre-generation stamp version.
+        if (oldLV >= PRE_GENERATIONSTAMP_LAYOUT_VERSION) {
+          // Link to the new file name.
+          to = new File(convertMetatadataFileName(to.getAbsolutePath()));
+        }
+        
         HardLink.createHardLink(from, to);
       }
       return;
@@ -379,7 +389,8 @@
       });
     
     for(int i = 0; i < blockNames.length; i++)
-      linkBlocks(new File(from, blockNames[i]), new File(to, blockNames[i]));
+      linkBlocks(new File(from, blockNames[i]), 
+                 new File(to, blockNames[i]), oldLV);
   }
 
   protected void corruptPreUpgradeStorage(File rootDir) throws IOException {
@@ -406,4 +417,22 @@
     um.setUpgradeState(false, getLayoutVersion());
     um.initializeUpgrade(nsInfo);
   }
+  
+  private static final Pattern PRE_GENSTAMP_META_FILE_PATTERN = 
+    Pattern.compile("(.*blk_[-]*\\d+)\\.meta$");
+  /**
+   * This is invoked on target file names when upgrading from pre generation 
+   * stamp version (version -13) to correct the metatadata file name.
+   * @param oldFileName
+   * @return the new metadata file name with the default generation stamp.
+   */
+  private static String convertMetatadataFileName(String oldFileName) {
+    Matcher matcher = PRE_GENSTAMP_META_FILE_PATTERN.matcher(oldFileName); 
+    if (matcher.matches()) {
+      //return the current metadata file name
+      return FSDataset.getMetaFileName(matcher.group(1),
+                                       Block.GRANDFATHER_GENERATION_STAMP); 
+    }
+    return oldFileName;
+  }
 }

Modified: hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/server/datanode/FSDataset.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/server/datanode/FSDataset.java?rev=677665&r1=677664&r2=677665&view=diff
==============================================================================
--- hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/server/datanode/FSDataset.java (original)
+++ hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/server/datanode/FSDataset.java Thu Jul
17 11:22:34 2008
@@ -564,9 +564,13 @@
     }
   } 
   
+  static String getMetaFileName(String blockFileName, long genStamp) {
+    return blockFileName + "_" + genStamp + METADATA_EXTENSION;
+  }
+  
   static File getMetaFile(File f , Block b) {
-    return new File( f.getAbsolutePath() +
-                     "_" + b.getGenerationStamp() + METADATA_EXTENSION ); 
+    return new File(getMetaFileName(f.getAbsolutePath(),
+                                    b.getGenerationStamp())); 
   }
   protected File getMetaFile(Block b) throws IOException {
     return getMetaFile(getBlockFile(b), b);



Mime
View raw message