hadoop-hdfs-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From omal...@apache.org
Subject svn commit: r1089313 - in /hadoop/hdfs/branches/yahoo-merge: ./ src/java/org/apache/hadoop/hdfs/server/datanode/ src/test/hdfs/org/apache/hadoop/hdfs/
Date Wed, 06 Apr 2011 06:22:24 GMT
Author: omalley
Date: Wed Apr  6 06:22:23 2011
New Revision: 1089313

URL: http://svn.apache.org/viewvc?rev=1089313&view=rev
Log:
commit 34db2b0edd6b01d4792ca13c86d9a0258be01a99
Author: Owen O'Malley <omalley@apache.org>
Date:   Tue Apr 5 11:30:35 2011 -0700

    HDFS-1445: Fast Snapshot via batch hardlinks.
    Requires coordinated change in hadoop-common. Contributed by Matt Foley.
    
    Conflicts:
    
    	YAHOO-CHANGES.txt
    	src/java/org/apache/hadoop/hdfs/server/datanode/DataStorage.java
    	src/test/hdfs/org/apache/hadoop/hdfs/TestFileAppend.java

Modified:
    hadoop/hdfs/branches/yahoo-merge/CHANGES.txt
    hadoop/hdfs/branches/yahoo-merge/src/java/org/apache/hadoop/hdfs/server/datanode/DataStorage.java
    hadoop/hdfs/branches/yahoo-merge/src/java/org/apache/hadoop/hdfs/server/datanode/ReplicaInfo.java
    hadoop/hdfs/branches/yahoo-merge/src/test/hdfs/org/apache/hadoop/hdfs/TestFileAppend.java

Modified: hadoop/hdfs/branches/yahoo-merge/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/yahoo-merge/CHANGES.txt?rev=1089313&r1=1089312&r2=1089313&view=diff
==============================================================================
--- hadoop/hdfs/branches/yahoo-merge/CHANGES.txt (original)
+++ hadoop/hdfs/branches/yahoo-merge/CHANGES.txt Wed Apr  6 06:22:23 2011
@@ -244,6 +244,8 @@ Trunk (unreleased changes)
 
     HDFS-1684. Balancer cannot start with with multiple namenodes.  (szetszwo)
 
+    HDFS-1445: Fast Snapshot via batch hardlinks. (Matt Foley)
+
 Release 0.22.0 - Unreleased
 
   NEW FEATURES

Modified: hadoop/hdfs/branches/yahoo-merge/src/java/org/apache/hadoop/hdfs/server/datanode/DataStorage.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/yahoo-merge/src/java/org/apache/hadoop/hdfs/server/datanode/DataStorage.java?rev=1089313&r1=1089312&r2=1089313&view=diff
==============================================================================
--- hadoop/hdfs/branches/yahoo-merge/src/java/org/apache/hadoop/hdfs/server/datanode/DataStorage.java
(original)
+++ hadoop/hdfs/branches/yahoo-merge/src/java/org/apache/hadoop/hdfs/server/datanode/DataStorage.java
Wed Apr  6 06:22:23 2011
@@ -39,6 +39,7 @@ import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.LocalFileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.FileUtil.HardLink;
+import org.apache.hadoop.fs.HardLink;
 import org.apache.hadoop.fs.permission.FsPermission;
 import org.apache.hadoop.hdfs.DFSConfigKeys;
 import org.apache.hadoop.hdfs.HdfsConfiguration;
@@ -617,24 +618,26 @@ public class DataStorage extends Storage
    * @throws IOException if error occurs during hardlink
    */
   private void linkAllBlocks(File fromDir, File toDir) throws IOException {
+    HardLink hardLink = new HardLink();
     // do the link
     int diskLayoutVersion = this.getLayoutVersion();
     if (diskLayoutVersion < PRE_RBW_LAYOUT_VERSION) { // RBW version
       // hardlink finalized blocks in tmpDir/finalized
       linkBlocks(new File(fromDir, STORAGE_DIR_FINALIZED), 
-          new File(toDir, STORAGE_DIR_FINALIZED), diskLayoutVersion);
+          new File(toDir, STORAGE_DIR_FINALIZED), diskLayoutVersion, hardLink);
       // hardlink rbw blocks in tmpDir/finalized
       linkBlocks(new File(fromDir, STORAGE_DIR_RBW), 
-          new File(toDir, STORAGE_DIR_RBW), diskLayoutVersion);
+          new File(toDir, STORAGE_DIR_RBW), diskLayoutVersion, hardLink);
     } else { // pre-RBW version
       // hardlink finalized blocks in tmpDir
-      linkBlocks(fromDir, 
-          new File(toDir, STORAGE_DIR_FINALIZED), diskLayoutVersion);      
-    }    
+      linkBlocks(fromDir, new File(toDir, STORAGE_DIR_FINALIZED), 
+          diskLayoutVersion, hardLink);      
+    } 
+    LOG.info( hardLink.linkStats.report() );
   }
   
-  static void linkBlocks(File from, File to, int oldLV)
-      throws IOException {
+  static void linkBlocks(File from, File to, int oldLV, HardLink hl) 
+  throws IOException {
     if (!from.exists()) {
       return;
     }
@@ -645,6 +648,7 @@ public class DataStorage extends Storage
           FileOutputStream out = new FileOutputStream(to);
           try {
             IOUtils.copyBytes(in, out, 16*1024);
+            hl.linkStats.countPhysicalFileCopies++;
           } finally {
             out.close();
           }
@@ -660,23 +664,60 @@ public class DataStorage extends Storage
         }
         
         HardLink.createHardLink(from, to);
+        hl.linkStats.countSingleLinks++;
       }
       return;
     }
     // from is a directory
+    hl.linkStats.countDirs++;
+    
     if (!to.mkdirs())
       throw new IOException("Cannot create directory " + to);
-    String[] blockNames = from.list(new java.io.FilenameFilter() {
+    
+    //If upgrading from old stuff, need to munge the filenames.  That has to
+    //be done one file at a time, so hardlink them one at a time (slow).
+    if (oldLV >= PRE_GENERATIONSTAMP_LAYOUT_VERSION) {
+      String[] blockNames = from.list(new java.io.FilenameFilter() {
+          public boolean accept(File dir, String name) {
+            return name.startsWith(BLOCK_SUBDIR_PREFIX) 
+              || name.startsWith(BLOCK_FILE_PREFIX)
+              || name.startsWith(COPY_FILE_PREFIX);
+          }
+        });
+      if (blockNames.length == 0) {
+        hl.linkStats.countEmptyDirs++;
+      }
+      else for(int i = 0; i < blockNames.length; i++)
+        linkBlocks(new File(from, blockNames[i]), 
+            new File(to, blockNames[i]), oldLV, hl);
+    } 
+    else {
+      //If upgrading from a relatively new version, we only need to create
+      //links with the same filename.  This can be done in bulk (much faster).
+      String[] blockNames = from.list(new java.io.FilenameFilter() {
         public boolean accept(File dir, String name) {
-          return name.startsWith(BLOCK_SUBDIR_PREFIX) 
-            || name.startsWith(BLOCK_FILE_PREFIX)
-            || name.startsWith(COPY_FILE_PREFIX);
+          return name.startsWith(BLOCK_FILE_PREFIX);
         }
       });
-    
-    for(int i = 0; i < blockNames.length; i++)
-      linkBlocks(new File(from, blockNames[i]), 
-                 new File(to, blockNames[i]), oldLV);
+      if (blockNames.length > 0) {
+        HardLink.createHardLinkMult(from, blockNames, to);
+        hl.linkStats.countMultLinks++;
+        hl.linkStats.countFilesMultLinks += blockNames.length;
+      } else {
+        hl.linkStats.countEmptyDirs++;
+      }
+      
+      //now take care of the rest of the files and subdirectories
+      String[] otherNames = from.list(new java.io.FilenameFilter() {
+          public boolean accept(File dir, String name) {
+            return name.startsWith(BLOCK_SUBDIR_PREFIX) 
+              || name.startsWith(COPY_FILE_PREFIX);
+          }
+        });
+      for(int i = 0; i < otherNames.length; i++)
+        linkBlocks(new File(from, otherNames[i]), 
+            new File(to, otherNames[i]), oldLV, hl);
+    }
   }
 
   protected void corruptPreUpgradeStorage(File rootDir) throws IOException {

Modified: hadoop/hdfs/branches/yahoo-merge/src/java/org/apache/hadoop/hdfs/server/datanode/ReplicaInfo.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/yahoo-merge/src/java/org/apache/hadoop/hdfs/server/datanode/ReplicaInfo.java?rev=1089313&r1=1089312&r2=1089313&view=diff
==============================================================================
--- hadoop/hdfs/branches/yahoo-merge/src/java/org/apache/hadoop/hdfs/server/datanode/ReplicaInfo.java
(original)
+++ hadoop/hdfs/branches/yahoo-merge/src/java/org/apache/hadoop/hdfs/server/datanode/ReplicaInfo.java
Wed Apr  6 06:22:23 2011
@@ -24,7 +24,7 @@ import java.io.IOException;
 
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.fs.FileUtil;
-import org.apache.hadoop.fs.FileUtil.HardLink;
+import org.apache.hadoop.fs.HardLink;
 import org.apache.hadoop.hdfs.protocol.Block;
 import org.apache.hadoop.hdfs.server.datanode.FSDataset.FSVolume;
 import org.apache.hadoop.io.IOUtils;

Modified: hadoop/hdfs/branches/yahoo-merge/src/test/hdfs/org/apache/hadoop/hdfs/TestFileAppend.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/branches/yahoo-merge/src/test/hdfs/org/apache/hadoop/hdfs/TestFileAppend.java?rev=1089313&r1=1089312&r2=1089313&view=diff
==============================================================================
--- hadoop/hdfs/branches/yahoo-merge/src/test/hdfs/org/apache/hadoop/hdfs/TestFileAppend.java
(original)
+++ hadoop/hdfs/branches/yahoo-merge/src/test/hdfs/org/apache/hadoop/hdfs/TestFileAppend.java
Wed Apr  6 06:22:23 2011
@@ -28,9 +28,11 @@ import org.apache.hadoop.conf.Configurat
 import org.apache.hadoop.fs.BlockLocation;
 import org.apache.hadoop.fs.FSDataOutputStream;
 import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.FileUtil.HardLink;
+import org.apache.hadoop.fs.HardLink;
+import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
+import org.apache.hadoop.hdfs.protocol.Block;
 import org.apache.hadoop.hdfs.protocol.LocatedBlock;
 import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
 import org.apache.hadoop.hdfs.server.datanode.DataNode;



Mime
View raw message