hadoop-hdfs-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From sur...@apache.org
Subject svn commit: r1104407 - in /hadoop/hdfs/trunk: CHANGES.txt src/java/org/apache/hadoop/hdfs/server/datanode/BlockPoolSliceStorage.java src/test/hdfs/org/apache/hadoop/hdfs/TestDFSUpgrade.java src/test/hdfs/org/apache/hadoop/hdfs/UpgradeUtilities.java
Date Tue, 17 May 2011 17:00:17 GMT
Author: suresh
Date: Tue May 17 17:00:17 2011
New Revision: 1104407

URL: http://svn.apache.org/viewvc?rev=1104407&view=rev
Log:
HDFS-1881. Federation: after taking snapshot the current directory of datanode is empty. Contributed
by Tanping Wang.


Modified:
    hadoop/hdfs/trunk/CHANGES.txt
    hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/datanode/BlockPoolSliceStorage.java
    hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/TestDFSUpgrade.java
    hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/UpgradeUtilities.java

Modified: hadoop/hdfs/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/CHANGES.txt?rev=1104407&r1=1104406&r2=1104407&view=diff
==============================================================================
--- hadoop/hdfs/trunk/CHANGES.txt (original)
+++ hadoop/hdfs/trunk/CHANGES.txt Tue May 17 17:00:17 2011
@@ -578,6 +578,9 @@ Trunk (unreleased changes)
     HDFS-1931. Update TestDFSShell for improved "du" shell command. (Daryn
     Sharp via todd)
 
+    HDFS-1881. Federation: after taking snapshot the current directory 
+    of datanode is empty. (Tanping Wang via suresh)
+
 Release 0.22.0 - Unreleased
 
   INCOMPATIBLE CHANGES

Modified: hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/datanode/BlockPoolSliceStorage.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/datanode/BlockPoolSliceStorage.java?rev=1104407&r1=1104406&r2=1104407&view=diff
==============================================================================
--- hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/datanode/BlockPoolSliceStorage.java
(original)
+++ hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/datanode/BlockPoolSliceStorage.java
Tue May 17 17:00:17 2011
@@ -465,8 +465,11 @@ public class BlockPoolSliceStorage exten
     int diskLayoutVersion = this.getLayoutVersion();
     // hardlink finalized blocks in tmpDir
     HardLink hardLink = new HardLink();
-    DataStorage.linkBlocks(fromDir, new File(toDir,
-        DataStorage.STORAGE_DIR_FINALIZED), diskLayoutVersion, hardLink);
+    DataStorage.linkBlocks(new File(fromDir, DataStorage.STORAGE_DIR_FINALIZED), 
+      new File(toDir,DataStorage.STORAGE_DIR_FINALIZED), diskLayoutVersion, hardLink);
+    DataStorage.linkBlocks(new File(fromDir, DataStorage.STORAGE_DIR_RBW), 
+        new File(toDir, DataStorage.STORAGE_DIR_RBW), diskLayoutVersion, hardLink);
+    LOG.info( hardLink.linkStats.report() );
   }
 
   private void verifyDistributedUpgradeProgress(NamespaceInfo nsInfo)

Modified: hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/TestDFSUpgrade.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/TestDFSUpgrade.java?rev=1104407&r1=1104406&r2=1104407&view=diff
==============================================================================
--- hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/TestDFSUpgrade.java (original)
+++ hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/TestDFSUpgrade.java Tue May 17
17:00:17 2011
@@ -93,10 +93,22 @@ public class TestDFSUpgrade {
       assertEquals(UpgradeUtilities.checksumContents(DATA_NODE, current),
         UpgradeUtilities.checksumMasterDataNodeContents());
       
+      // block files are placed under <sd>/current/<bpid>/current/finalized
+      File currentFinalized = 
+        MiniDFSCluster.getFinalizedDir(new File(baseDirs[i]), bpid);
+      assertEquals(UpgradeUtilities.checksumContents(DATA_NODE, currentFinalized),
+          UpgradeUtilities.checksumMasterBlockPoolFinalizedContents());
+      
       File previous = new File(baseDirs[i], "current/" + bpid + "/previous");
       assertTrue(previous.isDirectory());
       assertEquals(UpgradeUtilities.checksumContents(DATA_NODE, previous),
           UpgradeUtilities.checksumMasterDataNodeContents());
+      
+      File previousFinalized = 
+        new File(baseDirs[i], "current/" + bpid + "/previous"+"/finalized");
+      assertEquals(UpgradeUtilities.checksumContents(DATA_NODE, previousFinalized),
+          UpgradeUtilities.checksumMasterBlockPoolFinalizedContents());
+      
     }
   }
   /**

Modified: hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/UpgradeUtilities.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/UpgradeUtilities.java?rev=1104407&r1=1104406&r2=1104407&view=diff
==============================================================================
--- hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/UpgradeUtilities.java (original)
+++ hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/UpgradeUtilities.java Tue May 17
17:00:17 2011
@@ -82,6 +82,10 @@ public class UpgradeUtilities {
   private static long datanodeStorageChecksum;
   // A checksum of the contents in blockpool storage directory
   private static long blockPoolStorageChecksum;
+  // A checksum of the contents in blockpool finalize storage directory
+  private static long blockPoolFinalizedStorageChecksum;
+  // A checksum of the contents in blockpool rbw storage directory
+  private static long blockPoolRbwStorageChecksum;
 
   /**
    * Initialize the data structures used by this class.  
@@ -155,6 +159,14 @@ public class UpgradeUtilities {
     File bpCurDir = new File(BlockPoolSliceStorage.getBpRoot(bpid, dnCurDir),
         "current");
     blockPoolStorageChecksum = checksumContents(DATA_NODE, bpCurDir);
+    
+    File bpCurFinalizeDir = new File(BlockPoolSliceStorage.getBpRoot(bpid, dnCurDir),
+        "current/"+DataStorage.STORAGE_DIR_FINALIZED);
+    blockPoolFinalizedStorageChecksum = checksumContents(DATA_NODE, bpCurFinalizeDir);
+    
+    File bpCurRbwDir = new File(BlockPoolSliceStorage.getBpRoot(bpid, dnCurDir),
+        "current/"+DataStorage.STORAGE_DIR_RBW);
+    blockPoolRbwStorageChecksum = checksumContents(DATA_NODE, bpCurRbwDir);
   }
   
   // Private helper method that writes a file to the given file system.
@@ -230,6 +242,22 @@ public class UpgradeUtilities {
   }
   
   /**
+   * Return the checksum for the singleton master storage directory
+   * for finalized dir under block pool.
+   */
+  public static long checksumMasterBlockPoolFinalizedContents() {
+    return blockPoolFinalizedStorageChecksum;
+  }
+  
+  /**
+   * Return the checksum for the singleton master storage directory
+   * for rbw dir under block pool.
+   */
+  public static long checksumMasterBlockPoolRbwContents() {
+    return blockPoolRbwStorageChecksum;
+  }
+  
+  /**
    * Compute the checksum of all the files in the specified directory.
    * The contents of subdirectories are not included. This method provides
    * an easy way to ensure equality between the contents of two directories.



Mime
View raw message