hadoop-hdfs-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From s..@apache.org
Subject svn commit: r1067079 - in /hadoop/hdfs/trunk: CHANGES.txt src/java/org/apache/hadoop/hdfs/server/namenode/BlockManager.java src/java/org/apache/hadoop/hdfs/server/namenode/DatanodeDescriptor.java
Date Fri, 04 Feb 2011 04:09:00 GMT
Author: shv
Date: Fri Feb  4 04:09:00 2011
New Revision: 1067079

URL: http://svn.apache.org/viewvc?rev=1067079&view=rev
Log:
HDFS-900. Corrupt replicas are not processed correctly in block report. Contributed by Konstantin
Shvachko.

Modified:
    hadoop/hdfs/trunk/CHANGES.txt
    hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/BlockManager.java
    hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/DatanodeDescriptor.java

Modified: hadoop/hdfs/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/CHANGES.txt?rev=1067079&r1=1067078&r2=1067079&view=diff
==============================================================================
--- hadoop/hdfs/trunk/CHANGES.txt (original)
+++ hadoop/hdfs/trunk/CHANGES.txt Fri Feb  4 04:09:00 2011
@@ -523,6 +523,8 @@ Release 0.22.0 - Unreleased
 
     HDFS-1591. HDFS part of HADOOP-6642. (Chris Douglas, Po Cheung via shv)
 
+    HDFS-900. Corrupt replicas are not processed correctly in block report (shv)
+
 Release 0.21.1 - Unreleased
 
   IMPROVEMENTS

Modified: hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/BlockManager.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/BlockManager.java?rev=1067079&r1=1067078&r2=1067079&view=diff
==============================================================================
--- hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/BlockManager.java (original)
+++ hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/BlockManager.java Fri
Feb  4 04:09:00 2011
@@ -397,8 +397,7 @@ public class BlockManager {
       blocksMap.nodeIterator(block); it.hasNext();) {
       String storageID = it.next().getStorageID();
       // filter invalidate replicas
-      Collection<Block> blocks = recentInvalidateSets.get(storageID);
-      if(blocks==null || !blocks.contains(block)) {
+      if( ! belongsToInvalidates(storageID, block)) {
         machineSet.add(storageID);
       }
     }
@@ -496,16 +495,21 @@ public class BlockManager {
                             minReplication);
   }
 
-  void removeFromInvalidates(String datanodeId, Block block) {
-    Collection<Block> v = recentInvalidateSets.get(datanodeId);
+  void removeFromInvalidates(String storageID, Block block) {
+    Collection<Block> v = recentInvalidateSets.get(storageID);
     if (v != null && v.remove(block)) {
       pendingDeletionBlocksCount--;
       if (v.isEmpty()) {
-        recentInvalidateSets.remove(datanodeId);
+        recentInvalidateSets.remove(storageID);
       }
     }
   }
 
+  boolean belongsToInvalidates(String storageID, Block block) {
+    Collection<Block> invalidateSet = recentInvalidateSets.get(storageID);
+    return invalidateSet != null && invalidateSet.contains(block);
+  }
+
   /**
    * Adds block to list of blocks which will be invalidated on specified
    * datanode

Modified: hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/DatanodeDescriptor.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/DatanodeDescriptor.java?rev=1067079&r1=1067078&r2=1067079&view=diff
==============================================================================
--- hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/DatanodeDescriptor.java
(original)
+++ hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/DatanodeDescriptor.java
Fri Feb  4 04:09:00 2011
@@ -505,6 +505,13 @@ public class DatanodeDescriptor extends 
           storedBlock.getBlockUCState());
     }
 
+    // Ignore replicas already scheduled to be removed from the DN
+    if(blockManager.belongsToInvalidates(getStorageID(), block)) {
+      assert storedBlock.findDatanode(this) < 0 : "Block " + block 
+        + " in recentInvalidatesSet should not appear in DN " + this;
+      return storedBlock;
+    }
+
     // Block is on the DN
     boolean isCorrupt = false;
     switch(rState) {



Mime
View raw message