hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From cutt...@apache.org
Subject svn commit: r513984 - in /lucene/hadoop/trunk: CHANGES.txt src/java/org/apache/hadoop/dfs/FSNamesystem.java
Date Fri, 02 Mar 2007 22:03:03 GMT
Author: cutting
Date: Fri Mar  2 14:03:02 2007
New Revision: 513984

URL: http://svn.apache.org/viewvc?view=rev&rev=513984
Log:
HADOOP-994.  In HDFS, limit the number of blocks invalidated at once.  Contributed by Dhruba.

Modified:
    lucene/hadoop/trunk/CHANGES.txt
    lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/FSNamesystem.java

Modified: lucene/hadoop/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/CHANGES.txt?view=diff&rev=513984&r1=513983&r2=513984
==============================================================================
--- lucene/hadoop/trunk/CHANGES.txt (original)
+++ lucene/hadoop/trunk/CHANGES.txt Fri Mar  2 14:03:02 2007
@@ -195,6 +195,10 @@
 60. HADOOP-1056.  Check HDFS include/exclude node lists with both IP
     address and hostname.  (Wendy Chien via cutting)
 
+61. HADOOP-994.  In HDFS, limit the number of blocks invalidated at
+    once.  Large lists were causing datenodes to timeout.
+    (Dhruba Borthakur via cutting) 
+
 
 Release 0.11.2 - 2007-02-16
 

Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/FSNamesystem.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/FSNamesystem.java?view=diff&rev=513984&r1=513983&r2=513984
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/FSNamesystem.java (original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/FSNamesystem.java Fri Mar  2 14:03:02
2007
@@ -2047,10 +2047,20 @@
         for (Iterator<Block> it = node.getBlockIterator(); it.hasNext(); ) {
             Block b = it.next();
 
-            if (! dir.isValidBlock(b) && ! pendingCreateBlocks.contains(b)) {
-                obsolete.add(b);
-                NameNode.stateChangeLog.debug("BLOCK* NameSystem.processReport: "
+            // 
+            // A block report can only send BLOCK_INVALIDATE_CHUNK number of
+            // blocks to be deleted. If there are more blocks to be deleted, 
+            // they are added to recentInvalidateSets and will be sent out
+            // thorugh succeeding heartbeat responses.
+            //
+            if (obsolete.size() > FSConstants.BLOCK_INVALIDATE_CHUNK) {
+                addToInvalidates(b, node);
+            } else {
+                if (! dir.isValidBlock(b) && ! pendingCreateBlocks.contains(b)) {
+                  obsolete.add(b);
+                  NameNode.stateChangeLog.debug("BLOCK* NameSystem.processReport: "
                         +"ask "+nodeID.getName()+" to delete "+b.getBlockName() );
+                }
             }
         }
         return (Block[]) obsolete.toArray(new Block[obsolete.size()]);



Mime
View raw message