hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From tomwh...@apache.org
Subject svn commit: r519437 - in /lucene/hadoop/trunk: CHANGES.txt src/java/org/apache/hadoop/dfs/FSNamesystem.java
Date Sat, 17 Mar 2007 20:18:25 GMT
Author: tomwhite
Date: Sat Mar 17 13:18:24 2007
New Revision: 519437

URL: http://svn.apache.org/viewvc?view=rev&rev=519437
Log:
HADOOP-1126.  Optimize CPU usage for under replicated blocks when cluster restarts.  Contributed
by Hairong Kuang.

Modified:
    lucene/hadoop/trunk/CHANGES.txt
    lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/FSNamesystem.java

Modified: lucene/hadoop/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/CHANGES.txt?view=diff&rev=519437&r1=519436&r2=519437
==============================================================================
--- lucene/hadoop/trunk/CHANGES.txt (original)
+++ lucene/hadoop/trunk/CHANGES.txt Sat Mar 17 13:18:24 2007
@@ -76,6 +76,9 @@
 22. HADOOP-1129.  Fix DFSClient to not hide IOExceptions in
     flush method.  (Hairong Kuang via tomwhite)
 
+23. HADOOP-1126.  Optimize CPU usage for under replicated blocks
+    when cluster restarts.  (Hairong Kuang via tomwhite)
+
 
 Release 0.12.0 - 2007-03-02
 

Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/FSNamesystem.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/FSNamesystem.java?view=diff&rev=519437&r1=519436&r2=519437
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/FSNamesystem.java (original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/FSNamesystem.java Sat Mar 17 13:18:24
2007
@@ -155,7 +155,7 @@
     // We also store pending replication-orders.
     // Set of: Block
     //
-    private UnderReplicationBlocks neededReplications = new UnderReplicationBlocks();
+    private UnderReplicatedBlocks neededReplications = new UnderReplicatedBlocks();
     private PendingReplicationBlocks pendingReplications;
 
     //
@@ -334,12 +334,12 @@
      * Blocks have replication priority, with priority 0 indicating the highest
      * Blocks have only one replicas has the highest
      */
-    private class UnderReplicationBlocks {
+    private class UnderReplicatedBlocks {
         private static final int LEVEL = 3;
         TreeSet<Block>[] priorityQueues = new TreeSet[LEVEL];
         
         /* constructor */
-        UnderReplicationBlocks() {
+        UnderReplicatedBlocks() {
             for(int i=0; i<LEVEL; i++) {
                 priorityQueues[i] = new TreeSet<Block>();
             }
@@ -369,7 +369,7 @@
         */
         private int getPriority(Block block, 
                 int curReplicas, int expectedReplicas) {
-            if (curReplicas==0 || curReplicas>=expectedReplicas) {
+            if (curReplicas<=0 || curReplicas>=expectedReplicas) {
                 return LEVEL; // no need to replicate
             } else if(curReplicas==1) {
                 return 0; // highest priority
@@ -414,16 +414,14 @@
         /* remove a block from a under replication queue */
         synchronized boolean remove(Block block, 
                 int oldReplicas, int oldExpectedReplicas) {
-            if(oldExpectedReplicas <= oldReplicas) {
-                return false;
-            }
             int priLevel = getPriority(block, oldReplicas, oldExpectedReplicas);
             return remove(block, priLevel);
         }
         
         /* remove a block from a under replication queue given a priority*/
         private boolean remove(Block block, int priLevel ) {
-            if( priorityQueues[priLevel].remove(block) ) {
+            if( priLevel >= 0 && priLevel < LEVEL 
+                    && priorityQueues[priLevel].remove(block) ) {
                 NameNode.stateChangeLog.debug(
                      "BLOCK* NameSystem.UnderReplicationBlock.remove: "
                    + "Removing block " + block.getBlockName()
@@ -3536,6 +3534,8 @@
         NameNode.stateChangeLog.info("STATE* Network topology has "
                 +clusterMap.getNumOfRacks()+" racks and "
                 +clusterMap.getNumOfLeaves()+ " datanodes");
+        NameNode.stateChangeLog.info("STATE* UnderReplicatedBlocks has "
+                +neededReplications.size()+" blocks" );
       }
       
       /** 



Mime
View raw message