hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From hair...@apache.org
Subject svn commit: r762864 - in /hadoop/core/trunk: CHANGES.txt src/hdfs/org/apache/hadoop/hdfs/server/namenode/ReplicationTargetChooser.java
Date Tue, 07 Apr 2009 17:38:17 GMT
Author: hairong
Date: Tue Apr  7 17:38:17 2009
New Revision: 762864

URL: http://svn.apache.org/viewvc?rev=762864&view=rev
Log:
HADOOP-5603. Improve NameNode's block placement performance by reducing the number of iterations
on datanodes. Contributed by Hairong Kuang.

Modified:
    hadoop/core/trunk/CHANGES.txt
    hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/server/namenode/ReplicationTargetChooser.java

Modified: hadoop/core/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/CHANGES.txt?rev=762864&r1=762863&r2=762864&view=diff
==============================================================================
--- hadoop/core/trunk/CHANGES.txt (original)
+++ hadoop/core/trunk/CHANGES.txt Tue Apr  7 17:38:17 2009
@@ -214,6 +214,8 @@
     HADOOP-5595. NameNode does not need to run a replicator to choose a
     random DataNode. (hairong)
 
+    HADOOP-5603. Improve NameNode's block placement performance. (hairong)
+
   BUG FIXES
     
     HADOOP-5379. CBZip2InputStream to throw IOException on data crc error.

Modified: hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/server/namenode/ReplicationTargetChooser.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/server/namenode/ReplicationTargetChooser.java?rev=762864&r1=762863&r2=762864&view=diff
==============================================================================
--- hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/server/namenode/ReplicationTargetChooser.java
(original)
+++ hadoop/core/trunk/src/hdfs/org/apache/hadoop/hdfs/server/namenode/ReplicationTargetChooser.java
Tue Apr  7 17:38:17 2009
@@ -308,18 +308,24 @@
                                           int maxNodesPerRack,
                                           List<DatanodeDescriptor> results) 
     throws NotEnoughReplicasException {
-    DatanodeDescriptor result;
-    do {
-      DatanodeDescriptor[] selectedNodes = 
-        chooseRandom(1, nodes, excludedNodes);
-      if (selectedNodes.length == 0) {
-        throw new NotEnoughReplicasException(
-                                             "Not able to place enough replicas");
-      }
-      result = (DatanodeDescriptor)(selectedNodes[0]);
-    } while(!isGoodTarget(result, blocksize, maxNodesPerRack, results));
-    results.add(result);
-    return result;
+    int numOfAvailableNodes =
+      clusterMap.countNumOfAvailableNodes(nodes, excludedNodes);
+    while(numOfAvailableNodes > 0) {
+      DatanodeDescriptor choosenNode = 
+        (DatanodeDescriptor)(clusterMap.chooseRandom(nodes));
+      if (!excludedNodes.contains(choosenNode)) {
+        excludedNodes.add(choosenNode);
+        numOfAvailableNodes--;
+
+        if (isGoodTarget(choosenNode, blocksize, maxNodesPerRack, results)) {
+          results.add(choosenNode);
+          return choosenNode;
+        }
+      }
+    }
+
+    throw new NotEnoughReplicasException(
+        "Not able to place enough replicas");
   }
     
   /* Randomly choose <i>numOfReplicas</i> targets from <i>nodes</i>.
@@ -331,51 +337,27 @@
                             int maxNodesPerRack,
                             List<DatanodeDescriptor> results)
     throws NotEnoughReplicasException {
-    boolean toContinue = true;
-    do {
-      DatanodeDescriptor[] selectedNodes = 
-        chooseRandom(numOfReplicas, nodes, excludedNodes);
-      if (selectedNodes.length < numOfReplicas) {
-        toContinue = false;
-      }
-      for(int i=0; i<selectedNodes.length; i++) {
-        DatanodeDescriptor result = selectedNodes[i];
-        if (isGoodTarget(result, blocksize, maxNodesPerRack, results)) {
-          numOfReplicas--;
-          results.add(result);
-        }
-      } // end of for
-    } while (numOfReplicas>0 && toContinue);
       
-    if (numOfReplicas>0) {
-      throw new NotEnoughReplicasException(
-                                           "Not able to place enough replicas");
-    }
-  }
-    
-  /* Randomly choose <i>numOfNodes</i> nodes from <i>scope</i>.
-   * @return the choosen nodes
-   */
-  private DatanodeDescriptor[] chooseRandom(int numOfReplicas, 
-                                            String nodes,
-                                            List<Node> excludedNodes) {
-    List<DatanodeDescriptor> results = 
-      new ArrayList<DatanodeDescriptor>();
     int numOfAvailableNodes =
       clusterMap.countNumOfAvailableNodes(nodes, excludedNodes);
-    numOfReplicas = (numOfAvailableNodes<numOfReplicas)?
-      numOfAvailableNodes:numOfReplicas;
-    while(numOfReplicas > 0) {
+    while(numOfReplicas > 0 && numOfAvailableNodes > 0) {
       DatanodeDescriptor choosenNode = 
         (DatanodeDescriptor)(clusterMap.chooseRandom(nodes));
       if (!excludedNodes.contains(choosenNode)) {
-        results.add(choosenNode);
         excludedNodes.add(choosenNode);
-        numOfReplicas--;
+        numOfAvailableNodes--;
+
+        if (isGoodTarget(choosenNode, blocksize, maxNodesPerRack, results)) {
+          numOfReplicas--;
+          results.add(choosenNode);
+        }
       }
     }
-    return (DatanodeDescriptor[])results.toArray(
-                                                 new DatanodeDescriptor[results.size()]);
   
+      
+    if (numOfReplicas>0) {
+      throw new NotEnoughReplicasException(
+                                           "Not able to place enough replicas");
+    }
   }
     
   /* judge if a node is a good target.



Mime
View raw message