hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From z..@apache.org
Subject hadoop git commit: HADOOP-11676. Add API to NetworkTopology for getting all racks. Contributed by Walter Su.
Date Mon, 16 Mar 2015 21:20:13 GMT
Repository: hadoop
Updated Branches:
  refs/heads/HDFS-7285 4a6ab3758 -> c7a2f43e5


HADOOP-11676. Add API to NetworkTopology for getting all racks. Contributed by Walter Su.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c7a2f43e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c7a2f43e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c7a2f43e

Branch: refs/heads/HDFS-7285
Commit: c7a2f43e5ba18dad53ca4d1809a994800488368c
Parents: 4a6ab37
Author: Zhe Zhang <zhz@apache.org>
Authored: Mon Mar 16 14:18:17 2015 -0700
Committer: Zhe Zhang <zhz@apache.org>
Committed: Mon Mar 16 14:18:17 2015 -0700

----------------------------------------------------------------------
 .../org/apache/hadoop/net/NetworkTopology.java  | 24 +++++++++++++++-----
 .../net/NetworkTopologyWithNodeGroup.java       |  5 ++--
 .../apache/hadoop/net/TestNetworkTopology.java  | 23 +++++++++++++++++++
 3 files changed, 43 insertions(+), 9 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/c7a2f43e/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetworkTopology.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetworkTopology.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetworkTopology.java
index c60cc0b..311378b 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetworkTopology.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetworkTopology.java
@@ -376,8 +376,8 @@ public class NetworkTopology {
   InnerNode clusterMap;
   /** Depth of all leaf nodes */
   private int depthOfAllLeaves = -1;
-  /** rack counter */
-  protected int numOfRacks = 0;
+  
+  protected ArrayList<String> racks = new ArrayList<>();
   /** the lock used to manage access */
   protected ReadWriteLock netlock = new ReentrantReadWriteLock();
 
@@ -417,7 +417,7 @@ public class NetworkTopology {
       if (clusterMap.add(node)) {
         LOG.info("Adding a new node: "+NodeBase.getPath(node));
         if (rack == null) {
-          numOfRacks++;
+          racks.add(node.getNetworkLocation());
         }
         if (!(node instanceof InnerNode)) {
           if (depthOfAllLeaves == -1) {
@@ -490,7 +490,7 @@ public class NetworkTopology {
       if (clusterMap.remove(node)) {
         InnerNode rack = (InnerNode)getNode(node.getNetworkLocation());
         if (rack == null) {
-          numOfRacks--;
+          racks.remove(node.getNetworkLocation());
         }
       }
       if(LOG.isDebugEnabled()) {
@@ -554,12 +554,24 @@ public class NetworkTopology {
   public String getRack(String loc) {
     return loc;
   }
+
+  /** @return a copy of the array of all the racks. Changing the returned array
+   * will not affect the topology.
+   */
+  public String[] getRacks() {
+    netlock.readLock().lock();
+    try {
+      return racks.toArray(new String[racks.size()]);
+    } finally {
+      netlock.readLock().unlock();
+    }
+  }
   
   /** @return the total number of racks */
   public int getNumOfRacks() {
     netlock.readLock().lock();
     try {
-      return numOfRacks;
+      return racks.size();
     } finally {
       netlock.readLock().unlock();
     }
@@ -807,7 +819,7 @@ public class NetworkTopology {
     // print the number of racks
     StringBuilder tree = new StringBuilder();
     tree.append("Number of racks: ");
-    tree.append(numOfRacks);
+    tree.append(racks.size());
     tree.append("\n");
     // print the number of leaves
     int numOfLeaves = getNumOfLeaves();

http://git-wip-us.apache.org/repos/asf/hadoop/blob/c7a2f43e/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetworkTopologyWithNodeGroup.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetworkTopologyWithNodeGroup.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetworkTopologyWithNodeGroup.java
index 3de49dc..f32eaf6 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetworkTopologyWithNodeGroup.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetworkTopologyWithNodeGroup.java
@@ -204,8 +204,7 @@ public class NetworkTopologyWithNodeGroup extends NetworkTopology {
       if (clusterMap.add(node)) {
         LOG.info("Adding a new node: " + NodeBase.getPath(node));
         if (rack == null) {
-          // We only track rack number here
-          numOfRacks++;
+          racks.add(node.getNetworkLocation());
         }
       }
       if(LOG.isDebugEnabled()) {
@@ -237,7 +236,7 @@ public class NetworkTopologyWithNodeGroup extends NetworkTopology {
         }
         InnerNode rack = (InnerNode)getNode(nodeGroup.getNetworkLocation());
         if (rack == null) {
-          numOfRacks--;
+          racks.remove(node.getNetworkLocation());
         }
       }
       if(LOG.isDebugEnabled()) {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/c7a2f43e/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/net/TestNetworkTopology.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/net/TestNetworkTopology.java
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/net/TestNetworkTopology.java
index 1758807..91562a1 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/net/TestNetworkTopology.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/net/TestNetworkTopology.java
@@ -23,7 +23,9 @@ import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
 
+import java.util.Arrays;
 import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
 
 import org.apache.commons.logging.Log;
@@ -121,6 +123,27 @@ public class TestNetworkTopology {
     assertTrue(cluster.isOnSameRack(dataNodes[3], dataNodes[4]));
     assertFalse(cluster.isOnSameRack(dataNodes[4], dataNodes[5]));
     assertTrue(cluster.isOnSameRack(dataNodes[5], dataNodes[6]));
+    
+    List<String> rackList = Arrays.asList(cluster.getRacks());
+    for (int i = 0; i < dataNodes.length; i++) {
+      assertTrue(rackList.contains(dataNodes[i].getNetworkLocation()));
+    }
+    
+    cluster.remove(dataNodes[0]);
+    rackList = Arrays.asList(cluster.getRacks());
+    //Notice that DN0 and DN1 are on the same rack.
+    for (int i = 0; i < dataNodes.length; i++) {
+      assertTrue(rackList.contains(dataNodes[i].getNetworkLocation()));
+    }
+    
+    cluster.remove(dataNodes[1]);
+    rackList = Arrays.asList(cluster.getRacks());
+    assertFalse(rackList.contains(dataNodes[0].getNetworkLocation()));
+    assertFalse(rackList.contains(dataNodes[1].getNetworkLocation()));
+    for (int i = 2; i < dataNodes.length; i++) {
+      assertTrue(rackList.contains(dataNodes[i].getNetworkLocation()));
+    }
+    
   }
   
   @Test


Mime
View raw message