hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From vino...@apache.org
Subject git commit: YARN-2699. Fixed a bug in CommonNodeLabelsManager that caused tests to fail when using ephemeral ports on NodeIDs. Contributed by Wangda Tan.
Date Fri, 17 Oct 2014 15:58:44 GMT
Repository: hadoop
Updated Branches:
  refs/heads/trunk 368743140 -> abae63caf


YARN-2699. Fixed a bug in CommonNodeLabelsManager that caused tests to fail when using ephemeral
ports on NodeIDs. Contributed by Wangda Tan.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/abae63ca
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/abae63ca
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/abae63ca

Branch: refs/heads/trunk
Commit: abae63caf9c53b404f2f2db7d482555484eaeaf8
Parents: 3687431
Author: Vinod Kumar Vavilapalli <vinodkv@apache.org>
Authored: Fri Oct 17 08:58:08 2014 -0700
Committer: Vinod Kumar Vavilapalli <vinodkv@apache.org>
Committed: Fri Oct 17 08:58:08 2014 -0700

----------------------------------------------------------------------
 hadoop-yarn-project/CHANGES.txt                 |  3 ++
 .../nodelabels/CommonNodeLabelsManager.java     | 37 ++++++++++----------
 .../nodelabels/RMNodeLabelsManager.java         | 12 +++++--
 .../nodelabels/TestRMNodeLabelsManager.java     |  8 +++++
 4 files changed, 40 insertions(+), 20 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/abae63ca/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index e8b0ef5..5056470 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -594,6 +594,9 @@ Release 2.6.0 - UNRELEASED
     tracking per label when a host runs multiple node-managers. (Wangda Tan via
     vinodkv)
 
+    YARN-2699. Fixed a bug in CommonNodeLabelsManager that caused tests to fail
+    when using ephemeral ports on NodeIDs. (Wangda Tan via vinodkv)
+
   BREAKDOWN OF YARN-1051 SUBTASKS AND RELATED JIRAS
 
     YARN-1707. Introduce APIs to add/remove/resize queues in the

http://git-wip-us.apache.org/repos/asf/hadoop/blob/abae63ca/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/CommonNodeLabelsManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/CommonNodeLabelsManager.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/CommonNodeLabelsManager.java
index 511b5ee..8bb88f2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/CommonNodeLabelsManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/CommonNodeLabelsManager.java
@@ -299,14 +299,14 @@ public class CommonNodeLabelsManager extends AbstractService {
     for (Entry<NodeId, Set<String>> entry : addedLabelsToNode.entrySet()) {
       NodeId nodeId = entry.getKey();
       Set<String> labels = entry.getValue();
-      
-      createNodeIfNonExisted(entry.getKey());
-      
+ 
+      createHostIfNonExisted(nodeId.getHost());
       if (nodeId.getPort() == WILDCARD_PORT) {
         Host host = nodeCollections.get(nodeId.getHost());
         host.labels.addAll(labels);
         newNMToLabels.put(nodeId, host.labels);
       } else {
+        createNodeIfNonExisted(nodeId);
         Node nm = getNMInNodeSet(nodeId);
         if (nm.labels == null) {
           nm.labels = new HashSet<String>();
@@ -534,21 +534,21 @@ public class CommonNodeLabelsManager extends AbstractService {
   
   @SuppressWarnings("unchecked")
   protected void internalReplaceLabelsOnNode(
-      Map<NodeId, Set<String>> replaceLabelsToNode) {
+      Map<NodeId, Set<String>> replaceLabelsToNode) throws IOException {
     // do replace labels to nodes
     Map<NodeId, Set<String>> newNMToLabels = new HashMap<NodeId, Set<String>>();
     for (Entry<NodeId, Set<String>> entry : replaceLabelsToNode.entrySet()) {
       NodeId nodeId = entry.getKey();
       Set<String> labels = entry.getValue();
 
-      // update nodeCollections
-      createNodeIfNonExisted(entry.getKey());
+      createHostIfNonExisted(nodeId.getHost());      
       if (nodeId.getPort() == WILDCARD_PORT) {
         Host host = nodeCollections.get(nodeId.getHost());
         host.labels.clear();
         host.labels.addAll(labels);
         newNMToLabels.put(nodeId, host.labels);
       } else {
+        createNodeIfNonExisted(nodeId);
         Node nm = getNMInNodeSet(nodeId);
         if (nm.labels == null) {
           nm.labels = new HashSet<String>();
@@ -672,10 +672,6 @@ public class CommonNodeLabelsManager extends AbstractService {
 
   protected Node getNMInNodeSet(NodeId nodeId, Map<String, Host> map,
       boolean checkRunning) {
-    if (WILDCARD_PORT == nodeId.getPort()) {
-      return null;
-    }
-    
     Host host = map.get(nodeId.getHost());
     if (null == host) {
       return null;
@@ -707,17 +703,22 @@ public class CommonNodeLabelsManager extends AbstractService {
     }
   }
   
-  protected void createNodeIfNonExisted(NodeId nodeId) {
+  protected void createNodeIfNonExisted(NodeId nodeId) throws IOException {
     Host host = nodeCollections.get(nodeId.getHost());
     if (null == host) {
-      host = new Host();
-      nodeCollections.put(nodeId.getHost(), host);
+      throw new IOException("Should create host before creating node.");
     }
-    if (nodeId.getPort() != WILDCARD_PORT) {
-      Node nm = host.nms.get(nodeId);
-      if (null == nm) {
-        host.nms.put(nodeId, new Node());
-      }
+    Node nm = host.nms.get(nodeId);
+    if (null == nm) {
+      host.nms.put(nodeId, new Node());
+    }
+  }
+  
+  protected void createHostIfNonExisted(String hostName) {
+    Host host = nodeCollections.get(hostName);
+    if (null == host) {
+      host = new Host();
+      nodeCollections.put(hostName, host);
     }
   }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/abae63ca/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/RMNodeLabelsManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/RMNodeLabelsManager.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/RMNodeLabelsManager.java
index 1d7f6f1..ba1727c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/RMNodeLabelsManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/RMNodeLabelsManager.java
@@ -181,7 +181,15 @@ public class RMNodeLabelsManager extends CommonNodeLabelsManager {
       // save if we have a node before
       Map<String, Host> before = cloneNodeMap(ImmutableSet.of(nodeId));
       
-      createNodeIfNonExisted(nodeId);
+      createHostIfNonExisted(nodeId.getHost());
+      try {
+        createNodeIfNonExisted(nodeId);
+      } catch (IOException e) {
+        LOG.error("This shouldn't happen, cannot get host in nodeCollection"
+            + " associated to the node being activated");
+        return;
+      }
+
       Node nm = getNMInNodeSet(nodeId);
       nm.resource = resource;
       nm.running = true;
@@ -220,7 +228,7 @@ public class RMNodeLabelsManager extends CommonNodeLabelsManager {
     }
   }
 
-  public void updateNodeResource(NodeId node, Resource newResource) {
+  public void updateNodeResource(NodeId node, Resource newResource) throws IOException {
     deactivateNode(node);
     activateNode(node, newResource);
   }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/abae63ca/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/TestRMNodeLabelsManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/TestRMNodeLabelsManager.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/TestRMNodeLabelsManager.java
index 81eead9..1fbe968 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/TestRMNodeLabelsManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/TestRMNodeLabelsManager.java
@@ -96,6 +96,14 @@ public class TestRMNodeLabelsManager extends NodeLabelTestBase {
     Assert.assertEquals(mgr.getResourceByLabel(RMNodeLabelsManager.NO_LABEL, null),
         Resources.add(SMALL_RESOURCE, LARGE_NODE));
   }
+  
+  @Test(timeout = 5000)
+  public void testActivateNodeManagerWithZeroPort() throws Exception {
+    // active two NM, one is zero port , another is non-zero port. no exception
+    // should be raised
+    mgr.activateNode(NodeId.newInstance("n1", 0), SMALL_RESOURCE);
+    mgr.activateNode(NodeId.newInstance("n1", 2), LARGE_NODE);
+  }
 
   @SuppressWarnings({ "unchecked", "rawtypes" })
   @Test(timeout = 5000)


Mime
View raw message