hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From vino...@apache.org
Subject git commit: YARN-2685. Fixed a bug in CommonNodeLabelsManager that caused wrong resource tracking per label when a host runs multiple node-managers. Contributed by Wangda Tan.
Date Thu, 16 Oct 2014 01:48:51 GMT
Repository: hadoop
Updated Branches:
  refs/heads/branch-2 e8e3a3621 -> 76cf2250d


YARN-2685. Fixed a bug in CommonNodeLabelsManager that caused wrong resource tracking per
label when a host runs multiple node-managers. Contributed by Wangda Tan.

(cherry picked from commit b3056c266a628a65cf7ceb61b55ab3bd0a09baf2)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/76cf2250
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/76cf2250
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/76cf2250

Branch: refs/heads/branch-2
Commit: 76cf2250deed9609fd75929f1915740437395d0b
Parents: e8e3a36
Author: Vinod Kumar Vavilapalli <vinodkv@apache.org>
Authored: Wed Oct 15 18:47:26 2014 -0700
Committer: Vinod Kumar Vavilapalli <vinodkv@apache.org>
Committed: Wed Oct 15 18:48:36 2014 -0700

----------------------------------------------------------------------
 hadoop-yarn-project/CHANGES.txt                 |  4 +++
 .../nodelabels/CommonNodeLabelsManager.java     |  1 +
 .../nodelabels/TestRMNodeLabelsManager.java     | 29 ++++++++++++++++++++
 3 files changed, 34 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/76cf2250/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 9789222..c5321a6 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -553,6 +553,10 @@ Release 2.6.0 - UNRELEASED
     YARN-2628. Capacity scheduler with DominantResourceCalculator carries out
     reservation even though slots are free. (Varun Vasudev via jianhe)
 
+    YARN-2685. Fixed a bug in CommonNodeLabelsManager that caused wrong resource
+    tracking per label when a host runs multiple node-managers. (Wangda Tan via
+    vinodkv)
+
   BREAKDOWN OF YARN-1051 SUBTASKS AND RELATED JIRAS
 
     YARN-1707. Introduce APIs to add/remove/resize queues in the

http://git-wip-us.apache.org/repos/asf/hadoop/blob/76cf2250/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/CommonNodeLabelsManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/CommonNodeLabelsManager.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/CommonNodeLabelsManager.java
index 89fbf09..511b5ee 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/CommonNodeLabelsManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/CommonNodeLabelsManager.java
@@ -129,6 +129,7 @@ public class CommonNodeLabelsManager extends AbstractService {
       if (labels != null) {
         c.labels =
             Collections.newSetFromMap(new ConcurrentHashMap<String, Boolean>());
+        c.labels.addAll(labels);
       } else {
         c.labels = null;
       }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/76cf2250/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/TestRMNodeLabelsManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/TestRMNodeLabelsManager.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/TestRMNodeLabelsManager.java
index 26284e2..81eead9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/TestRMNodeLabelsManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/TestRMNodeLabelsManager.java
@@ -18,6 +18,7 @@
 
 package org.apache.hadoop.yarn.server.resourcemanager.nodelabels;
 
+import java.io.IOException;
 import java.util.HashMap;
 import java.util.Map;
 import java.util.Set;
@@ -25,6 +26,7 @@ import java.util.Set;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.yarn.api.records.NodeId;
 import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager;
 import org.apache.hadoop.yarn.nodelabels.NodeLabelTestBase;
 import org.apache.hadoop.yarn.util.resource.Resources;
 import org.junit.After;
@@ -364,4 +366,31 @@ public class TestRMNodeLabelsManager extends NodeLabelTestBase {
     Assert.assertEquals(clusterResource,
         mgr.getQueueResource("Q5", q5Label, clusterResource));
   }
+  
+  @Test(timeout=5000)
+  public void testGetLabelResourceWhenMultipleNMsExistingInSameHost() throws IOException
{
+    // active two NM to n1, one large and one small
+    mgr.activateNode(NodeId.newInstance("n1", 1), SMALL_RESOURCE);
+    mgr.activateNode(NodeId.newInstance("n1", 2), SMALL_RESOURCE);
+    mgr.activateNode(NodeId.newInstance("n1", 3), SMALL_RESOURCE);
+    mgr.activateNode(NodeId.newInstance("n1", 4), SMALL_RESOURCE);
+    
+    // check resource of no label, it should be small * 4
+    Assert.assertEquals(
+        mgr.getResourceByLabel(CommonNodeLabelsManager.NO_LABEL, null),
+        Resources.multiply(SMALL_RESOURCE, 4));
+    
+    // change two of these nodes to p1, check resource of no_label and P1
+    mgr.addToCluserNodeLabels(toSet("p1"));
+    mgr.addLabelsToNode(ImmutableMap.of(toNodeId("n1:1"), toSet("p1"),
+        toNodeId("n1:2"), toSet("p1")));
+    
+    // check resource
+    Assert.assertEquals(
+        mgr.getResourceByLabel(CommonNodeLabelsManager.NO_LABEL, null),
+        Resources.multiply(SMALL_RESOURCE, 2));    
+    Assert.assertEquals(
+            mgr.getResourceByLabel("p1", null),
+            Resources.multiply(SMALL_RESOURCE, 2));
+  }
 }


Mime
View raw message