Return-Path: X-Original-To: apmail-hadoop-common-commits-archive@www.apache.org Delivered-To: apmail-hadoop-common-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id B10821975B for ; Tue, 5 Apr 2016 23:30:35 +0000 (UTC) Received: (qmail 5861 invoked by uid 500); 5 Apr 2016 23:30:35 -0000 Delivered-To: apmail-hadoop-common-commits-archive@hadoop.apache.org Received: (qmail 5793 invoked by uid 500); 5 Apr 2016 23:30:35 -0000 Mailing-List: contact common-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: common-dev@hadoop.apache.org Delivered-To: mailing list common-commits@hadoop.apache.org Received: (qmail 5784 invoked by uid 99); 5 Apr 2016 23:30:35 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 05 Apr 2016 23:30:35 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 5817FDFDEC; Tue, 5 Apr 2016 23:30:35 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: wangda@apache.org To: common-commits@hadoop.apache.org Message-Id: <201686fe36f745c0afbd7901c727b50a@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: hadoop git commit: YARN-4699. Scheduler UI and REST o/p is not in sync when -replaceLabelsOnNode is used to change label of a node. (Sunil G via wangda) Date: Tue, 5 Apr 2016 23:30:35 +0000 (UTC) Repository: hadoop Updated Branches: refs/heads/branch-2 6e37c5fe8 -> 11e796b5c YARN-4699. Scheduler UI and REST o/p is not in sync when -replaceLabelsOnNode is used to change label of a node. (Sunil G via wangda) (cherry picked from commit 21eb4284487d6f8e4beedb8a0c3168e952f224fc) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/11e796b5 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/11e796b5 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/11e796b5 Branch: refs/heads/branch-2 Commit: 11e796b5cdff39d1f4fcd6109c4c2701426947af Parents: 6e37c5f Author: Wangda Tan Authored: Tue Apr 5 16:24:11 2016 -0700 Committer: Wangda Tan Committed: Tue Apr 5 16:25:55 2016 -0700 ---------------------------------------------------------------------- .../scheduler/capacity/AbstractCSQueue.java | 6 +++ .../scheduler/capacity/CSQueueUtils.java | 2 +- .../TestCapacitySchedulerNodeLabelUpdate.java | 40 +++++++++++++++++++- 3 files changed, 46 insertions(+), 2 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/11e796b5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java index 6e715fb..c7d6d02 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java @@ -591,6 +591,9 @@ public abstract class AbstractCSQueue implements CSQueue { } // ResourceUsage has its own lock, no addition lock needs here. queueUsage.incUsed(nodeLabel, resourceToInc); + CSQueueUtils.updateUsedCapacity(resourceCalculator, + labelManager.getResourceByLabel(nodeLabel, Resources.none()), + minimumAllocation, queueUsage, queueCapacities, nodeLabel); if (null != parent) { parent.incUsedResource(nodeLabel, resourceToInc, null); } @@ -604,6 +607,9 @@ public abstract class AbstractCSQueue implements CSQueue { } // ResourceUsage has its own lock, no addition lock needs here. queueUsage.decUsed(nodeLabel, resourceToDec); + CSQueueUtils.updateUsedCapacity(resourceCalculator, + labelManager.getResourceByLabel(nodeLabel, Resources.none()), + minimumAllocation, queueUsage, queueCapacities, nodeLabel); if (null != parent) { parent.decUsedResource(nodeLabel, resourceToDec, null); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/11e796b5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java index 9cdcb72..0166d83 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java @@ -180,7 +180,7 @@ class CSQueueUtils { * Update partitioned resource usage, if nodePartition == null, will update * used resource for all partitions of this queue. */ - private static void updateUsedCapacity(final ResourceCalculator rc, + public static void updateUsedCapacity(final ResourceCalculator rc, final Resource totalPartitionResource, final Resource minimumAllocation, ResourceUsage queueResourceUsage, QueueCapacities queueCapacities, String nodePartition) { http://git-wip-us.apache.org/repos/asf/hadoop/blob/11e796b5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNodeLabelUpdate.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNodeLabelUpdate.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNodeLabelUpdate.java index fe24b2d..cff79cd 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNodeLabelUpdate.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNodeLabelUpdate.java @@ -100,6 +100,12 @@ public class TestCapacitySchedulerNodeLabelUpdate { checkAMUsedResource(rm, queueName, memory, RMNodeLabelsManager.NO_LABEL); } + private void checkUsedCapacity(MockRM rm, String queueName, int capacity, + int total) { + checkUsedCapacity(rm, queueName, capacity, total, + RMNodeLabelsManager.NO_LABEL); + } + private void checkUsedResource(MockRM rm, String queueName, int memory, String label) { CapacityScheduler scheduler = (CapacityScheduler) rm.getResourceScheduler(); @@ -108,6 +114,15 @@ public class TestCapacitySchedulerNodeLabelUpdate { .getMemory()); } + private void checkUsedCapacity(MockRM rm, String queueName, int capacity, + int total, String label) { + float epsillon = 0.0001f; + CapacityScheduler scheduler = (CapacityScheduler) rm.getResourceScheduler(); + CSQueue queue = scheduler.getQueue(queueName); + Assert.assertEquals((float)capacity/total, + queue.getQueueCapacities().getUsedCapacity(label), epsillon); + } + private void checkAMUsedResource(MockRM rm, String queueName, int memory, String label) { CapacityScheduler scheduler = (CapacityScheduler) rm.getResourceScheduler(); @@ -188,7 +203,7 @@ public class TestCapacitySchedulerNodeLabelUpdate { rm.stop(); } - @Test (timeout = 60000) + @Test public void testResourceUsageWhenNodeUpdatesPartition() throws Exception { // set node -> label @@ -233,16 +248,23 @@ public class TestCapacitySchedulerNodeLabelUpdate { // queue-a used x=1G, ""=1G checkUsedResource(rm, "a", 1024, "x"); checkUsedResource(rm, "a", 1024); + checkUsedCapacity(rm, "a", 1024, 8000, "x"); + checkUsedCapacity(rm, "a", 1024, 8000); CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler(); FiCaSchedulerApp app = cs.getApplicationAttempt(am1.getApplicationAttemptId()); // change h1's label to z + mgr.replaceLabelsOnNode(ImmutableMap.of(nm1.getNodeId(), toSet("z"))); cs.handle(new NodeLabelsUpdateSchedulerEvent(ImmutableMap.of(nm1.getNodeId(), toSet("z")))); + Thread.sleep(100); checkUsedResource(rm, "a", 0, "x"); checkUsedResource(rm, "a", 1024, "z"); checkUsedResource(rm, "a", 1024); + checkUsedCapacity(rm, "a", 0, 8000, "x"); + checkUsedCapacity(rm, "a", 1024, 8000, "z"); + checkUsedCapacity(rm, "a", 1024, 8000); checkUsedResource(rm, "root", 0, "x"); checkUsedResource(rm, "root", 1024, "z"); checkUsedResource(rm, "root", 1024); @@ -254,12 +276,18 @@ public class TestCapacitySchedulerNodeLabelUpdate { app.getAppAttemptResourceUsage().getUsed("z").getMemory()); // change h1's label to y + mgr.replaceLabelsOnNode(ImmutableMap.of(nm1.getNodeId(), toSet("y"))); cs.handle(new NodeLabelsUpdateSchedulerEvent(ImmutableMap.of(nm1.getNodeId(), toSet("y")))); + Thread.sleep(100); checkUsedResource(rm, "a", 0, "x"); checkUsedResource(rm, "a", 1024, "y"); checkUsedResource(rm, "a", 0, "z"); checkUsedResource(rm, "a", 1024); + checkUsedCapacity(rm, "a", 0, 8000, "x"); + checkUsedCapacity(rm, "a", 1024, 16000, "y"); + checkUsedCapacity(rm, "a", 0, 8000, "z"); + checkUsedCapacity(rm, "a", 1024, 8000); checkUsedResource(rm, "root", 0, "x"); checkUsedResource(rm, "root", 1024, "y"); checkUsedResource(rm, "root", 0, "z"); @@ -278,11 +306,17 @@ public class TestCapacitySchedulerNodeLabelUpdate { Set emptyLabels = new HashSet<>(); Map> map = ImmutableMap.of(nm1.getNodeId(), emptyLabels); + mgr.replaceLabelsOnNode(map); cs.handle(new NodeLabelsUpdateSchedulerEvent(map)); + Thread.sleep(100); checkUsedResource(rm, "a", 0, "x"); checkUsedResource(rm, "a", 0, "y"); checkUsedResource(rm, "a", 0, "z"); checkUsedResource(rm, "a", 2048); + checkUsedCapacity(rm, "a", 0, 8000, "x"); + checkUsedCapacity(rm, "a", 0, 8000, "y"); + checkUsedCapacity(rm, "a", 0, 8000, "z"); + checkUsedCapacity(rm, "a", 2048, 16000); checkUsedResource(rm, "root", 0, "x"); checkUsedResource(rm, "root", 0, "y"); checkUsedResource(rm, "root", 0, "z"); @@ -314,6 +348,10 @@ public class TestCapacitySchedulerNodeLabelUpdate { checkUsedResource(rm, "a", 0, "y"); checkUsedResource(rm, "a", 0, "z"); checkUsedResource(rm, "a", 0); + checkUsedCapacity(rm, "a", 0, 8000, "x"); + checkUsedCapacity(rm, "a", 0, 8000, "y"); + checkUsedCapacity(rm, "a", 0, 8000, "z"); + checkUsedCapacity(rm, "a", 0, 16000); checkUsedResource(rm, "root", 0, "x"); checkUsedResource(rm, "root", 0, "y"); checkUsedResource(rm, "root", 0, "z");