Return-Path: X-Original-To: apmail-hadoop-common-commits-archive@www.apache.org Delivered-To: apmail-hadoop-common-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 2816A17D76 for ; Fri, 7 Nov 2014 18:44:04 +0000 (UTC) Received: (qmail 46524 invoked by uid 500); 7 Nov 2014 18:44:04 -0000 Delivered-To: apmail-hadoop-common-commits-archive@hadoop.apache.org Received: (qmail 46457 invoked by uid 500); 7 Nov 2014 18:44:03 -0000 Mailing-List: contact common-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: common-dev@hadoop.apache.org Delivered-To: mailing list common-commits@hadoop.apache.org Received: (qmail 46448 invoked by uid 99); 7 Nov 2014 18:44:03 -0000 Received: from tyr.zones.apache.org (HELO tyr.zones.apache.org) (140.211.11.114) by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 07 Nov 2014 18:44:03 +0000 Received: by tyr.zones.apache.org (Postfix, from userid 65534) id 9B07798BA6F; Fri, 7 Nov 2014 18:44:03 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: vinodkv@apache.org To: common-commits@hadoop.apache.org Message-Id: <7114eb3d101d48e4a7abfe1178cd4012@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: hadoop git commit: YARN-2824. Fixed Capacity Scheduler to not crash when some node-labels are not mapped to queues by making default capacities per label to be zero. Contributed by Wangda Tan. Date: Fri, 7 Nov 2014 18:44:03 +0000 (UTC) Repository: hadoop Updated Branches: refs/heads/trunk 1e97f2f09 -> 2ac1be7de YARN-2824. Fixed Capacity Scheduler to not crash when some node-labels are not mapped to queues by making default capacities per label to be zero. Contributed by Wangda Tan. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/2ac1be7d Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/2ac1be7d Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/2ac1be7d Branch: refs/heads/trunk Commit: 2ac1be7dec4aef001e3162e364249933b2c4a6c4 Parents: 1e97f2f Author: Vinod Kumar Vavilapalli Authored: Fri Nov 7 10:39:37 2014 -0800 Committer: Vinod Kumar Vavilapalli Committed: Fri Nov 7 10:39:37 2014 -0800 ---------------------------------------------------------------------- hadoop-yarn-project/CHANGES.txt | 4 + .../dev-support/findbugs-exclude.xml | 1 + .../distributedshell/TestDistributedShell.java | 2 + .../scheduler/capacity/AbstractCSQueue.java | 28 ++++--- .../CapacitySchedulerConfiguration.java | 17 +--- .../capacity/TestContainerAllocation.java | 5 ++ .../scheduler/capacity/TestQueueParsing.java | 84 ++++++++++++++++++-- 7 files changed, 107 insertions(+), 34 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/2ac1be7d/hadoop-yarn-project/CHANGES.txt ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 111aaaa..c48fb4f 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -894,6 +894,10 @@ Release 2.6.0 - UNRELEASED YARN-2810. TestRMProxyUsersConf fails on Windows VMs. (Varun Vasudev via xgong) + YARN-2824. Fixed Capacity Scheduler to not crash when some node-labels are + not mapped to queues by making default capacities per label to be zero. + (Wangda Tan via vinodkv) + Release 2.5.2 - UNRELEASED INCOMPATIBLE CHANGES http://git-wip-us.apache.org/repos/asf/hadoop/blob/2ac1be7d/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml b/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml index e6da24c..45d7294 100644 --- a/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml +++ b/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml @@ -202,6 +202,7 @@ + http://git-wip-us.apache.org/repos/asf/hadoop/blob/2ac1be7d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java index 0ded5bd..eb0fb94 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java @@ -86,6 +86,8 @@ public class TestDistributedShell { // Setup queue access to node labels conf.set("yarn.scheduler.capacity.root.accessible-node-labels", "x"); + conf.set("yarn.scheduler.capacity.root.accessible-node-labels.x.capacity", + "100"); conf.set("yarn.scheduler.capacity.root.default.accessible-node-labels", "x"); conf.set( "yarn.scheduler.capacity.root.default.accessible-node-labels.x.capacity", http://git-wip-us.apache.org/repos/asf/hadoop/blob/2ac1be7d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java index c612846..fc0fbb4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java @@ -395,16 +395,15 @@ public abstract class AbstractCSQueue implements CSQueue { @Private public float getCapacityByNodeLabel(String label) { - if (null == parent) { - return 1f; - } - if (StringUtils.equals(label, RMNodeLabelsManager.NO_LABEL)) { + if (null == parent) { + return 1f; + } return getCapacity(); } if (!capacitiyByNodeLabels.containsKey(label)) { - return 0; + return 0f; } else { return capacitiyByNodeLabels.get(label); } @@ -412,18 +411,17 @@ public abstract class AbstractCSQueue implements CSQueue { @Private public float getAbsoluteCapacityByNodeLabel(String label) { - if (null == parent) { - return 1; - } - if (StringUtils.equals(label, RMNodeLabelsManager.NO_LABEL)) { + if (null == parent) { + return 1f; + } return getAbsoluteCapacity(); } - if (!absoluteMaxCapacityByNodeLabels.containsKey(label)) { - return 0; + if (!absoluteCapacityByNodeLabels.containsKey(label)) { + return 0f; } else { - return absoluteMaxCapacityByNodeLabels.get(label); + return absoluteCapacityByNodeLabels.get(label); } } @@ -433,7 +431,11 @@ public abstract class AbstractCSQueue implements CSQueue { return getAbsoluteMaximumCapacity(); } - return getAbsoluteCapacityByNodeLabel(label); + if (!absoluteMaxCapacityByNodeLabels.containsKey(label)) { + return 0f; + } else { + return absoluteMaxCapacityByNodeLabels.get(label); + } } @Private http://git-wip-us.apache.org/repos/asf/hadoop/blob/2ac1be7d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java index b36172c..23bf381 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java @@ -461,19 +461,8 @@ public class CapacitySchedulerConfiguration extends Configuration { for (String label : labels.contains(CommonNodeLabelsManager.ANY) ? mgr .getClusterNodeLabels() : labels) { - // capacity of all labels in each queue should be 1 - if (org.apache.commons.lang.StringUtils.equals(ROOT, queue)) { - nodeLabelCapacities.put(label, 1.0f); - continue; - } String capacityPropertyName = getNodeLabelPrefix(queue, label) + CAPACITY; - float capacity = getFloat(capacityPropertyName, UNDEFINED); - if (capacity == UNDEFINED) { - throw new IllegalArgumentException("Configuration issue: " - + " node-label=" + label + " is accessible from queue=" + queue - + " but has no capacity set, you should set " - + capacityPropertyName + " in range of [0, 100]."); - } + float capacity = getFloat(capacityPropertyName, 0f); if (capacity < MINIMUM_CAPACITY_VALUE || capacity > MAXIMUM_CAPACITY_VALUE) { throw new IllegalArgumentException("Illegal capacity of " + capacity @@ -501,9 +490,7 @@ public class CapacitySchedulerConfiguration extends Configuration { .getClusterNodeLabels() : labels) { float maxCapacity = getFloat(getNodeLabelPrefix(queue, label) + MAXIMUM_CAPACITY, - UNDEFINED); - maxCapacity = (maxCapacity == DEFAULT_MAXIMUM_CAPACITY_VALUE) ? - MAXIMUM_CAPACITY_VALUE : maxCapacity; + 100f); if (maxCapacity < MINIMUM_CAPACITY_VALUE || maxCapacity > MAXIMUM_CAPACITY_VALUE) { throw new IllegalArgumentException("Illegal " + "capacity of " http://git-wip-us.apache.org/repos/asf/hadoop/blob/2ac1be7d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerAllocation.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerAllocation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerAllocation.java index b90df8e..0c32c0c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerAllocation.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerAllocation.java @@ -340,6 +340,8 @@ public class TestContainerAllocation { // Define top-level queues conf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[] {"a", "b", "c"}); + conf.setCapacityByLabel(CapacitySchedulerConfiguration.ROOT, "x", 100); + conf.setCapacityByLabel(CapacitySchedulerConfiguration.ROOT, "y", 100); final String A = CapacitySchedulerConfiguration.ROOT + ".a"; conf.setCapacity(A, 10); @@ -403,6 +405,9 @@ public class TestContainerAllocation { // Define top-level queues conf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[] {"a", "b"}); + conf.setCapacityByLabel(CapacitySchedulerConfiguration.ROOT, "x", 100); + conf.setCapacityByLabel(CapacitySchedulerConfiguration.ROOT, "y", 100); + conf.setCapacityByLabel(CapacitySchedulerConfiguration.ROOT, "z", 100); final String A = CapacitySchedulerConfiguration.ROOT + ".a"; conf.setCapacity(A, 10); http://git-wip-us.apache.org/repos/asf/hadoop/blob/2ac1be7d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueParsing.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueParsing.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueParsing.java index 42db030..cf2e5ce 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueParsing.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueParsing.java @@ -18,10 +18,6 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity; -import static org.mockito.Matchers.any; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - import java.io.IOException; import org.apache.commons.logging.Log; @@ -51,8 +47,9 @@ public class TestQueueParsing { @Before public void setup() { - nodeLabelManager = mock(RMNodeLabelsManager.class); - when(nodeLabelManager.containsNodeLabel(any(String.class))).thenReturn(true); + nodeLabelManager = new MemoryRMNodeLabelsManager(); + nodeLabelManager.init(new YarnConfiguration()); + nodeLabelManager.start(); } @Test @@ -255,6 +252,8 @@ public class TestQueueParsing { private void setupQueueConfigurationWithLabels(CapacitySchedulerConfiguration conf) { // Define top-level queues conf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[] {"a", "b"}); + conf.setCapacityByLabel(CapacitySchedulerConfiguration.ROOT, "red", 100); + conf.setCapacityByLabel(CapacitySchedulerConfiguration.ROOT, "blue", 100); final String A = CapacitySchedulerConfiguration.ROOT + ".a"; conf.setCapacity(A, 10); @@ -271,6 +270,7 @@ public class TestQueueParsing { conf.setQueues(A, new String[] {"a1", "a2"}); conf.setAccessibleNodeLabels(A, ImmutableSet.of("red", "blue")); conf.setCapacityByLabel(A, "red", 50); + conf.setMaximumCapacityByLabel(A, "red", 50); conf.setCapacityByLabel(A, "blue", 50); conf.setCapacity(A1, 30); @@ -282,6 +282,7 @@ public class TestQueueParsing { conf.setMaximumCapacity(A2, 85); conf.setAccessibleNodeLabels(A2, ImmutableSet.of("red")); conf.setCapacityByLabel(A2, "red", 50); + conf.setMaximumCapacityByLabel(A2, "red", 60); final String B1 = B + ".b1"; final String B2 = B + ".b2"; @@ -311,6 +312,8 @@ public class TestQueueParsing { CapacitySchedulerConfiguration conf) { // Define top-level queues conf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[] {"a", "b"}); + conf.setCapacityByLabel(CapacitySchedulerConfiguration.ROOT, "red", 100); + conf.setCapacityByLabel(CapacitySchedulerConfiguration.ROOT, "blue", 100); // Set A configuration final String A = CapacitySchedulerConfiguration.ROOT + ".a"; @@ -364,6 +367,7 @@ public class TestQueueParsing { @Test public void testQueueParsingReinitializeWithLabels() throws IOException { + nodeLabelManager.addToCluserNodeLabels(ImmutableSet.of("red", "blue")); CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration(); setupQueueConfigurationWithoutLabels(csConf); @@ -410,6 +414,22 @@ public class TestQueueParsing { // queue-B2 inherits "red"/"blue" Assert.assertTrue(capacityScheduler.getQueue("b2") .getAccessibleNodeLabels().containsAll(ImmutableSet.of("red", "blue"))); + + // check capacity of A2 + CSQueue qA2 = capacityScheduler.getQueue("a2"); + Assert.assertEquals(0.7, qA2.getCapacity(), DELTA); + Assert.assertEquals(0.5, qA2.getCapacityByNodeLabel("red"), DELTA); + Assert.assertEquals(0.07, qA2.getAbsoluteCapacity(), DELTA); + Assert.assertEquals(0.25, qA2.getAbsoluteCapacityByNodeLabel("red"), DELTA); + Assert.assertEquals(0.1275, qA2.getAbsoluteMaximumCapacity(), DELTA); + Assert.assertEquals(0.3, qA2.getAbsoluteMaximumCapacityByNodeLabel("red"), DELTA); + + // check capacity of B3 + CSQueue qB3 = capacityScheduler.getQueue("b3"); + Assert.assertEquals(0.18, qB3.getAbsoluteCapacity(), DELTA); + Assert.assertEquals(0.125, qB3.getAbsoluteCapacityByNodeLabel("red"), DELTA); + Assert.assertEquals(0.35, qB3.getAbsoluteMaximumCapacity(), DELTA); + Assert.assertEquals(1, qB3.getAbsoluteMaximumCapacityByNodeLabel("red"), DELTA); } private void @@ -435,6 +455,8 @@ public class TestQueueParsing { @Test public void testQueueParsingWithLabels() throws IOException { + nodeLabelManager.addToCluserNodeLabels(ImmutableSet.of("red", "blue")); + YarnConfiguration conf = new YarnConfiguration(); CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration(conf); @@ -457,6 +479,8 @@ public class TestQueueParsing { @Test public void testQueueParsingWithLabelsInherit() throws IOException { + nodeLabelManager.addToCluserNodeLabels(ImmutableSet.of("red", "blue")); + YarnConfiguration conf = new YarnConfiguration(); CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration(conf); @@ -587,4 +611,52 @@ public class TestQueueParsing { ServiceOperations.stopQuietly(capacityScheduler); ServiceOperations.stopQuietly(nodeLabelsManager); } + + @Test + public void testQueueParsingWithUnusedLabels() throws IOException { + final ImmutableSet labels = ImmutableSet.of("red", "blue"); + + // Initialize a cluster with labels, but doesn't use them, reinitialize + // shouldn't fail + nodeLabelManager.addToCluserNodeLabels(labels); + + CapacitySchedulerConfiguration csConf = + new CapacitySchedulerConfiguration(); + setupQueueConfiguration(csConf); + csConf.setAccessibleNodeLabels(CapacitySchedulerConfiguration.ROOT, labels); + YarnConfiguration conf = new YarnConfiguration(csConf); + + CapacityScheduler capacityScheduler = new CapacityScheduler(); + capacityScheduler.setConf(conf); + RMContextImpl rmContext = + new RMContextImpl(null, null, null, null, null, null, + new RMContainerTokenSecretManager(csConf), + new NMTokenSecretManagerInRM(csConf), + new ClientToAMTokenSecretManagerInRM(), null); + rmContext.setNodeLabelManager(nodeLabelManager); + capacityScheduler.setRMContext(rmContext); + capacityScheduler.init(conf); + capacityScheduler.start(); + capacityScheduler.reinitialize(conf, rmContext); + + // check root queue's capacity by label -- they should be all zero + CSQueue root = capacityScheduler.getQueue(CapacitySchedulerConfiguration.ROOT); + Assert.assertEquals(0, root.getCapacityByNodeLabel("red"), DELTA); + Assert.assertEquals(0, root.getCapacityByNodeLabel("blue"), DELTA); + + CSQueue a = capacityScheduler.getQueue("a"); + Assert.assertEquals(0.10, a.getAbsoluteCapacity(), DELTA); + Assert.assertEquals(0.15, a.getAbsoluteMaximumCapacity(), DELTA); + + CSQueue b1 = capacityScheduler.getQueue("b1"); + Assert.assertEquals(0.2 * 0.5, b1.getAbsoluteCapacity(), DELTA); + Assert.assertEquals("Parent B has no MAX_CAP", 0.85, + b1.getAbsoluteMaximumCapacity(), DELTA); + + CSQueue c12 = capacityScheduler.getQueue("c12"); + Assert.assertEquals(0.7 * 0.5 * 0.45, c12.getAbsoluteCapacity(), DELTA); + Assert.assertEquals(0.7 * 0.55 * 0.7, c12.getAbsoluteMaximumCapacity(), + DELTA); + capacityScheduler.stop(); + } }