Return-Path: X-Original-To: apmail-hadoop-common-commits-archive@www.apache.org Delivered-To: apmail-hadoop-common-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 73C3CCAB7 for ; Tue, 9 Dec 2014 03:31:11 +0000 (UTC) Received: (qmail 57155 invoked by uid 500); 9 Dec 2014 03:30:59 -0000 Delivered-To: apmail-hadoop-common-commits-archive@hadoop.apache.org Received: (qmail 56981 invoked by uid 500); 9 Dec 2014 03:30:59 -0000 Mailing-List: contact common-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: common-dev@hadoop.apache.org Delivered-To: mailing list common-commits@hadoop.apache.org Received: (qmail 54650 invoked by uid 99); 9 Dec 2014 03:30:57 -0000 Received: from tyr.zones.apache.org (HELO tyr.zones.apache.org) (140.211.11.114) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 09 Dec 2014 03:30:57 +0000 Received: by tyr.zones.apache.org (Postfix, from userid 65534) id 0641EA1F4A4; Tue, 9 Dec 2014 03:30:56 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: kasha@apache.org To: common-commits@hadoop.apache.org Date: Tue, 09 Dec 2014 03:31:23 -0000 Message-Id: In-Reply-To: References: X-Mailer: ASF-Git Admin Mailer Subject: [29/41] hadoop git commit: YARN-2869. CapacityScheduler should trim sub queue names when parse configuration. Contributed by Wangda Tan YARN-2869. CapacityScheduler should trim sub queue names when parse configuration. Contributed by Wangda Tan Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e69af836 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e69af836 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e69af836 Branch: refs/heads/YARN-2139 Commit: e69af836f34f16fba565ab112c9bf0d367675b16 Parents: 475c6b4 Author: Jian He Authored: Fri Dec 5 17:33:39 2014 -0800 Committer: Jian He Committed: Fri Dec 5 17:33:39 2014 -0800 ---------------------------------------------------------------------- hadoop-yarn-project/CHANGES.txt | 3 + .../CapacitySchedulerConfiguration.java | 10 +- .../scheduler/capacity/TestQueueParsing.java | 110 +++++++++++++++++++ 3 files changed, 122 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/e69af836/hadoop-yarn-project/CHANGES.txt ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 0b88959..0d7a843 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -194,6 +194,9 @@ Release 2.7.0 - UNRELEASED YARN-2461. Fix PROCFS_USE_SMAPS_BASED_RSS_ENABLED property in YarnConfiguration. (rchiang via rkanter) + YARN-2869. CapacityScheduler should trim sub queue names when parse + configuration. (Wangda Tan via jianhe) + Release 2.6.0 - 2014-11-18 INCOMPATIBLE CHANGES http://git-wip-us.apache.org/repos/asf/hadoop/blob/e69af836/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java index 0a49224..5bbb436 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java @@ -260,7 +260,7 @@ public class CapacitySchedulerConfiguration extends ReservationSchedulerConfigur } } - private String getQueuePrefix(String queue) { + static String getQueuePrefix(String queue) { String queueName = PREFIX + queue + DOT; return queueName; } @@ -538,6 +538,14 @@ public class CapacitySchedulerConfiguration extends ReservationSchedulerConfigur public String[] getQueues(String queue) { LOG.debug("CSConf - getQueues called for: queuePrefix=" + getQueuePrefix(queue)); String[] queues = getStrings(getQueuePrefix(queue) + QUEUES); + List trimmedQueueNames = new ArrayList(); + if (null != queues) { + for (String s : queues) { + trimmedQueueNames.add(s.trim()); + } + queues = trimmedQueueNames.toArray(new String[0]); + } + LOG.debug("CSConf - getQueues: queuePrefix=" + getQueuePrefix(queue) + ", queues=" + ((queues == null) ? "" : StringUtils.arrayToString(queues))); return queues; http://git-wip-us.apache.org/repos/asf/hadoop/blob/e69af836/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueParsing.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueParsing.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueParsing.java index cf2e5ce..5a9fbe1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueParsing.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueParsing.java @@ -82,6 +82,56 @@ public class TestQueueParsing { ServiceOperations.stopQuietly(capacityScheduler); } + private void setupQueueConfigurationWithSpacesShouldBeTrimmed( + CapacitySchedulerConfiguration conf) { + // Define top-level queues + conf.set( + CapacitySchedulerConfiguration + .getQueuePrefix(CapacitySchedulerConfiguration.ROOT) + + CapacitySchedulerConfiguration.QUEUES, " a ,b, c"); + + final String A = CapacitySchedulerConfiguration.ROOT + ".a"; + conf.setCapacity(A, 10); + conf.setMaximumCapacity(A, 15); + + final String B = CapacitySchedulerConfiguration.ROOT + ".b"; + conf.setCapacity(B, 20); + + final String C = CapacitySchedulerConfiguration.ROOT + ".c"; + conf.setCapacity(C, 70); + conf.setMaximumCapacity(C, 70); + } + + private void setupNestedQueueConfigurationWithSpacesShouldBeTrimmed( + CapacitySchedulerConfiguration conf) { + // Define top-level queues + conf.set( + CapacitySchedulerConfiguration + .getQueuePrefix(CapacitySchedulerConfiguration.ROOT) + + CapacitySchedulerConfiguration.QUEUES, " a ,b, c"); + + final String A = CapacitySchedulerConfiguration.ROOT + ".a"; + conf.setCapacity(A, 10); + conf.setMaximumCapacity(A, 15); + + final String B = CapacitySchedulerConfiguration.ROOT + ".b"; + conf.setCapacity(B, 20); + + final String C = CapacitySchedulerConfiguration.ROOT + ".c"; + conf.setCapacity(C, 70); + conf.setMaximumCapacity(C, 70); + + // sub queues for A + conf.set(CapacitySchedulerConfiguration.getQueuePrefix(A) + + CapacitySchedulerConfiguration.QUEUES, "a1, a2 "); + + final String A1 = CapacitySchedulerConfiguration.ROOT + ".a.a1"; + conf.setCapacity(A1, 60); + + final String A2 = CapacitySchedulerConfiguration.ROOT + ".a.a2"; + conf.setCapacity(A2, 40); + } + private void setupQueueConfiguration(CapacitySchedulerConfiguration conf) { // Define top-level queues @@ -659,4 +709,64 @@ public class TestQueueParsing { DELTA); capacityScheduler.stop(); } + + @Test + public void testQueueParsingShouldTrimSpaces() throws Exception { + CapacitySchedulerConfiguration csConf = + new CapacitySchedulerConfiguration(); + setupQueueConfigurationWithSpacesShouldBeTrimmed(csConf); + YarnConfiguration conf = new YarnConfiguration(csConf); + + CapacityScheduler capacityScheduler = new CapacityScheduler(); + capacityScheduler.setConf(conf); + capacityScheduler.setRMContext(TestUtils.getMockRMContext()); + capacityScheduler.init(conf); + capacityScheduler.start(); + capacityScheduler.reinitialize(conf, TestUtils.getMockRMContext()); + + CSQueue a = capacityScheduler.getQueue("a"); + Assert.assertNotNull(a); + Assert.assertEquals(0.10, a.getAbsoluteCapacity(), DELTA); + Assert.assertEquals(0.15, a.getAbsoluteMaximumCapacity(), DELTA); + + CSQueue c = capacityScheduler.getQueue("c"); + Assert.assertNotNull(c); + Assert.assertEquals(0.70, c.getAbsoluteCapacity(), DELTA); + Assert.assertEquals(0.70, c.getAbsoluteMaximumCapacity(), DELTA); + } + + @Test + public void testNestedQueueParsingShouldTrimSpaces() throws Exception { + CapacitySchedulerConfiguration csConf = + new CapacitySchedulerConfiguration(); + setupNestedQueueConfigurationWithSpacesShouldBeTrimmed(csConf); + YarnConfiguration conf = new YarnConfiguration(csConf); + + CapacityScheduler capacityScheduler = new CapacityScheduler(); + capacityScheduler.setConf(conf); + capacityScheduler.setRMContext(TestUtils.getMockRMContext()); + capacityScheduler.init(conf); + capacityScheduler.start(); + capacityScheduler.reinitialize(conf, TestUtils.getMockRMContext()); + + CSQueue a = capacityScheduler.getQueue("a"); + Assert.assertNotNull(a); + Assert.assertEquals(0.10, a.getAbsoluteCapacity(), DELTA); + Assert.assertEquals(0.15, a.getAbsoluteMaximumCapacity(), DELTA); + + CSQueue c = capacityScheduler.getQueue("c"); + Assert.assertNotNull(c); + Assert.assertEquals(0.70, c.getAbsoluteCapacity(), DELTA); + Assert.assertEquals(0.70, c.getAbsoluteMaximumCapacity(), DELTA); + + CSQueue a1 = capacityScheduler.getQueue("a1"); + Assert.assertNotNull(a1); + Assert.assertEquals(0.10 * 0.6, a1.getAbsoluteCapacity(), DELTA); + Assert.assertEquals(0.15, a1.getAbsoluteMaximumCapacity(), DELTA); + + CSQueue a2 = capacityScheduler.getQueue("a2"); + Assert.assertNotNull(a2); + Assert.assertEquals(0.10 * 0.4, a2.getAbsoluteCapacity(), DELTA); + Assert.assertEquals(0.15, a2.getAbsoluteMaximumCapacity(), DELTA); + } }