Return-Path: X-Original-To: apmail-hadoop-common-commits-archive@www.apache.org Delivered-To: apmail-hadoop-common-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id A51271820D for ; Thu, 4 Jun 2015 17:27:23 +0000 (UTC) Received: (qmail 5223 invoked by uid 500); 4 Jun 2015 17:27:23 -0000 Delivered-To: apmail-hadoop-common-commits-archive@hadoop.apache.org Received: (qmail 5150 invoked by uid 500); 4 Jun 2015 17:27:23 -0000 Mailing-List: contact common-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: common-dev@hadoop.apache.org Delivered-To: mailing list common-commits@hadoop.apache.org Received: (qmail 5141 invoked by uid 99); 4 Jun 2015 17:27:23 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Thu, 04 Jun 2015 17:27:23 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 1F397E1083; Thu, 4 Jun 2015 17:27:23 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: wangda@apache.org To: common-commits@hadoop.apache.org Message-Id: <1bbb99a636964deeb993b45f9a7daf4e@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: hadoop git commit: YARN-3733. Fix DominantRC#compare() does not work as expected if cluster resource is empty. (Rohith Sharmaks via wangda) Date: Thu, 4 Jun 2015 17:27:23 +0000 (UTC) Repository: hadoop Updated Branches: refs/heads/branch-2.7 718dca485 -> e74e4d7bb YARN-3733. Fix DominantRC#compare() does not work as expected if cluster resource is empty. (Rohith Sharmaks via wangda) (cherry picked from commit ebd797c48fe236b404cf3a125ac9d1f7714e291e) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e74e4d7b Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e74e4d7b Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e74e4d7b Branch: refs/heads/branch-2.7 Commit: e74e4d7bb9f7799d91e250aab6281cb6d3faf2ac Parents: 718dca4 Author: Wangda Tan Authored: Thu Jun 4 10:22:57 2015 -0700 Committer: Wangda Tan Committed: Thu Jun 4 10:26:06 2015 -0700 ---------------------------------------------------------------------- hadoop-yarn-project/CHANGES.txt | 3 + .../resource/DominantResourceCalculator.java | 15 +++++ .../capacity/TestCapacityScheduler.java | 58 +++++++++++++++++++- 3 files changed, 75 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/e74e4d7b/hadoop-yarn-project/CHANGES.txt ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 2ba4286..d18c477 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -148,6 +148,9 @@ Release 2.7.1 - UNRELEASED YARN-3585. NodeManager cannot exit on SHUTDOWN event triggered and NM recovery is enabled (Rohith Sharmaks via jlowe) + YARN-3733. Fix DominantRC#compare() does not work as expected if + cluster resource is empty. (Rohith Sharmaks via wangda) + Release 2.7.0 - 2015-04-20 INCOMPATIBLE CHANGES http://git-wip-us.apache.org/repos/asf/hadoop/blob/e74e4d7b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java index 6f5b40e..2ee95ce 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java @@ -53,6 +53,21 @@ public class DominantResourceCalculator extends ResourceCalculator { return 0; } + if (isInvalidDivisor(clusterResource)) { + if ((lhs.getMemory() < rhs.getMemory() && lhs.getVirtualCores() > rhs + .getVirtualCores()) + || (lhs.getMemory() > rhs.getMemory() && lhs.getVirtualCores() < rhs + .getVirtualCores())) { + return 0; + } else if (lhs.getMemory() > rhs.getMemory() + || lhs.getVirtualCores() > rhs.getVirtualCores()) { + return 1; + } else if (lhs.getMemory() < rhs.getMemory() + || lhs.getVirtualCores() < rhs.getVirtualCores()) { + return -1; + } + } + float l = getResourceAsValue(clusterResource, lhs, true); float r = getResourceAsValue(clusterResource, rhs, true); http://git-wip-us.apache.org/repos/asf/hadoop/blob/e74e4d7b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java index d27cc47..e2e8cc5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java @@ -123,6 +123,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.CapacitySchedule import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.CapacitySchedulerQueueInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.CapacitySchedulerQueueInfoList; import org.apache.hadoop.yarn.server.utils.BuilderUtils; +import org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator; import org.apache.hadoop.yarn.util.resource.DominantResourceCalculator; import org.apache.hadoop.yarn.util.resource.Resources; import org.junit.After; @@ -1159,9 +1160,15 @@ public class TestCapacityScheduler { private MockRM setUpMove() { CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration(); + return setUpMove(conf); + } + + private MockRM setUpMove(Configuration config) { + CapacitySchedulerConfiguration conf = + new CapacitySchedulerConfiguration(config); setupQueueConfiguration(conf); conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class, - ResourceScheduler.class); + ResourceScheduler.class); MockRM rm = new MockRM(conf); rm.start(); return rm; @@ -2640,6 +2647,55 @@ public class TestCapacityScheduler { Assert.assertEquals(queueInfoB.getDefaultNodeLabelExpression(), "y"); } + @Test(timeout = 30000) + public void testAMLimitUsage() throws Exception { + + CapacitySchedulerConfiguration config = + new CapacitySchedulerConfiguration(); + + config.set(CapacitySchedulerConfiguration.RESOURCE_CALCULATOR_CLASS, + DefaultResourceCalculator.class.getName()); + verifyAMLimitForLeafQueue(config); + + config.set(CapacitySchedulerConfiguration.RESOURCE_CALCULATOR_CLASS, + DominantResourceCalculator.class.getName()); + verifyAMLimitForLeafQueue(config); + + } + + private void verifyAMLimitForLeafQueue(CapacitySchedulerConfiguration config) + throws Exception { + MockRM rm = setUpMove(config); + + String queueName = "a1"; + String userName = "user_0"; + ResourceScheduler scheduler = rm.getRMContext().getScheduler(); + LeafQueue queueA = + (LeafQueue) ((CapacityScheduler) scheduler).getQueue(queueName); + Resource amResourceLimit = queueA.getAMResourceLimit(); + + Resource amResource = + Resource.newInstance(amResourceLimit.getMemory() + 1, + amResourceLimit.getVirtualCores() + 1); + + rm.submitApp(amResource.getMemory(), "app-1", userName, null, queueName); + + rm.submitApp(amResource.getMemory(), "app-1", userName, null, queueName); + + // When AM limit is exceeded, 1 applications will be activated.Rest all + // applications will be in pending + Assert.assertEquals("PendingApplications should be 1", 1, + queueA.getNumPendingApplications()); + Assert.assertEquals("Active applications should be 1", 1, + queueA.getNumActiveApplications()); + + Assert.assertEquals("User PendingApplications should be 1", 1, queueA + .getUser(userName).getPendingApplications()); + Assert.assertEquals("User Active applications should be 1", 1, queueA + .getUser(userName).getActiveApplications()); + rm.stop(); + } + private void setMaxAllocMb(Configuration conf, int maxAllocMb) { conf.setInt(YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_MB, maxAllocMb);