Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id 46B66200CC0 for ; Sun, 9 Jul 2017 14:06:36 +0200 (CEST) Received: by cust-asf.ponee.io (Postfix) id 4536716B088; Sun, 9 Jul 2017 12:06:36 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id 6485816B085 for ; Sun, 9 Jul 2017 14:06:35 +0200 (CEST) Received: (qmail 32620 invoked by uid 500); 9 Jul 2017 12:06:34 -0000 Mailing-List: contact common-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Delivered-To: mailing list common-commits@hadoop.apache.org Received: (qmail 32611 invoked by uid 99); 9 Jul 2017 12:06:34 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Sun, 09 Jul 2017 12:06:34 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 6940CE3823; Sun, 9 Jul 2017 12:06:33 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: naganarasimha_gr@apache.org To: common-commits@hadoop.apache.org Message-Id: X-Mailer: ASF-Git Admin Mailer Subject: hadoop git commit: YARN-6428. Queue AM limit is not honored in CS always. Contributed by Bibin A Chundutt. Date: Sun, 9 Jul 2017 12:06:33 +0000 (UTC) archived-at: Sun, 09 Jul 2017 12:06:36 -0000 Repository: hadoop Updated Branches: refs/heads/branch-2.8 5ad710bc7 -> ad70e929b YARN-6428. Queue AM limit is not honored in CS always. Contributed by Bibin A Chundutt. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ad70e929 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ad70e929 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ad70e929 Branch: refs/heads/branch-2.8 Commit: ad70e929b2b4278f7340ec629e0062e0073d7900 Parents: 5ad710b Author: Naganarasimha Authored: Sun Jul 9 17:34:47 2017 +0530 Committer: Naganarasimha Committed: Sun Jul 9 17:34:47 2017 +0530 ---------------------------------------------------------------------- .../resource/DominantResourceCalculator.java | 22 ++++------- .../capacity/TestCapacityScheduler.java | 40 ++++++++++++++++++++ 2 files changed, 47 insertions(+), 15 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/ad70e929/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java index 7e4cf76..1472f0e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java @@ -186,27 +186,19 @@ public class DominantResourceCalculator extends ResourceCalculator { public Resource multiplyAndNormalizeUp(Resource r, double by, Resource stepFactor) { return Resources.createResource( - roundUp( - (int)Math.ceil(r.getMemorySize() * by), stepFactor.getMemorySize()), - roundUp( - (int)Math.ceil(r.getVirtualCores() * by), - stepFactor.getVirtualCores()) - ); + roundUp((long) Math.ceil((float) (r.getMemorySize() * by)), + stepFactor.getMemorySize()), + roundUp((int) Math.ceil((float) (r.getVirtualCores() * by)), + stepFactor.getVirtualCores())); } @Override public Resource multiplyAndNormalizeDown(Resource r, double by, Resource stepFactor) { return Resources.createResource( - roundDown( - (int)(r.getMemorySize() * by), - stepFactor.getMemorySize() - ), - roundDown( - (int)(r.getVirtualCores() * by), - stepFactor.getVirtualCores() - ) - ); + roundDown((long) (r.getMemorySize() * by), stepFactor.getMemorySize()), + roundDown((int) (r.getVirtualCores() * by), + stepFactor.getVirtualCores())); } @Override http://git-wip-us.apache.org/repos/asf/hadoop/blob/ad70e929/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java index bd0691c..15fe2b4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java @@ -3690,4 +3690,44 @@ public class TestCapacityScheduler { cs.handle(addAttemptEvent1); return appAttemptId1; } + + @Test(timeout = 30000) + public void testAMLimitDouble() throws Exception { + CapacitySchedulerConfiguration config = + new CapacitySchedulerConfiguration(); + config.set(CapacitySchedulerConfiguration.RESOURCE_CALCULATOR_CLASS, + DominantResourceCalculator.class.getName()); + CapacitySchedulerConfiguration conf = + new CapacitySchedulerConfiguration(config); + conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class, + ResourceScheduler.class); + conf.setInt("yarn.scheduler.minimum-allocation-mb", 512); + conf.setInt("yarn.scheduler.minimum-allocation-vcores", 1); + MockRM rm = new MockRM(conf); + rm.start(); + rm.registerNode("127.0.0.1:1234", 10 * GB); + rm.registerNode("127.0.0.1:1235", 10 * GB); + rm.registerNode("127.0.0.1:1236", 10 * GB); + rm.registerNode("127.0.0.1:1237", 10 * GB); + ResourceScheduler scheduler = rm.getRMContext().getScheduler(); + waitforNMRegistered(scheduler, 4, 5); + LeafQueue queueA = + (LeafQueue) ((CapacityScheduler) scheduler).getQueue("default"); + Resource amResourceLimit = queueA.getAMResourceLimit(); + Assert.assertEquals(4096, amResourceLimit.getMemorySize()); + Assert.assertEquals(4, amResourceLimit.getVirtualCores()); + rm.stop(); + } + + private void waitforNMRegistered(ResourceScheduler scheduler, int nodecount, + int timesec) throws InterruptedException { + long start = System.currentTimeMillis(); + while (System.currentTimeMillis() - start < timesec * 1000) { + if (scheduler.getNumClusterNodes() < nodecount) { + Thread.sleep(100); + } else { + break; + } + } + } } --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org For additional commands, e-mail: common-commits-help@hadoop.apache.org