Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id A0D9B200C36 for ; Fri, 10 Mar 2017 08:16:11 +0100 (CET) Received: by cust-asf.ponee.io (Postfix) id 9F73E160B79; Fri, 10 Mar 2017 07:16:11 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id C307C160B69 for ; Fri, 10 Mar 2017 08:16:10 +0100 (CET) Received: (qmail 35112 invoked by uid 500); 10 Mar 2017 07:16:09 -0000 Mailing-List: contact common-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Delivered-To: mailing list common-commits@hadoop.apache.org Received: (qmail 35103 invoked by uid 99); 10 Mar 2017 07:16:09 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 10 Mar 2017 07:16:09 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id B47F7DFE1E; Fri, 10 Mar 2017 07:16:09 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: kasha@apache.org To: common-commits@hadoop.apache.org Message-Id: <8f7680eb59224f8eac67774b0c5a3d33@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: hadoop git commit: YARN-6264. AM not launched when a single vcore is available on the cluster. (Yufei Gu via kasha) Date: Fri, 10 Mar 2017 07:16:09 +0000 (UTC) archived-at: Fri, 10 Mar 2017 07:16:11 -0000 Repository: hadoop Updated Branches: refs/heads/trunk c5ee7fded -> a96afae12 YARN-6264. AM not launched when a single vcore is available on the cluster. (Yufei Gu via kasha) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a96afae1 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a96afae1 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a96afae1 Branch: refs/heads/trunk Commit: a96afae125ba02fb4480542d3fb0891623ee4c37 Parents: c5ee7fde Author: Karthik Kambatla Authored: Thu Mar 9 23:11:54 2017 -0800 Committer: Karthik Kambatla Committed: Thu Mar 9 23:11:54 2017 -0800 ---------------------------------------------------------------------- .../hadoop/yarn/util/resource/Resources.java | 7 +++++ .../yarn/util/resource/TestResources.java | 24 ++++++++++++++++- .../scheduler/fair/FSLeafQueue.java | 3 ++- .../scheduler/fair/TestFairScheduler.java | 28 ++++++++++---------- 4 files changed, 46 insertions(+), 16 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/a96afae1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java index 57b3a46..7020300 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java @@ -242,6 +242,13 @@ public class Resources { out.setVirtualCores((int)(lhs.getVirtualCores() * by)); return out; } + + public static Resource multiplyAndRoundUp(Resource lhs, double by) { + Resource out = clone(lhs); + out.setMemorySize((long)Math.ceil(lhs.getMemorySize() * by)); + out.setVirtualCores((int)Math.ceil(lhs.getVirtualCores() * by)); + return out; + } public static Resource normalize( ResourceCalculator calculator, Resource lhs, Resource min, http://git-wip-us.apache.org/repos/asf/hadoop/blob/a96afae1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResources.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResources.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResources.java index 057214b..f8570a8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResources.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResources.java @@ -20,6 +20,8 @@ package org.apache.hadoop.yarn.util.resource; import org.apache.hadoop.yarn.api.records.Resource; import org.junit.Test; + +import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; public class TestResources { @@ -46,5 +48,25 @@ public class TestResources { assertTrue(Resources.none().compareTo( createResource(0, 1)) < 0); } - + + @Test + public void testMultipleRoundUp() { + final double by = 0.5; + final String memoryErrorMsg = "Invalid memory size."; + final String vcoreErrorMsg = "Invalid virtual core number."; + Resource resource = Resources.createResource(1, 1); + Resource result = Resources.multiplyAndRoundUp(resource, by); + assertEquals(memoryErrorMsg, result.getMemorySize(), 1); + assertEquals(vcoreErrorMsg, result.getVirtualCores(), 1); + + resource = Resources.createResource(2, 2); + result = Resources.multiplyAndRoundUp(resource, by); + assertEquals(memoryErrorMsg, result.getMemorySize(), 1); + assertEquals(vcoreErrorMsg, result.getVirtualCores(), 1); + + resource = Resources.createResource(0, 0); + result = Resources.multiplyAndRoundUp(resource, by); + assertEquals(memoryErrorMsg, result.getMemorySize(), 0); + assertEquals(vcoreErrorMsg, result.getVirtualCores(), 0); + } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/a96afae1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java index aad2916..5be4b36 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java @@ -515,7 +515,8 @@ public class FSLeafQueue extends FSQueue { getMaxShare().getVirtualCores())); } - return Resources.multiply(maxResource, maxAMShare); + // Round up to allow AM to run when there is only one vcore on the cluster + return Resources.multiplyAndRoundUp(maxResource, maxAMShare); } /** http://git-wip-us.apache.org/repos/asf/hadoop/blob/a96afae1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java index 028eea6..cdf0d81 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java @@ -659,15 +659,13 @@ public class TestFairScheduler extends FairSchedulerTestBase { // case, we use maxShare, since it is smaller than available resource. assertEquals("QueueFSZeroWithMax's fair share should be zero", 0, queueFSZeroWithMax.getFairShare().getMemorySize()); + Resource expectedAMResource = Resources.multiplyAndRoundUp( + queueFSZeroWithMax.getMaxShare(), queueFSZeroWithMax.getMaxAMShare()); assertEquals("QueueFSZeroWithMax's maximum AM resource should be " - + "maxShare * maxAMShare", - (long)(queueFSZeroWithMax.getMaxShare().getMemorySize() * - queueFSZeroWithMax.getMaxAMShare()), + + "maxShare * maxAMShare", expectedAMResource.getMemorySize(), queueFSZeroWithMax.getMetrics().getMaxAMShareMB()); assertEquals("QueueFSZeroWithMax's maximum AM resource should be " - + "maxShare * maxAMShare", - (long)(queueFSZeroWithMax.getMaxShare().getVirtualCores() * - queueFSZeroWithMax.getMaxAMShare()), + + "maxShare * maxAMShare", expectedAMResource.getVirtualCores(), queueFSZeroWithMax.getMetrics().getMaxAMShareVCores()); assertEquals("QueueFSZeroWithMax's AM resource usage should be the same to " + "AM resource request", @@ -689,17 +687,19 @@ public class TestFairScheduler extends FairSchedulerTestBase { // the min(maxShare, available resource) to compute maxAMShare, in this // case, we use available resource since it is smaller than the // default maxShare. + expectedAMResource = Resources.multiplyAndRoundUp( + Resources.createResource(memCapacity - amResource.getMemorySize(), + cpuCapacity - amResource.getVirtualCores()), + queueFSZeroWithAVL.getMaxAMShare()); assertEquals("QueueFSZeroWithAVL's fair share should be zero", 0, queueFSZeroWithAVL.getFairShare().getMemorySize()); assertEquals("QueueFSZeroWithAVL's maximum AM resource should be " + " available resource * maxAMShare", - (long) ((memCapacity - amResource.getMemorySize()) * - queueFSZeroWithAVL.getMaxAMShare()), + expectedAMResource.getMemorySize(), queueFSZeroWithAVL.getMetrics().getMaxAMShareMB()); assertEquals("QueueFSZeroWithAVL's maximum AM resource should be " + " available resource * maxAMShare", - (long) ((cpuCapacity - amResource.getVirtualCores()) * - queueFSZeroWithAVL.getMaxAMShare()), + expectedAMResource.getVirtualCores(), queueFSZeroWithAVL.getMetrics().getMaxAMShareVCores()); assertEquals("QueueFSZeroWithMax's AM resource usage should be the same to " + "AM resource request", @@ -721,13 +721,13 @@ public class TestFairScheduler extends FairSchedulerTestBase { // fair share to compute maxAMShare assertNotEquals("QueueFSNonZero's fair share shouldn't be zero", 0, queueFSNonZero.getFairShare().getMemorySize()); + expectedAMResource = Resources.multiplyAndRoundUp( + queueFSNonZero.getFairShare(), queueFSNonZero.getMaxAMShare()); assertEquals("QueueFSNonZero's maximum AM resource should be " - + " fair share * maxAMShare", - (long)(memCapacity * queueFSNonZero.getMaxAMShare()), + + " fair share * maxAMShare", expectedAMResource.getMemorySize(), queueFSNonZero.getMetrics().getMaxAMShareMB()); assertEquals("QueueFSNonZero's maximum AM resource should be " - + " fair share * maxAMShare", - (long)(cpuCapacity * queueFSNonZero.getMaxAMShare()), + + " fair share * maxAMShare", expectedAMResource.getVirtualCores(), queueFSNonZero.getMetrics().getMaxAMShareVCores()); assertEquals("QueueFSNonZero's AM resource usage should be the same to " + "AM resource request", --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org For additional commands, e-mail: common-commits-help@hadoop.apache.org