From common-commits-return-90372-archive-asf-public=cust-asf.ponee.io@hadoop.apache.org Tue Nov 6 07:19:20 2018 Return-Path: X-Original-To: archive-asf-public@cust-asf.ponee.io Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by mx-eu-01.ponee.io (Postfix) with SMTP id 86C78180658 for ; Tue, 6 Nov 2018 07:19:19 +0100 (CET) Received: (qmail 48457 invoked by uid 500); 6 Nov 2018 06:19:18 -0000 Mailing-List: contact common-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Delivered-To: mailing list common-commits@hadoop.apache.org Received: (qmail 48448 invoked by uid 99); 6 Nov 2018 06:19:18 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 06 Nov 2018 06:19:18 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 659FEDFF66; Tue, 6 Nov 2018 06:19:18 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: aajisaka@apache.org To: common-commits@hadoop.apache.org Message-Id: <496b99d6aaf040fb857b413228f364d9@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: hadoop git commit: YARN-8858. CapacityScheduler should respect maximum node resource when per-queue maximum-allocation is being used. Contributed by Wangda Tan and WeiWei Yang. Date: Tue, 6 Nov 2018 06:19:18 +0000 (UTC) Repository: hadoop Updated Branches: refs/heads/branch-2.8 3d76d4785 -> 8b2363afe YARN-8858. CapacityScheduler should respect maximum node resource when per-queue maximum-allocation is being used. Contributed by Wangda Tan and WeiWei Yang. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8b2363af Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8b2363af Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8b2363af Branch: refs/heads/branch-2.8 Commit: 8b2363afed4e06512f133c56d9430dd18b815823 Parents: 3d76d47 Author: Akira Ajisaka Authored: Tue Nov 6 13:58:30 2018 +0900 Committer: Akira Ajisaka Committed: Tue Nov 6 13:58:30 2018 +0900 ---------------------------------------------------------------------- .../scheduler/AbstractYarnScheduler.java | 10 ++++ .../scheduler/capacity/CapacityScheduler.java | 12 +++- .../capacity/TestContainerAllocation.java | 58 ++++++++++++++++++++ 3 files changed, 79 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b2363af/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java index 46e732e..2f690b8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java @@ -230,6 +230,16 @@ public abstract class AbstractYarnScheduler return getMaximumResourceCapability(); } + @VisibleForTesting + public void setForceConfiguredMaxAllocation(boolean flag) { + maxAllocWriteLock.lock(); + try { + useConfiguredMaximumAllocationOnly = flag; + } finally { + maxAllocWriteLock.unlock(); + } + } + protected void initMaximumResourceCapability(Resource maximumAllocation) { maxAllocWriteLock.lock(); try { http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b2363af/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java index cc9f93c..7acde84 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java @@ -2056,7 +2056,17 @@ public class CapacityScheduler extends LOG.error("queue " + queueName + " is not an leaf queue"); return getMaximumResourceCapability(); } - return ((LeafQueue)queue).getMaximumAllocation(); + + // queue.getMaxAllocation returns *configured* maximum allocation. + // getMaximumResourceCapability() returns maximum allocation considers + // per-node maximum resources. So return (component-wise) min of the two. + + Resource queueMaxAllocation = ((LeafQueue)queue).getMaximumAllocation(); + Resource clusterMaxAllocationConsiderNodeMax = + getMaximumResourceCapability(); + + return Resources.componentwiseMin(queueMaxAllocation, + clusterMaxAllocationConsiderNodeMax); } private String handleMoveToPlanQueue(String targetQueueName) { http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b2363af/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerAllocation.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerAllocation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerAllocation.java index aa63e6c..24b5468 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerAllocation.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerAllocation.java @@ -36,6 +36,7 @@ import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.ResourceRequest; import org.apache.hadoop.yarn.api.records.Token; import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.exceptions.InvalidResourceRequestException; import org.apache.hadoop.yarn.security.ContainerTokenIdentifier; import org.apache.hadoop.yarn.server.api.ContainerType; import org.apache.hadoop.yarn.server.api.records.NodeHealthStatus; @@ -70,6 +71,7 @@ import org.junit.Assert; import org.junit.Before; import org.junit.Test; +import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.MAXIMUM_ALLOCATION_MB; public class TestContainerAllocation { @@ -736,4 +738,60 @@ public class TestContainerAllocation { rm1.close(); } + + @Test(timeout = 60000) + public void testContainerRejectionWhenAskBeyondDynamicMax() + throws Exception { + CapacitySchedulerConfiguration newConf = + (CapacitySchedulerConfiguration) TestUtils + .getConfigurationWithMultipleQueues(conf); + newConf.setClass(CapacitySchedulerConfiguration.RESOURCE_CALCULATOR_CLASS, + DominantResourceCalculator.class, ResourceCalculator.class); + newConf.set(CapacitySchedulerConfiguration.getQueuePrefix("root.a") + + MAXIMUM_ALLOCATION_MB, "4096"); + + MockRM rm1 = new MockRM(newConf); + rm1.start(); + + // before any node registered or before registration timeout, + // submit an app beyond queue max leads to failure. + boolean submitFailed = false; + MockNM nm1 = rm1.registerNode("h1:1234", 2 * GB, 1); + RMApp app1 = rm1.submitApp(1 * GB, "app", "user", null, "a"); + MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1); + try { + am1.allocate("*", 5 * GB, 1, null); + } catch (InvalidResourceRequestException e) { + submitFailed = true; + } + Assert.assertTrue(submitFailed); + + // Ask 3GB should also fail because node max allocation is 2GB. + try { + am1.allocate("*", 3 * GB, 1, null); + Assert.fail("expect a failure because request exceeds" + + " max node allocation"); + } catch (Exception e) { + Assert.assertTrue(e instanceof InvalidResourceRequestException); + } + + // Add a new node, now the cluster maximum should be refreshed to 3GB. + CapacityScheduler cs = (CapacityScheduler)rm1.getResourceScheduler(); + cs.setForceConfiguredMaxAllocation(false); + rm1.registerNode("h2:1234", 3 * GB, 1); + + // Now ask 4 GB still fails + submitFailed = false; + try { + am1.allocate("*", 4 * GB, 1, null); + } catch (InvalidResourceRequestException e) { + submitFailed = true; + } + Assert.assertTrue(submitFailed); + + // But ask 3 GB succeeded. + am1.allocate("*", 3 * GB, 1, null); + + rm1.close(); + } } --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org For additional commands, e-mail: common-commits-help@hadoop.apache.org