From common-commits-return-96331-archive-asf-public=cust-asf.ponee.io@hadoop.apache.org Tue Sep 3 07:10:32 2019 Return-Path: X-Original-To: archive-asf-public@cust-asf.ponee.io Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [207.244.88.153]) by mx-eu-01.ponee.io (Postfix) with SMTP id 62520180637 for ; Tue, 3 Sep 2019 09:10:32 +0200 (CEST) Received: (qmail 59194 invoked by uid 500); 3 Sep 2019 08:18:15 -0000 Mailing-List: contact common-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Delivered-To: mailing list common-commits@hadoop.apache.org Received: (qmail 59185 invoked by uid 99); 3 Sep 2019 08:18:15 -0000 Received: from ec2-52-202-80-70.compute-1.amazonaws.com (HELO gitbox.apache.org) (52.202.80.70) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 03 Sep 2019 08:18:15 +0000 Received: by gitbox.apache.org (ASF Mail Server at gitbox.apache.org, from userid 33) id F000B87436; Tue, 3 Sep 2019 07:10:30 +0000 (UTC) Date: Tue, 03 Sep 2019 07:10:30 +0000 To: "common-commits@hadoop.apache.org" Subject: [hadoop] branch branch-3.1 updated: YARN-9797. LeafQueue#activateApplications should use resourceCalculator#fitsIn. Contributed by Bilwa S T. MIME-Version: 1.0 Content-Type: text/plain; charset=utf-8 Content-Transfer-Encoding: 8bit Message-ID: <156749463048.7686.4540566447024792975@gitbox.apache.org> From: bibinchundatt@apache.org X-Git-Host: gitbox.apache.org X-Git-Repo: hadoop X-Git-Refname: refs/heads/branch-3.1 X-Git-Reftype: branch X-Git-Oldrev: 63b18d764ae267f5c9331ef0f839ec7bf3e1a07e X-Git-Newrev: 3210d1e993639b2b59c0fd542d30a90f075500eb X-Git-Rev: 3210d1e993639b2b59c0fd542d30a90f075500eb X-Git-NotificationType: ref_changed_plus_diff X-Git-Multimail-Version: 1.5.dev Auto-Submitted: auto-generated This is an automated email from the ASF dual-hosted git repository. bibinchundatt pushed a commit to branch branch-3.1 in repository https://gitbox.apache.org/repos/asf/hadoop.git The following commit(s) were added to refs/heads/branch-3.1 by this push: new 3210d1e YARN-9797. LeafQueue#activateApplications should use resourceCalculator#fitsIn. Contributed by Bilwa S T. 3210d1e is described below commit 3210d1e993639b2b59c0fd542d30a90f075500eb Author: bibinchundatt AuthorDate: Tue Sep 3 11:53:34 2019 +0530 YARN-9797. LeafQueue#activateApplications should use resourceCalculator#fitsIn. Contributed by Bilwa S T. (cherry picked from commit 03489124ea1b8d5648ade5e3563e39b5bc323384) --- .../scheduler/capacity/LeafQueue.java | 6 +- .../capacity/TestApplicationLimitsByPartition.java | 85 ++++++++++++++++++++++ 2 files changed, 87 insertions(+), 4 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java index 58157a4..4745326 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java @@ -838,8 +838,7 @@ public class LeafQueue extends AbstractCSQueue { + " AM node-partition name " + partitionName); } - if (!Resources.lessThanOrEqual(resourceCalculator, lastClusterResource, - amIfStarted, amLimit)) { + if (!resourceCalculator.fitsIn(amIfStarted, amLimit)) { if (getNumActiveApplications() < 1 || (Resources.lessThanOrEqual( resourceCalculator, lastClusterResource, queueUsage.getAMUsed(partitionName), Resources.none()))) { @@ -874,8 +873,7 @@ public class LeafQueue extends AbstractCSQueue { application.getAMResource(partitionName), user.getConsumedAMResources(partitionName)); - if (!Resources.lessThanOrEqual(resourceCalculator, lastClusterResource, - userAmIfStarted, userAMLimit)) { + if (!resourceCalculator.fitsIn(userAmIfStarted, userAMLimit)) { if (getNumActiveApplications() < 1 || (Resources.lessThanOrEqual( resourceCalculator, lastClusterResource, queueUsage.getAMUsed(partitionName), Resources.none()))) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimitsByPartition.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimitsByPartition.java index d73f1c8..9c578e5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimitsByPartition.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimitsByPartition.java @@ -38,6 +38,7 @@ import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.api.records.ResourceInformation; import org.apache.hadoop.yarn.api.records.ResourceRequest; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.factories.RecordFactory; @@ -48,6 +49,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.MockRM; import org.apache.hadoop.yarn.server.resourcemanager.RMContext; import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.NullRMNodeLabelsManager; import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager; +import org.apache.hadoop.yarn.server.resourcemanager.resource.TestResourceProfiles; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceLimits; @@ -58,7 +60,9 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.preempti import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerApp; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerNode; import org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator; +import org.apache.hadoop.yarn.util.resource.DominantResourceCalculator; import org.apache.hadoop.yarn.util.resource.ResourceCalculator; +import org.apache.hadoop.yarn.util.resource.ResourceUtils; import org.apache.hadoop.yarn.util.resource.Resources; import org.junit.Assert; import org.junit.Before; @@ -747,4 +751,85 @@ public class TestApplicationLimitsByPartition { } + + /** + * {@link LeafQueue#activateApplications()} should validate values of all + * resourceTypes before activating application. + * + * @throws Exception + */ + @Test + public void testAMLimitByAllResources() throws Exception { + CapacitySchedulerConfiguration csconf = + new CapacitySchedulerConfiguration(); + csconf.setResourceComparator(DominantResourceCalculator.class); + String queueName = "a1"; + csconf.setQueues(CapacitySchedulerConfiguration.ROOT, + new String[] {queueName}); + csconf.setCapacity("root." + queueName, 100); + + ResourceInformation res0 = ResourceInformation.newInstance("memory-mb", + ResourceInformation.MEMORY_MB.getUnits(), GB, Long.MAX_VALUE); + ResourceInformation res1 = ResourceInformation.newInstance("vcores", + ResourceInformation.VCORES.getUnits(), 1, Integer.MAX_VALUE); + ResourceInformation res2 = ResourceInformation.newInstance("gpu", + ResourceInformation.GPUS.getUnits(), 0, Integer.MAX_VALUE); + Map riMap = new HashMap<>(); + riMap.put(ResourceInformation.MEMORY_URI, res0); + riMap.put(ResourceInformation.VCORES_URI, res1); + riMap.put(ResourceInformation.GPU_URI, res2); + ResourceUtils.initializeResourcesFromResourceInformationMap(riMap); + + YarnConfiguration config = new YarnConfiguration(csconf); + config.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class, + ResourceScheduler.class); + config.setBoolean(TestResourceProfiles.TEST_CONF_RESET_RESOURCE_TYPES, + false); + + MockRM rm = new MockRM(config); + rm.start(); + + Map res = new HashMap<>(); + res.put("gpu", 0L); + + Resource clusterResource = Resource.newInstance(16 * GB, 64, res); + + // Cluster Resource - 16GB, 64vcores + // AMLimit 16384 x .1 mb , 64 x .1 vcore + // Effective AM limit after normalized to minimum resource 2048,7 + + rm.registerNode("127.0.0.1:1234", clusterResource); + + String userName = "user_0"; + ResourceScheduler scheduler = rm.getRMContext().getScheduler(); + LeafQueue queueA = (LeafQueue) ((CapacityScheduler) scheduler) + .getQueue(queueName); + + Resource amResource = Resource.newInstance(GB, 1); + + rm.submitApp(amResource, "app-1", userName, null, queueName); + rm.submitApp(amResource, "app-2", userName, null, queueName); + + // app-3 should not be activated as amLimit will be reached + // for memory + rm.submitApp(amResource, "app-3", userName, null, queueName); + + Assert.assertEquals("PendingApplications should be 1", 1, + queueA.getNumPendingApplications()); + Assert.assertEquals("Active applications should be 2", 2, + queueA.getNumActiveApplications()); + // AMLimit is 2048,7 + Assert.assertEquals(2048, + queueA.getQueueResourceUsage().getAMLimit().getMemorySize()); + Assert.assertEquals(7, + queueA.getQueueResourceUsage().getAMLimit().getVirtualCores()); + // Used AM Resource is 2048,2 + Assert.assertEquals(2048, + queueA.getQueueResourceUsage().getAMUsed().getMemorySize()); + Assert.assertEquals(2, + queueA.getQueueResourceUsage().getAMUsed().getVirtualCores()); + + rm.close(); + + } } --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org For additional commands, e-mail: common-commits-help@hadoop.apache.org