From common-commits-return-96404-archive-asf-public=cust-asf.ponee.io@hadoop.apache.org Thu Sep 5 21:05:58 2019 Return-Path: X-Original-To: archive-asf-public@cust-asf.ponee.io Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [207.244.88.153]) by mx-eu-01.ponee.io (Postfix) with SMTP id 2205A180656 for ; Thu, 5 Sep 2019 23:05:58 +0200 (CEST) Received: (qmail 58451 invoked by uid 500); 6 Sep 2019 06:18:09 -0000 Mailing-List: contact common-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Delivered-To: mailing list common-commits@hadoop.apache.org Received: (qmail 58436 invoked by uid 99); 6 Sep 2019 06:18:09 -0000 Received: from ec2-52-202-80-70.compute-1.amazonaws.com (HELO gitbox.apache.org) (52.202.80.70) by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 06 Sep 2019 06:18:09 +0000 Received: by gitbox.apache.org (ASF Mail Server at gitbox.apache.org, from userid 33) id D0D2E807C9; Thu, 5 Sep 2019 21:05:56 +0000 (UTC) Date: Thu, 05 Sep 2019 21:05:56 +0000 To: "common-commits@hadoop.apache.org" Subject: [hadoop] branch branch-3.2 updated: YARN-9810. Add queue capacity/maxcapacity percentage metrics. Contributed by Shubham Gupta MIME-Version: 1.0 Content-Type: text/plain; charset=utf-8 Content-Transfer-Encoding: 8bit Message-ID: <156771755630.32490.17663856860709447864@gitbox.apache.org> From: jhung@apache.org X-Git-Host: gitbox.apache.org X-Git-Repo: hadoop X-Git-Refname: refs/heads/branch-3.2 X-Git-Reftype: branch X-Git-Oldrev: 66627749d0543d67588775f45ac215767b950257 X-Git-Newrev: 79ca399a30d48b15b6b8d9eae101f016b69ad7a9 X-Git-Rev: 79ca399a30d48b15b6b8d9eae101f016b69ad7a9 X-Git-NotificationType: ref_changed_plus_diff X-Git-Multimail-Version: 1.5.dev Auto-Submitted: auto-generated This is an automated email from the ASF dual-hosted git repository. jhung pushed a commit to branch branch-3.2 in repository https://gitbox.apache.org/repos/asf/hadoop.git The following commit(s) were added to refs/heads/branch-3.2 by this push: new 79ca399 YARN-9810. Add queue capacity/maxcapacity percentage metrics. Contributed by Shubham Gupta 79ca399 is described below commit 79ca399a30d48b15b6b8d9eae101f016b69ad7a9 Author: Jonathan Hung AuthorDate: Thu Sep 5 14:01:42 2019 -0700 YARN-9810. Add queue capacity/maxcapacity percentage metrics. Contributed by Shubham Gupta (cherry picked from commit 0ccf4b0fe16a8c879a560f2a612a3185eb2df72b) --- .../scheduler/capacity/CSQueueMetrics.java | 39 ++++++++++++++++++++++ .../scheduler/capacity/CSQueueUtils.java | 6 ++++ .../scheduler/capacity/TestCapacityScheduler.java | 18 ++++++++++ 3 files changed, 63 insertions(+) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueMetrics.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueMetrics.java index 8382665..c50a1b1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueMetrics.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueMetrics.java @@ -55,6 +55,14 @@ public class CSQueueMetrics extends QueueMetrics { MutableGaugeLong maxCapacityMB; @Metric("Maximum CPU in virtual cores") MutableGaugeInt maxCapacityVCores; + @Metric("Guaranteed capacity in percentage relative to parent") + private MutableGaugeFloat guaranteedCapacity; + @Metric("Guaranteed capacity in percentage relative to total partition") + private MutableGaugeFloat guaranteedAbsoluteCapacity; + @Metric("Maximum capacity in percentage relative to parent") + private MutableGaugeFloat maxCapacity; + @Metric("Maximum capacity in percentage relative to total partition") + private MutableGaugeFloat maxAbsoluteCapacity; CSQueueMetrics(MetricsSystem ms, String queueName, Queue parent, boolean enableUserMetrics, Configuration conf) { @@ -204,4 +212,35 @@ public class CSQueueMetrics extends QueueMetrics { return metrics; } + public float getGuaranteedCapacity() { + return guaranteedCapacity.value(); + } + + public float getGuaranteedAbsoluteCapacity() { + return guaranteedAbsoluteCapacity.value(); + } + + public void setGuaranteedCapacities(String partition, float capacity, + float absoluteCapacity) { + if (partition == null || partition.equals(RMNodeLabelsManager.NO_LABEL)) { + guaranteedCapacity.set(capacity); + guaranteedAbsoluteCapacity.set(absoluteCapacity); + } + } + + public float getMaxCapacity() { + return maxCapacity.value(); + } + + public float getMaxAbsoluteCapacity() { + return maxAbsoluteCapacity.value(); + } + + public void setMaxCapacities(String partition, float capacity, + float absoluteCapacity) { + if (partition == null || partition.equals(RMNodeLabelsManager.NO_LABEL)) { + maxCapacity.set(capacity); + maxAbsoluteCapacity.set(absoluteCapacity); + } + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java index 5c5f30f..715f80b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java @@ -336,5 +336,11 @@ public class CSQueueUtils { queue.getMetrics().setMaxCapacityResources(partition, rc.multiplyAndNormalizeDown( partitionResource, queue.getQueueCapacities().getAbsoluteMaximumCapacity(partition), queue.getMinimumAllocation())); + queue.getMetrics().setGuaranteedCapacities(partition, + queue.getQueueCapacities().getCapacity(partition), + queue.getQueueCapacities().getAbsoluteCapacity(partition)); + queue.getMetrics().setMaxCapacities(partition, + queue.getQueueCapacities().getMaximumCapacity(partition), + queue.getQueueCapacities().getAbsoluteMaximumCapacity(partition)); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java index 56882a7..51900b5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java @@ -184,6 +184,8 @@ public class TestCapacityScheduler extends CapacitySchedulerTestBase { private ResourceManager resourceManager = null; private RMContext mockContext; + private static final double DELTA = 0.000001; + @Before public void setUp() throws Exception { resourceManager = new ResourceManager() { @@ -5196,6 +5198,22 @@ public class TestCapacityScheduler extends CapacitySchedulerTestBase { assertEquals(35840, ((CSQueueMetrics)cs.getQueue("b1").getMetrics()).getGuaranteedMB()); assertEquals(51200, ((CSQueueMetrics)cs.getQueue("a").getMetrics()).getMaxCapacityMB()); assertEquals(51200, ((CSQueueMetrics)cs.getQueue("b1").getMetrics()).getMaxCapacityMB()); + assertEquals(A_CAPACITY / 100, ((CSQueueMetrics)cs.getQueue("a") + .getMetrics()).getGuaranteedCapacity(), DELTA); + assertEquals(A_CAPACITY / 100, ((CSQueueMetrics)cs.getQueue("a") + .getMetrics()).getGuaranteedAbsoluteCapacity(), DELTA); + assertEquals(B1_CAPACITY / 100, ((CSQueueMetrics)cs.getQueue("b1") + .getMetrics()).getGuaranteedCapacity(), DELTA); + assertEquals((B_CAPACITY / 100) * (B1_CAPACITY / 100), ((CSQueueMetrics)cs + .getQueue("b1").getMetrics()).getGuaranteedAbsoluteCapacity(), DELTA); + assertEquals(1, ((CSQueueMetrics)cs.getQueue("a").getMetrics()) + .getMaxCapacity(), DELTA); + assertEquals(1, ((CSQueueMetrics)cs.getQueue("a").getMetrics()) + .getMaxAbsoluteCapacity(), DELTA); + assertEquals(1, ((CSQueueMetrics)cs.getQueue("b1").getMetrics()) + .getMaxCapacity(), DELTA); + assertEquals(1, ((CSQueueMetrics)cs.getQueue("b1").getMetrics()) + .getMaxAbsoluteCapacity(), DELTA); // Add child queue to a, and reinitialize. Metrics should be updated conf.setQueues(CapacitySchedulerConfiguration.ROOT + ".a", new String[] {"a1", "a2", "a3"} ); --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org For additional commands, e-mail: common-commits-help@hadoop.apache.org