hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jh...@apache.org
Subject [hadoop] branch branch-3.2 updated: YARN-9810. Add queue capacity/maxcapacity percentage metrics. Contributed by Shubham Gupta
Date Thu, 05 Sep 2019 21:05:56 GMT
This is an automated email from the ASF dual-hosted git repository.

jhung pushed a commit to branch branch-3.2
in repository https://gitbox.apache.org/repos/asf/hadoop.git


The following commit(s) were added to refs/heads/branch-3.2 by this push:
     new 79ca399  YARN-9810. Add queue capacity/maxcapacity percentage metrics. Contributed
by Shubham Gupta
79ca399 is described below

commit 79ca399a30d48b15b6b8d9eae101f016b69ad7a9
Author: Jonathan Hung <jhung@linkedin.com>
AuthorDate: Thu Sep 5 14:01:42 2019 -0700

    YARN-9810. Add queue capacity/maxcapacity percentage metrics. Contributed by Shubham Gupta
    
    (cherry picked from commit 0ccf4b0fe16a8c879a560f2a612a3185eb2df72b)
---
 .../scheduler/capacity/CSQueueMetrics.java         | 39 ++++++++++++++++++++++
 .../scheduler/capacity/CSQueueUtils.java           |  6 ++++
 .../scheduler/capacity/TestCapacityScheduler.java  | 18 ++++++++++
 3 files changed, 63 insertions(+)

diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueMetrics.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueMetrics.java
index 8382665..c50a1b1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueMetrics.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueMetrics.java
@@ -55,6 +55,14 @@ public class CSQueueMetrics extends QueueMetrics {
   MutableGaugeLong maxCapacityMB;
   @Metric("Maximum CPU in virtual cores")
   MutableGaugeInt maxCapacityVCores;
+  @Metric("Guaranteed capacity in percentage relative to parent")
+  private MutableGaugeFloat guaranteedCapacity;
+  @Metric("Guaranteed capacity in percentage relative to total partition")
+  private MutableGaugeFloat guaranteedAbsoluteCapacity;
+  @Metric("Maximum capacity in percentage relative to parent")
+  private MutableGaugeFloat maxCapacity;
+  @Metric("Maximum capacity in percentage relative to total partition")
+  private MutableGaugeFloat maxAbsoluteCapacity;
 
   CSQueueMetrics(MetricsSystem ms, String queueName, Queue parent,
       boolean enableUserMetrics, Configuration conf) {
@@ -204,4 +212,35 @@ public class CSQueueMetrics extends QueueMetrics {
     return metrics;
   }
 
+  public float getGuaranteedCapacity() {
+    return guaranteedCapacity.value();
+  }
+
+  public float getGuaranteedAbsoluteCapacity() {
+    return guaranteedAbsoluteCapacity.value();
+  }
+
+  public void setGuaranteedCapacities(String partition, float capacity,
+      float absoluteCapacity) {
+    if (partition == null || partition.equals(RMNodeLabelsManager.NO_LABEL)) {
+      guaranteedCapacity.set(capacity);
+      guaranteedAbsoluteCapacity.set(absoluteCapacity);
+    }
+  }
+
+  public float getMaxCapacity() {
+    return maxCapacity.value();
+  }
+
+  public float getMaxAbsoluteCapacity() {
+    return maxAbsoluteCapacity.value();
+  }
+
+  public void setMaxCapacities(String partition, float capacity,
+      float absoluteCapacity) {
+    if (partition == null || partition.equals(RMNodeLabelsManager.NO_LABEL)) {
+      maxCapacity.set(capacity);
+      maxAbsoluteCapacity.set(absoluteCapacity);
+    }
+  }
 }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java
index 5c5f30f..715f80b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java
@@ -336,5 +336,11 @@ public class CSQueueUtils {
      queue.getMetrics().setMaxCapacityResources(partition, rc.multiplyAndNormalizeDown(
          partitionResource, queue.getQueueCapacities().getAbsoluteMaximumCapacity(partition),
          queue.getMinimumAllocation()));
+    queue.getMetrics().setGuaranteedCapacities(partition,
+        queue.getQueueCapacities().getCapacity(partition),
+        queue.getQueueCapacities().getAbsoluteCapacity(partition));
+    queue.getMetrics().setMaxCapacities(partition,
+        queue.getQueueCapacities().getMaximumCapacity(partition),
+        queue.getQueueCapacities().getAbsoluteMaximumCapacity(partition));
    }
 }
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
index 56882a7..51900b5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
@@ -184,6 +184,8 @@ public class TestCapacityScheduler extends CapacitySchedulerTestBase {
   private ResourceManager resourceManager = null;
   private RMContext mockContext;
 
+  private static final double DELTA = 0.000001;
+
   @Before
   public void setUp() throws Exception {
     resourceManager = new ResourceManager() {
@@ -5196,6 +5198,22 @@ public class TestCapacityScheduler extends CapacitySchedulerTestBase
{
     assertEquals(35840, ((CSQueueMetrics)cs.getQueue("b1").getMetrics()).getGuaranteedMB());
     assertEquals(51200, ((CSQueueMetrics)cs.getQueue("a").getMetrics()).getMaxCapacityMB());
     assertEquals(51200, ((CSQueueMetrics)cs.getQueue("b1").getMetrics()).getMaxCapacityMB());
+    assertEquals(A_CAPACITY / 100, ((CSQueueMetrics)cs.getQueue("a")
+        .getMetrics()).getGuaranteedCapacity(), DELTA);
+    assertEquals(A_CAPACITY / 100, ((CSQueueMetrics)cs.getQueue("a")
+        .getMetrics()).getGuaranteedAbsoluteCapacity(), DELTA);
+    assertEquals(B1_CAPACITY / 100, ((CSQueueMetrics)cs.getQueue("b1")
+        .getMetrics()).getGuaranteedCapacity(), DELTA);
+    assertEquals((B_CAPACITY / 100) * (B1_CAPACITY / 100), ((CSQueueMetrics)cs
+        .getQueue("b1").getMetrics()).getGuaranteedAbsoluteCapacity(), DELTA);
+    assertEquals(1, ((CSQueueMetrics)cs.getQueue("a").getMetrics())
+        .getMaxCapacity(), DELTA);
+    assertEquals(1, ((CSQueueMetrics)cs.getQueue("a").getMetrics())
+        .getMaxAbsoluteCapacity(), DELTA);
+    assertEquals(1, ((CSQueueMetrics)cs.getQueue("b1").getMetrics())
+        .getMaxCapacity(), DELTA);
+    assertEquals(1, ((CSQueueMetrics)cs.getQueue("b1").getMetrics())
+        .getMaxAbsoluteCapacity(), DELTA);
 
     // Add child queue to a, and reinitialize. Metrics should be updated
     conf.setQueues(CapacitySchedulerConfiguration.ROOT + ".a", new String[] {"a1", "a2",
"a3"} );


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


Mime
View raw message