hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ka...@apache.org
Subject hadoop git commit: YARN-4878. Expose scheduling policy and max running apps over JMX for Yarn queues. (Yufei Gu via kasha)
Date Tue, 24 May 2016 17:54:17 GMT
Repository: hadoop
Updated Branches:
  refs/heads/trunk 57c31a3fe -> f979d779e


YARN-4878. Expose scheduling policy and max running apps over JMX for Yarn queues. (Yufei
Gu via kasha)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f979d779
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f979d779
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f979d779

Branch: refs/heads/trunk
Commit: f979d779e192ac05906b7eb880dfc8884d0078a0
Parents: 57c31a3
Author: Karthik Kambatla <kasha@apache.org>
Authored: Tue May 24 10:54:11 2016 -0700
Committer: Karthik Kambatla <kasha@apache.org>
Committed: Tue May 24 10:54:11 2016 -0700

----------------------------------------------------------------------
 .../resourcemanager/scheduler/fair/FSQueue.java |  4 ++++
 .../scheduler/fair/FSQueueMetrics.java          | 25 +++++++++++++++++---
 .../scheduler/fair/QueueManager.java            |  5 +++-
 .../scheduler/fair/TestFSLeafQueue.java         | 12 ++++++++++
 4 files changed, 42 insertions(+), 4 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/f979d779/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java
index a33084f..4e6b99d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java
@@ -70,6 +70,10 @@ public abstract class FSQueue implements Queue, Schedulable {
     this.metrics = FSQueueMetrics.forQueue(getName(), parent, true, scheduler.getConf());
     metrics.setMinShare(getMinShare());
     metrics.setMaxShare(getMaxShare());
+
+    AllocationConfiguration allocConf = scheduler.getAllocationConfiguration();
+    metrics.setMaxApps(allocConf.getQueueMaxApps(name));
+    metrics.setSchedulingPolicy(allocConf.getSchedulingPolicy(name).getName());
     this.parent = parent;
   }
   

http://git-wip-us.apache.org/repos/asf/hadoop/blob/f979d779/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueueMetrics.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueueMetrics.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueueMetrics.java
index 82c422b..109cce5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueueMetrics.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueueMetrics.java
@@ -39,7 +39,10 @@ public class FSQueueMetrics extends QueueMetrics {
   @Metric("Minimum share of CPU in vcores") MutableGaugeInt minShareVCores;
   @Metric("Maximum share of memory in MB") MutableGaugeInt maxShareMB;
   @Metric("Maximum share of CPU in vcores") MutableGaugeInt maxShareVCores;
-  
+  @Metric("Maximum number of applications") MutableGaugeInt maxApps;
+
+  private String schedulingPolicy;
+
   FSQueueMetrics(MetricsSystem ms, String queueName, Queue parent,
       boolean enableUserMetrics, Configuration conf) {
     super(ms, queueName, parent, enableUserMetrics, conf);
@@ -96,8 +99,24 @@ public class FSQueueMetrics extends QueueMetrics {
   public int getMaxShareVirtualCores() {
     return maxShareVCores.value();
   }
-  
-  public synchronized 
+
+  public int getMaxApps() {
+    return maxApps.value();
+  }
+
+  public void setMaxApps(int max) {
+    maxApps.set(max);
+  }
+
+  public String getSchedulingPolicy() {
+    return schedulingPolicy;
+  }
+
+  public void setSchedulingPolicy(String policy) {
+    schedulingPolicy = policy;
+  }
+
+  public synchronized
   static FSQueueMetrics forQueue(String queueName, Queue parent,
       boolean enableUserMetrics, Configuration conf) {
     MetricsSystem ms = DefaultMetricsSystem.instance();

http://git-wip-us.apache.org/repos/asf/hadoop/blob/f979d779/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueueManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueueManager.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueueManager.java
index 51a298b..aeadcf6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueueManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueueManager.java
@@ -430,11 +430,14 @@ public class QueueManager {
       FSQueueMetrics queueMetrics = queue.getMetrics();
       queueMetrics.setMinShare(queue.getMinShare());
       queueMetrics.setMaxShare(queue.getMaxShare());
-      // Set scheduling policies
+      // Set scheduling policies and update queue metrics
       try {
         SchedulingPolicy policy = queueConf.getSchedulingPolicy(queue.getName());
         policy.initialize(scheduler.getClusterResource());
         queue.setPolicy(policy);
+
+        queueMetrics.setMaxApps(queueConf.getQueueMaxApps(queue.getName()));
+        queueMetrics.setSchedulingPolicy(policy.getName());
       } catch (AllocationConfigurationException ex) {
         LOG.warn("Cannot apply configured scheduling policy to queue "
             + queue.getName(), ex);

http://git-wip-us.apache.org/repos/asf/hadoop/blob/f979d779/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSLeafQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSLeafQueue.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSLeafQueue.java
index 7daccad..867fd22 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSLeafQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSLeafQueue.java
@@ -86,7 +86,14 @@ public class TestFSLeafQueue extends FairSchedulerTestBase {
     String queueName = "root.queue1";
     when(scheduler.allocConf.getMaxResources(queueName)).thenReturn(maxResource);
     when(scheduler.allocConf.getMinResources(queueName)).thenReturn(Resources.none());
+    when(scheduler.allocConf.getQueueMaxApps(queueName)).
+        thenReturn(Integer.MAX_VALUE);
+    when(scheduler.allocConf.getSchedulingPolicy(queueName))
+        .thenReturn(SchedulingPolicy.DEFAULT_POLICY);
     FSLeafQueue schedulable = new FSLeafQueue(queueName, scheduler, null);
+    assertEquals(schedulable.getMetrics().getMaxApps(), Integer.MAX_VALUE);
+    assertEquals(schedulable.getMetrics().getSchedulingPolicy(),
+        SchedulingPolicy.DEFAULT_POLICY.getName());
 
     FSAppAttempt app = mock(FSAppAttempt.class);
     Mockito.when(app.getDemand()).thenReturn(maxResource);
@@ -118,6 +125,11 @@ public class TestFSLeafQueue extends FairSchedulerTestBase {
     resourceManager = new MockRM(conf);
     resourceManager.start();
     scheduler = (FairScheduler) resourceManager.getResourceScheduler();
+    for(FSQueue queue: scheduler.getQueueManager().getQueues()) {
+      assertEquals(queue.getMetrics().getMaxApps(), Integer.MAX_VALUE);
+      assertEquals(queue.getMetrics().getSchedulingPolicy(),
+          SchedulingPolicy.DEFAULT_POLICY.getName());
+    }
 
     // Add one big node (only care about aggregate capacity)
     RMNode node1 =


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


Mime
View raw message