hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ji...@apache.org
Subject [27/50] [abbrv] git commit: YARN-2360. Fair Scheduler: Display dynamic fair share for queues on the scheduler page. (Ashwin Shankar and Wei Yan via kasha)
Date Wed, 03 Sep 2014 22:49:39 GMT
YARN-2360. Fair Scheduler: Display dynamic fair share for queues on the scheduler page. (Ashwin
Shankar and Wei Yan via kasha)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/270a271f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/270a271f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/270a271f

Branch: refs/heads/HDFS-6584
Commit: 270a271f53f52a1f33e63a0520ae6032536dd4c6
Parents: 9ad413b
Author: Karthik Kambatla <kasha@apache.org>
Authored: Fri Aug 29 17:15:38 2014 -0700
Committer: Karthik Kambatla <kasha@apache.org>
Committed: Fri Aug 29 17:15:38 2014 -0700

----------------------------------------------------------------------
 hadoop-yarn-project/CHANGES.txt                 |  3 ++
 .../webapp/FairSchedulerPage.java               | 31 ++++++++++++------
 .../webapp/dao/FairSchedulerQueueInfo.java      | 34 ++++++++++++++++----
 .../src/site/apt/FairScheduler.apt.vm           | 20 ++++++++----
 4 files changed, 65 insertions(+), 23 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/270a271f/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 4cd4529..f390967 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -163,6 +163,9 @@ Release 2.6.0 - UNRELEASED
     YARN-2406. Move RM recovery related proto to
     yarn_server_resourcemanager_recovery.proto. (Tsuyoshi Ozawa via jianhe)
 
+    YARN-2360. Fair Scheduler: Display dynamic fair share for queues on the 
+    scheduler page. (Ashwin Shankar and Wei Yan via kasha)
+
   OPTIMIZATIONS
 
   BUG FIXES

http://git-wip-us.apache.org/repos/asf/hadoop/blob/270a271f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/FairSchedulerPage.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/FairSchedulerPage.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/FairSchedulerPage.java
index aca3e44..bcf7781 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/FairSchedulerPage.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/FairSchedulerPage.java
@@ -44,10 +44,12 @@ public class FairSchedulerPage extends RmView {
   static final float Q_MAX_WIDTH = 0.8f;
   static final float Q_STATS_POS = Q_MAX_WIDTH + 0.05f;
   static final String Q_END = "left:101%";
-  static final String Q_GIVEN = "left:0%;background:none;border:1px dashed rgba(0,0,0,0.25)";
+  static final String Q_GIVEN = "left:0%;background:none;border:1px solid rgba(0,0,0,1)";
+  static final String Q_INSTANTANEOUS_FS = "left:0%;background:none;border:1px dashed rgba(0,0,0,1)";
   static final String Q_OVER = "background:rgba(255, 140, 0, 0.8)";
   static final String Q_UNDER = "background:rgba(50, 205, 50, 0.8)";
-  
+  static final String STEADY_FAIR_SHARE = "Steady Fair Share";
+  static final String INSTANTANEOUS_FAIR_SHARE = "Instantaneous Fair Share";
   @RequestScoped
   static class FSQInfo {
     FairSchedulerQueueInfo qinfo;
@@ -73,8 +75,8 @@ public class FairSchedulerPage extends RmView {
       if (maxApps < Integer.MAX_VALUE) {
           ri._("Max Running Applications:", qinfo.getMaxApplications());
       }
-      ri._("Fair Share:", qinfo.getFairShare().toString());
-
+      ri._(STEADY_FAIR_SHARE + ":", qinfo.getSteadyFairShare().toString());
+      ri._(INSTANTANEOUS_FAIR_SHARE + ":", qinfo.getFairShare().toString());
       html._(InfoBlock.class);
 
       // clear the info contents so this queue's info doesn't accumulate into another queue's
info
@@ -95,16 +97,21 @@ public class FairSchedulerPage extends RmView {
       UL<Hamlet> ul = html.ul("#pq");
       for (FairSchedulerQueueInfo info : subQueues) {
         float capacity = info.getMaxResourcesFraction();
-        float fairShare = info.getFairShareMemoryFraction();
+        float steadyFairShare = info.getSteadyFairShareMemoryFraction();
+        float instantaneousFairShare = info.getFairShareMemoryFraction();
         float used = info.getUsedMemoryFraction();
         LI<UL<Hamlet>> li = ul.
           li().
             a(_Q).$style(width(capacity * Q_MAX_WIDTH)).
-              $title(join("Fair Share:", percent(fairShare))).
-              span().$style(join(Q_GIVEN, ";font-size:1px;", width(fairShare/capacity))).
+              $title(join(join(STEADY_FAIR_SHARE + ":", percent(steadyFairShare)),
+                  join(" " + INSTANTANEOUS_FAIR_SHARE + ":", percent(instantaneousFairShare)))).
+              span().$style(join(Q_GIVEN, ";font-size:1px;", width(steadyFairShare / capacity))).
+                _('.')._().
+              span().$style(join(Q_INSTANTANEOUS_FS, ";font-size:1px;",
+                  width(instantaneousFairShare/capacity))).
                 _('.')._().
               span().$style(join(width(used/capacity),
-                ";font-size:1px;left:0%;", used > fairShare ? Q_OVER : Q_UNDER)).
+                ";font-size:1px;left:0%;", used > instantaneousFairShare ? Q_OVER : Q_UNDER)).
                 _('.')._().
               span(".q", info.getQueueName())._().
             span().$class("qstats").$style(left(Q_STATS_POS)).
@@ -156,7 +163,13 @@ public class FairSchedulerPage extends RmView {
           li().$style("margin-bottom: 1em").
             span().$style("font-weight: bold")._("Legend:")._().
             span().$class("qlegend ui-corner-all").$style(Q_GIVEN).
-              _("Fair Share")._().
+              $title("The steady fair shares consider all queues, " +
+                  "both active (with running applications) and inactive.").
+              _(STEADY_FAIR_SHARE)._().
+            span().$class("qlegend ui-corner-all").$style(Q_INSTANTANEOUS_FS).
+              $title("The instantaneous fair shares consider only active " +
+                  "queues (with running applications).").
+              _(INSTANTANEOUS_FAIR_SHARE)._().
             span().$class("qlegend ui-corner-all").$style(Q_UNDER).
               _("Used")._().
             span().$class("qlegend ui-corner-all").$style(Q_OVER).

http://git-wip-us.apache.org/repos/asf/hadoop/blob/270a271f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerQueueInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerQueueInfo.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerQueueInfo.java
index 2c1bc47..c62aaf0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerQueueInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerQueueInfo.java
@@ -28,7 +28,6 @@ import javax.xml.bind.annotation.XmlRootElement;
 import javax.xml.bind.annotation.XmlSeeAlso;
 import javax.xml.bind.annotation.XmlTransient;
 
-import org.apache.hadoop.yarn.api.records.Resource;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.AllocationConfiguration;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FSLeafQueue;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FSQueue;
@@ -44,6 +43,8 @@ public class FairSchedulerQueueInfo {
   @XmlTransient
   private float fractionMemUsed;
   @XmlTransient
+  private float fractionMemSteadyFairShare;
+  @XmlTransient
   private float fractionMemFairShare;
   @XmlTransient
   private float fractionMemMinShare;
@@ -53,6 +54,7 @@ public class FairSchedulerQueueInfo {
   private ResourceInfo minResources;
   private ResourceInfo maxResources;
   private ResourceInfo usedResources;
+  private ResourceInfo steadyFairResources;
   private ResourceInfo fairResources;
   private ResourceInfo clusterResources;
   
@@ -75,15 +77,19 @@ public class FairSchedulerQueueInfo {
     usedResources = new ResourceInfo(queue.getResourceUsage());
     fractionMemUsed = (float)usedResources.getMemory() /
         clusterResources.getMemory();
-    
+
+    steadyFairResources = new ResourceInfo(queue.getSteadyFairShare());
     fairResources = new ResourceInfo(queue.getFairShare());
     minResources = new ResourceInfo(queue.getMinShare());
     maxResources = new ResourceInfo(queue.getMaxShare());
     maxResources = new ResourceInfo(
         Resources.componentwiseMin(queue.getMaxShare(),
             scheduler.getClusterResource()));
-    
-    fractionMemFairShare = (float)fairResources.getMemory() / clusterResources.getMemory();
+
+    fractionMemSteadyFairShare =
+        (float)steadyFairResources.getMemory() / clusterResources.getMemory();
+    fractionMemFairShare = (float) fairResources.getMemory()
+        / clusterResources.getMemory();
     fractionMemMinShare = (float)minResources.getMemory() / clusterResources.getMemory();
     fractionMemMaxShare = (float)maxResources.getMemory() / clusterResources.getMemory();
     
@@ -101,19 +107,33 @@ public class FairSchedulerQueueInfo {
   }
   
   /**
+   * Returns the steady fair share as a fraction of the entire cluster capacity.
+   */
+  public float getSteadyFairShareMemoryFraction() {
+    return fractionMemSteadyFairShare;
+  }
+
+  /**
    * Returns the fair share as a fraction of the entire cluster capacity.
    */
   public float getFairShareMemoryFraction() {
     return fractionMemFairShare;
   }
-  
+
+  /**
+   * Returns the steady fair share of this queue in megabytes.
+   */
+  public ResourceInfo getSteadyFairShare() {
+    return steadyFairResources;
+  }
+
   /**
-   * Returns the fair share of this queue in megabytes.
+   * Returns the fair share of this queue in megabytes
    */
   public ResourceInfo getFairShare() {
     return fairResources;
   }
-    
+
   public ResourceInfo getMinResources() {
     return minResources;
   }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/270a271f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/FairScheduler.apt.vm
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/FairScheduler.apt.vm
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/FairScheduler.apt.vm
index 9bb8563..a3edade 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/FairScheduler.apt.vm
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/FairScheduler.apt.vm
@@ -429,13 +429,19 @@ Monitoring through web UI
   	
  * Max Resources - The configured maximum resources that are allowed to the queue.
  
- * Fair Share - The queue's fair share of resources.  Queues may be allocated
-   resources beyond their fair share when other queues aren't using them.  A
-   queue whose resource consumption lies at or below its fair share will never
-   have its containers preempted.  
-
-  In addition to the information that the ResourceManager normally displays
-  about each application, the web interface includes the application's fair share.
+ * Instantaneous Fair Share - The queue's instantaneous fair share of resources.
+   These shares consider only actives queues (those with running applications),
+   and are used for scheduling decisions. Queues may be allocated resources
+   beyond their shares when other queues aren't using them. A queue whose
+   resource consumption lies at or below its instantaneous fair share will never
+   have its containers preempted.
+
+ * Steady Fair Share - The queue's steady fair share of resources. These shares
+   consider all the queues irrespective of whether they are active (have
+   running applications) or not. These are computed less frequently and
+   change only when the configuration or capacity changes.They are meant to
+   provide visibility into resources the user can expect, and hence displayed
+   in the Web UI.
 
 Moving applications between queues
 


Mime
View raw message