hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From naganarasimha...@apache.org
Subject hadoop git commit: YARN-3362. Add node label usage in RM CapacityScheduler web UI. Contributed by Eric Payne.
Date Mon, 16 May 2016 18:44:00 GMT
Repository: hadoop
Updated Branches:
  refs/heads/branch-2.7 7ded648ae -> 8f1739502


YARN-3362. Add node label usage in RM CapacityScheduler web UI. Contributed by Eric Payne.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8f173950
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8f173950
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8f173950

Branch: refs/heads/branch-2.7
Commit: 8f173950299ac3ff31ffa60ce96cdea23d0bc9e2
Parents: 7ded648
Author: Naganarasimha <naganarasimha_gr@apache.org>
Authored: Tue May 17 02:39:50 2016 +0800
Committer: Naganarasimha <naganarasimha_gr@apache.org>
Committed: Tue May 17 02:39:50 2016 +0800

----------------------------------------------------------------------
 .../scheduler/capacity/AbstractCSQueue.java     |  23 +++
 .../webapp/CapacitySchedulerPage.java           | 170 +++++++++++++++----
 .../resourcemanager/webapp/RMWebServices.java   |   6 +-
 .../webapp/dao/CapacitySchedulerInfo.java       |  40 ++++-
 .../dao/CapacitySchedulerLeafQueueInfo.java     |   8 +-
 .../webapp/dao/CapacitySchedulerQueueInfo.java  |  33 ++--
 .../capacity/TestCapacityScheduler.java         |   9 +-
 7 files changed, 233 insertions(+), 56 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/8f173950/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
index 3679de6..fcf6c85 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
@@ -504,4 +504,27 @@ public abstract class AbstractCSQueue implements CSQueue {
     // non-empty
     return false;
   }
+
+  /**
+   * @param nodePartition node label to check for accessibility
+   * @return true if queue can access nodes with specified label, false if not.
+   */
+  public final boolean accessibleToPartition(final String nodePartition) {
+    // if queue's label is *, it can access any node
+    if (accessibleLabels != null
+        && accessibleLabels.contains(RMNodeLabelsManager.ANY)) {
+      return true;
+    }
+    // any queue can access to a node without label
+    if (nodePartition == null
+        || nodePartition.equals(RMNodeLabelsManager.NO_LABEL)) {
+      return true;
+    }
+    // a queue can access to a node only if it contains any label of the node
+    if (accessibleLabels != null && accessibleLabels.contains(nodePartition)) {
+      return true;
+    }
+    // sorry, you cannot access
+    return false;
+  }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/8f173950/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/CapacitySchedulerPage.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/CapacitySchedulerPage.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/CapacitySchedulerPage.java
index 2fcc12a..1f33ab6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/CapacitySchedulerPage.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/CapacitySchedulerPage.java
@@ -21,11 +21,15 @@ package org.apache.hadoop.yarn.server.resourcemanager.webapp;
 import static org.apache.hadoop.yarn.util.StringHelper.join;
 
 import java.util.ArrayList;
+import java.util.List;
 
 import org.apache.hadoop.util.StringUtils;
+import org.apache.hadoop.yarn.nodelabels.NodeLabel;
 import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
+import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueue;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueueCapacities;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.UserInfo;
 import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.CapacitySchedulerInfo;
 import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.CapacitySchedulerLeafQueueInfo;
@@ -59,46 +63,103 @@ class CapacitySchedulerPage extends RmView {
   static class CSQInfo {
     CapacitySchedulerInfo csinfo;
     CapacitySchedulerQueueInfo qinfo;
+    String label;
   }
 
   static class LeafQueueInfoBlock extends HtmlBlock {
     final CapacitySchedulerLeafQueueInfo lqinfo;
+    private String nodeLabel;
 
     @Inject LeafQueueInfoBlock(ViewContext ctx, CSQInfo info) {
       super(ctx);
       lqinfo = (CapacitySchedulerLeafQueueInfo) info.qinfo;
+      nodeLabel = info.label;
     }
 
     @Override
     protected void render(Block html) {
+      if (nodeLabel == null) {
+        renderLeafQueueInfoWithoutParition(html);
+      } else {
+        renderLeafQueueInfoWithPartition(html);
+      }
+    }
 
-      ResponseInfo ri = info("\'" + lqinfo.getQueuePath().substring(5) + "\' Queue Status").
-          _("Queue State:", lqinfo.getQueueState()).
-          _("Used Capacity:", percent(lqinfo.getUsedCapacity() / 100)).
-          _("Absolute Used Capacity:", percent(lqinfo.getAbsoluteUsedCapacity() / 100)).
-          _("Absolute Capacity:", percent(lqinfo.getAbsoluteCapacity() / 100)).
-          _("Absolute Max Capacity:", percent(lqinfo.getAbsoluteMaxCapacity() / 100)).
-          _("Used Resources:", lqinfo.getResourcesUsed().toString()).
-          _("Num Schedulable Applications:", Integer.toString(lqinfo.getNumActiveApplications())).
-          _("Num Non-Schedulable Applications:", Integer.toString(lqinfo.getNumPendingApplications())).
-          _("Num Containers:", Integer.toString(lqinfo.getNumContainers())).
-          _("Max Applications:", Integer.toString(lqinfo.getMaxApplications())).
-          _("Max Applications Per User:", Integer.toString(lqinfo.getMaxApplicationsPerUser())).
-          _("Max Application Master Resources:", lqinfo.getAMResourceLimit().toString()).
-          _("Used Application Master Resources:", lqinfo.getUsedAMResource().toString()).
-          _("Max Application Master Resources Per User:", lqinfo.getUserAMResourceLimit().toString()).
-          _("Configured Capacity:", percent(lqinfo.getCapacity() / 100)).
-          _("Configured Max Capacity:", percent(lqinfo.getMaxCapacity() / 100)).
-          _("Configured Minimum User Limit Percent:", Integer.toString(lqinfo.getUserLimit())
+ "%").
-          _("Configured User Limit Factor:", lqinfo.getUserLimitFactor()).
-          _("Accessible Node Labels:", StringUtils.join(",", lqinfo.getNodeLabels())).
-          _("Preemption:", lqinfo.getPreemptionDisabled() ? "disabled" : "enabled");
+    /*
+     * @param html block to be rendered
+     */
+    private void renderLeafQueueInfoWithPartition(final Block html) {
+      nodeLabel = nodeLabel.length() == 0 ? "<DEFAULT_PARTITION>" : nodeLabel;
+      // first display the queue's label specific details :
+      ResponseInfo ri =
+          info("\'" + lqinfo.getQueuePath().substring(5)
+              + "\' Queue Status for Partition \'" + nodeLabel + "\'");
+      renderQueueCapacityInfo(ri);
+      html._(InfoBlock.class);
+      // clear the info contents so this queue's info doesn't accumulate into
+      // another queue's info
+      ri.clear();
+
+      // second display the queue specific details :
+      ri =
+          info("\'" + lqinfo.getQueuePath().substring(5) + "\' Queue Status")
+              ._("Queue State:", lqinfo.getQueueState());
+      renderCommonLeafQueueInfo(ri);
 
       html._(InfoBlock.class);
+      // clear the info contents so this queue's info doesn't accumulate into
+      // another queue's info
+      ri.clear();
+    }
 
-      // clear the info contents so this queue's info doesn't accumulate into another queue's
info
+    /**
+     * @param html block to be rendered
+     */
+    private void renderLeafQueueInfoWithoutParition(final Block html) {
+      ResponseInfo ri =
+          info("\'" + lqinfo.getQueuePath().substring(5) + "\' Queue Status")
+              ._("Queue State:", lqinfo.getQueueState());
+      renderQueueCapacityInfo(ri);
+      renderCommonLeafQueueInfo(ri);
+      html._(InfoBlock.class);
+      // clear the info contents so this queue's info doesn't accumulate into
+      // another queue's info
       ri.clear();
     }
+
+    /**
+     * @param ri response info to be rendered
+     */
+    private void renderQueueCapacityInfo(final ResponseInfo ri) {
+      ri.
+      _("Used Capacity:", percent(lqinfo.getUsedCapacity() / 100)).
+      _("Configured Capacity:", percent(lqinfo.getCapacity() / 100)).
+      _("Configured Max Capacity:", percent(lqinfo.getMaxCapacity() / 100)).
+      _("Absolute Used Capacity:", percent(lqinfo.getAbsoluteUsedCapacity() / 100)).
+      _("Absolute Configured Capacity:", percent(lqinfo.getAbsoluteCapacity() / 100)).
+      _("Absolute Configured Max Capacity:", percent(lqinfo.getAbsoluteMaxCapacity() / 100)).
+      _("Used Resources:", lqinfo.getResourcesUsed().toString());
+    }
+
+    /**
+     * @param ri response info to be rendered
+     */
+    private void renderCommonLeafQueueInfo(final ResponseInfo ri) {
+      ri.
+      _("Num Schedulable Applications:", Integer.toString(lqinfo.getNumActiveApplications())).
+      _("Num Non-Schedulable Applications:", Integer.toString(lqinfo.getNumPendingApplications())).
+      _("Num Containers:", Integer.toString(lqinfo.getNumContainers())).
+      _("Max Applications:", Integer.toString(lqinfo.getMaxApplications())).
+      _("Max Applications Per User:", Integer.toString(lqinfo.getMaxApplicationsPerUser())).
+      _("Max Application Master Resources:", lqinfo.getAMResourceLimit().toString()).
+      _("Used Application Master Resources:", lqinfo.getUsedAMResource().toString()).
+      _("Max Application Master Resources Per User:", lqinfo.getUserAMResourceLimit().toString()).
+      _("Configured Minimum User Limit Percent:", Integer.toString(lqinfo.getUserLimit())
+ "%").
+      _("Configured User Limit Factor:", StringUtils.format(
+          "%.1f", lqinfo.getUserLimitFactor())).
+      _("Accessible Node Labels:", StringUtils.join(",", lqinfo.getNodeLabels())).
+      _("Preemption:", lqinfo.getPreemptionDisabled() ? "disabled" : "enabled");
+    }
   }
 
   static class QueueUsersInfoBlock extends HtmlBlock {
@@ -166,7 +227,7 @@ class CapacitySchedulerPage extends RmView {
               span().$style(join(width(absUsedCap/absMaxCap),
                 ";font-size:1px;left:0%;", absUsedCap > absCap ? Q_OVER : Q_UNDER)).
                 _('.')._().
-              span(".q", info.getQueuePath().substring(5))._().
+              span(".q", "Queue: " + info.getQueuePath().substring(5))._().
             span().$class("qstats").$style(left(Q_STATS_POS)).
               _(join(percent(used), " used"))._();
 
@@ -187,10 +248,14 @@ class CapacitySchedulerPage extends RmView {
   static class QueuesBlock extends HtmlBlock {
     final CapacityScheduler cs;
     final CSQInfo csqinfo;
+    private List<NodeLabel> nodeLabelsInfo;
 
     @Inject QueuesBlock(ResourceManager rm, CSQInfo info) {
       cs = (CapacityScheduler) rm.getResourceScheduler();
       csqinfo = info;
+      RMNodeLabelsManager nodeLabelManager =
+          rm.getRMContext().getNodeLabelManager();
+      nodeLabelsInfo = nodeLabelManager.pullRMNodeLabelsInfo();
     }
 
     @Override
@@ -209,12 +274,6 @@ class CapacitySchedulerPage extends RmView {
               span().$style(Q_END)._("100% ")._().
               span(".q", "default")._()._();
       } else {
-        CSQueue root = cs.getRootQueue();
-        CapacitySchedulerInfo sinfo = new CapacitySchedulerInfo(root);
-        csqinfo.csinfo = sinfo;
-        csqinfo.qinfo = null;
-
-        float used = sinfo.getUsedCapacity() / 100;
         ul.
           li().$style("margin-bottom: 1em").
             span().$style("font-weight: bold")._("Legend:")._().
@@ -226,8 +285,22 @@ class CapacitySchedulerPage extends RmView {
               _("Used (over capacity)")._().
             span().$class("qlegend ui-corner-all ui-state-default").
               _("Max Capacity")._().
-          _().
-          li().
+          _();
+
+        float used = 0;
+        if (null == nodeLabelsInfo
+            || (nodeLabelsInfo.size() == 1 && nodeLabelsInfo.get(0)
+                .getLabelName().isEmpty())) {
+          CSQueue root = cs.getRootQueue();
+          CapacitySchedulerInfo sinfo =
+              new CapacitySchedulerInfo(root, new NodeLabel(
+                  RMNodeLabelsManager.NO_LABEL));
+          csqinfo.csinfo = sinfo;
+          csqinfo.qinfo = null;
+
+          used = sinfo.getUsedCapacity() / 100;
+          //label is not enabled in the cluster or there's only "default" label,
+          ul.li().
             a(_Q).$style(width(Q_MAX_WIDTH)).
               span().$style(join(width(used), ";left:0%;",
                   used > 1 ? Q_OVER : Q_UNDER))._(".")._().
@@ -235,6 +308,41 @@ class CapacitySchedulerPage extends RmView {
             span().$class("qstats").$style(left(Q_STATS_POS)).
               _(join(percent(used), " used"))._().
             _(QueueBlock.class)._();
+        } else {
+          for (NodeLabel label : nodeLabelsInfo) {
+            CSQueue root = cs.getRootQueue();
+            CapacitySchedulerInfo sinfo =
+                new CapacitySchedulerInfo(root, label);
+            csqinfo.csinfo = sinfo;
+            csqinfo.qinfo = null;
+            csqinfo.label = label.getLabelName();
+            String nodeLabel =
+                csqinfo.label.length() == 0 ? "<DEFAULT_PARTITION>"
+                    : csqinfo.label;
+            QueueCapacities queueCapacities = root.getQueueCapacities();
+            used = queueCapacities.getUsedCapacity(label.getLabelName());
+            String partitionUiTag =
+                "Partition: " + nodeLabel + " " + label.getResource();
+            ul.li().
+            a(_Q).$style(width(Q_MAX_WIDTH)).
+              span().$style(join(width(used), ";left:0%;",
+                  used > 1 ? Q_OVER : Q_UNDER))._(".")._().
+              span(".q", partitionUiTag)._().
+            span().$class("qstats").$style(left(Q_STATS_POS)).
+              _(join(percent(used), " used"))._();
+
+            //for the queue hierarchy under label
+            UL<Hamlet> underLabel = html.ul("#pq");
+            underLabel.li().
+            a(_Q).$style(width(Q_MAX_WIDTH)).
+              span().$style(join(width(used), ";left:0%;",
+                  used > 1 ? Q_OVER : Q_UNDER))._(".")._().
+              span(".q", "Queue: root")._().
+            span().$class("qstats").$style(left(Q_STATS_POS)).
+              _(join(percent(used), " used"))._().
+            _(QueueBlock.class)._()._();
+          }
+        }
       }
       ul._()._().
       script().$type("text/javascript").

http://git-wip-us.apache.org/repos/asf/hadoop/blob/8f173950/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
index 474630a..53ef988 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
@@ -100,11 +100,13 @@ import org.apache.hadoop.yarn.exceptions.YarnException;
 import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
 import org.apache.hadoop.yarn.factories.RecordFactory;
 import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
+import org.apache.hadoop.yarn.nodelabels.NodeLabel;
 import org.apache.hadoop.yarn.security.client.RMDelegationTokenIdentifier;
 import org.apache.hadoop.yarn.server.resourcemanager.RMAuditLogger;
 import org.apache.hadoop.yarn.server.resourcemanager.RMAuditLogger.AuditConstants;
 import org.apache.hadoop.yarn.server.resourcemanager.RMServerUtils;
 import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
+import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
 import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
 import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt;
 import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
@@ -226,7 +228,9 @@ public class RMWebServices {
     if (rs instanceof CapacityScheduler) {
       CapacityScheduler cs = (CapacityScheduler) rs;
       CSQueue root = cs.getRootQueue();
-      sinfo = new CapacitySchedulerInfo(root);
+      sinfo =
+          new CapacitySchedulerInfo(root, new NodeLabel(
+              RMNodeLabelsManager.NO_LABEL));
     } else if (rs instanceof FairScheduler) {
       FairScheduler fs = (FairScheduler) rs;
       sinfo = new FairSchedulerInfo(fs);

http://git-wip-us.apache.org/repos/asf/hadoop/blob/8f173950/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerInfo.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerInfo.java
index 8580f05..7982ec3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerInfo.java
@@ -24,8 +24,11 @@ import javax.xml.bind.annotation.XmlRootElement;
 import javax.xml.bind.annotation.XmlTransient;
 import javax.xml.bind.annotation.XmlType;
 
+import org.apache.hadoop.yarn.nodelabels.NodeLabel;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.AbstractCSQueue;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueue;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.LeafQueue;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueueCapacities;
 
 import java.util.ArrayList;
 import java.util.List;
@@ -47,16 +50,22 @@ public class CapacitySchedulerInfo extends SchedulerInfo {
   public CapacitySchedulerInfo() {
   } // JAXB needs this
 
-  public CapacitySchedulerInfo(CSQueue parent) {
+  /*
+   * @param parent parent queue
+   * @param nodeLabel node partition
+   */
+  public CapacitySchedulerInfo(final CSQueue parent, final NodeLabel nodeLabel) {
+    String label = nodeLabel.getLabelName();
+    QueueCapacities parentQueueCapacities = parent.getQueueCapacities();
     this.queueName = parent.getQueueName();
-    this.usedCapacity = parent.getUsedCapacity() * 100;
-    this.capacity = parent.getCapacity() * 100;
-    float max = parent.getMaximumCapacity();
+    this.usedCapacity = parentQueueCapacities.getUsedCapacity(label) * 100;
+    this.capacity = parentQueueCapacities.getCapacity(label) * 100;
+    float max = parentQueueCapacities.getMaximumCapacity(label);
     if (max < EPSILON || max > 1f)
       max = 1f;
     this.maxCapacity = max * 100;
 
-    queues = getQueues(parent);
+    queues = getQueues(parent, nodeLabel);
   }
 
   public float getCapacity() {
@@ -79,7 +88,12 @@ public class CapacitySchedulerInfo extends SchedulerInfo {
     return this.queues;
   }
 
-  protected CapacitySchedulerQueueInfoList getQueues(CSQueue parent) {
+  /*
+   * @param parent parent queue
+   * @param nodeLabel node partition
+   */
+  protected final CapacitySchedulerQueueInfoList getQueues(final CSQueue parent,
+      final NodeLabel nodeLabel) {
     CapacitySchedulerQueueInfoList queuesInfo =
         new CapacitySchedulerQueueInfoList();
 
@@ -91,6 +105,12 @@ public class CapacitySchedulerInfo extends SchedulerInfo {
     List<CSQueue> childLeafQueues = new ArrayList<>();
     List<CSQueue> childNonLeafQueues = new ArrayList<>();
     for (CSQueue queue : parent.getChildQueues()) {
+      if (!((AbstractCSQueue) queue).accessibleToPartition(nodeLabel
+              .getLabelName())) {
+        // Skip displaying the hierarchy for the queues for which the exclusive
+        // labels are not accessible
+        continue;
+      }
       if (queue instanceof LeafQueue) {
         childLeafQueues.add(queue);
       } else {
@@ -103,10 +123,12 @@ public class CapacitySchedulerInfo extends SchedulerInfo {
     for (CSQueue queue : childQueues) {
       CapacitySchedulerQueueInfo info;
       if (queue instanceof LeafQueue) {
-        info = new CapacitySchedulerLeafQueueInfo((LeafQueue) queue);
+        info =
+            new CapacitySchedulerLeafQueueInfo((LeafQueue) queue,
+                nodeLabel.getLabelName());
       } else {
-        info = new CapacitySchedulerQueueInfo(queue);
-        info.queues = getQueues(queue);
+        info = new CapacitySchedulerQueueInfo(queue, nodeLabel.getLabelName());
+        info.queues = getQueues(queue, nodeLabel);
       }
       queuesInfo.addToQueueInfoList(info);
     }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/8f173950/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerLeafQueueInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerLeafQueueInfo.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerLeafQueueInfo.java
index 5258b3d..08552ab 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerLeafQueueInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerLeafQueueInfo.java
@@ -43,8 +43,12 @@ public class CapacitySchedulerLeafQueueInfo extends CapacitySchedulerQueueInfo
{
   CapacitySchedulerLeafQueueInfo() {
   };
 
-  CapacitySchedulerLeafQueueInfo(LeafQueue q) {
-    super(q);
+  /*
+   * @param q leaf queue
+   * @param nodeLabel node partition
+   */
+  CapacitySchedulerLeafQueueInfo(final LeafQueue q, final String nodeLabel) {
+    super(q, nodeLabel);
     numActiveApplications = q.getNumActiveApplications();
     numPendingApplications = q.getNumPendingApplications();
     numContainers = q.getNumContainers();

http://git-wip-us.apache.org/repos/asf/hadoop/blob/8f173950/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java
index 40dddea..d83a240 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java
@@ -28,8 +28,10 @@ import javax.xml.bind.annotation.XmlSeeAlso;
 import javax.xml.bind.annotation.XmlTransient;
 
 import org.apache.hadoop.yarn.api.records.QueueState;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceUsage;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueue;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.PlanQueue;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueueCapacities;
 
 @XmlRootElement
 @XmlAccessorType(XmlAccessType.FIELD)
@@ -59,28 +61,37 @@ public class CapacitySchedulerQueueInfo {
   CapacitySchedulerQueueInfo() {
   };
 
-  CapacitySchedulerQueueInfo(CSQueue q) {
+  /*
+   * @param q capacity scheduler queue
+   * @param nodeLabel node partition
+   */
+  CapacitySchedulerQueueInfo(final CSQueue q, final String nodeLabel) {
+    QueueCapacities qCapacities = q.getQueueCapacities();
+    ResourceUsage queueResourceUsage = q.getQueueResourceUsage();
+
     queuePath = q.getQueuePath();
-    capacity = q.getCapacity() * 100;
-    usedCapacity = q.getUsedCapacity() * 100;
+    capacity = qCapacities.getCapacity(nodeLabel) * 100;
+    usedCapacity = qCapacities.getUsedCapacity(nodeLabel) * 100;
 
-    maxCapacity = q.getMaximumCapacity();
+    maxCapacity = qCapacities.getMaximumCapacity(nodeLabel);
     if (maxCapacity < EPSILON || maxCapacity > 1f)
       maxCapacity = 1f;
     maxCapacity *= 100;
 
-    absoluteCapacity = cap(q.getAbsoluteCapacity(), 0f, 1f) * 100;
-    absoluteMaxCapacity = cap(q.getAbsoluteMaximumCapacity(), 0f, 1f) * 100;
-    absoluteUsedCapacity = cap(q.getAbsoluteUsedCapacity(), 0f, 1f) * 100;
+    absoluteCapacity =
+        cap(qCapacities.getAbsoluteCapacity(nodeLabel), 0f, 1f) * 100;
+    absoluteMaxCapacity =
+        cap(qCapacities.getAbsoluteMaximumCapacity(nodeLabel), 0f, 1f) * 100;
+    absoluteUsedCapacity =
+        cap(qCapacities.getAbsoluteUsedCapacity(nodeLabel), 0f, 1f) * 100;
     numApplications = q.getNumApplications();
     queueName = q.getQueueName();
     state = q.getState();
-    resourcesUsed = new ResourceInfo(q.getUsedResources());
-    if(q instanceof PlanQueue &&
-       !((PlanQueue)q).showReservationsAsQueues()) {
+    resourcesUsed = new ResourceInfo(queueResourceUsage.getUsed(nodeLabel));
+    if (q instanceof PlanQueue && !((PlanQueue) q).showReservationsAsQueues()) {
       hideReservationQueues = true;
     }
-    
+
     // add labels
     Set<String> labelSet = q.getAccessibleNodeLabels();
     if (labelSet != null) {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/8f173950/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
index 85a2a7b..e4920e8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
@@ -75,6 +75,7 @@ import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
 import org.apache.hadoop.yarn.factories.RecordFactory;
 import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
 import org.apache.hadoop.yarn.ipc.YarnRPC;
+import org.apache.hadoop.yarn.nodelabels.NodeLabel;
 import org.apache.hadoop.yarn.server.api.protocolrecords.UpdateNodeResourceRequest;
 import org.apache.hadoop.yarn.server.resourcemanager.AdminService;
 import org.apache.hadoop.yarn.server.resourcemanager.Application;
@@ -1675,7 +1676,9 @@ public class TestCapacityScheduler {
     CapacityScheduler cs =
         (CapacityScheduler) resourceManager.getResourceScheduler();
     CSQueue origRootQ = cs.getRootQueue();
-    CapacitySchedulerInfo oldInfo = new CapacitySchedulerInfo(origRootQ);
+    CapacitySchedulerInfo oldInfo =
+        new CapacitySchedulerInfo(origRootQ, new NodeLabel(
+            RMNodeLabelsManager.NO_LABEL));
     int origNumAppsA = getNumAppsInQueue("a", origRootQ.getChildQueues());
     int origNumAppsRoot = origRootQ.getNumApplications();
 
@@ -1684,7 +1687,9 @@ public class TestCapacityScheduler {
     CSQueue newRootQ = cs.getRootQueue();
     int newNumAppsA = getNumAppsInQueue("a", newRootQ.getChildQueues());
     int newNumAppsRoot = newRootQ.getNumApplications();
-    CapacitySchedulerInfo newInfo = new CapacitySchedulerInfo(newRootQ);
+    CapacitySchedulerInfo newInfo =
+        new CapacitySchedulerInfo(newRootQ, new NodeLabel(
+            RMNodeLabelsManager.NO_LABEL));
     CapacitySchedulerLeafQueueInfo origOldA1 =
         (CapacitySchedulerLeafQueueInfo) getQueueInfo("a1", oldInfo.getQueues());
     CapacitySchedulerLeafQueueInfo origNewA1 =


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


Mime
View raw message