Return-Path: X-Original-To: apmail-hadoop-common-commits-archive@www.apache.org Delivered-To: apmail-hadoop-common-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id AD32C10660 for ; Wed, 31 Dec 2014 00:49:36 +0000 (UTC) Received: (qmail 70941 invoked by uid 500); 31 Dec 2014 00:49:37 -0000 Delivered-To: apmail-hadoop-common-commits-archive@hadoop.apache.org Received: (qmail 70872 invoked by uid 500); 31 Dec 2014 00:49:36 -0000 Mailing-List: contact common-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: common-dev@hadoop.apache.org Delivered-To: mailing list common-commits@hadoop.apache.org Received: (qmail 70860 invoked by uid 99); 31 Dec 2014 00:49:36 -0000 Received: from tyr.zones.apache.org (HELO tyr.zones.apache.org) (140.211.11.114) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 31 Dec 2014 00:49:36 +0000 Received: by tyr.zones.apache.org (Postfix, from userid 65534) id 5EDD6A3A216; Wed, 31 Dec 2014 00:49:35 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: jianhe@apache.org To: common-commits@hadoop.apache.org Date: Wed, 31 Dec 2014 00:49:36 -0000 Message-Id: In-Reply-To: <1109f6cb50934892a9681d1c4192d824@git.apache.org> References: <1109f6cb50934892a9681d1c4192d824@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [2/2] hadoop git commit: YARN-2493. Added node-labels page on RM web UI. Contributed by Wangda Tan YARN-2493. Added node-labels page on RM web UI. Contributed by Wangda Tan Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b7442bf9 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b7442bf9 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b7442bf9 Branch: refs/heads/trunk Commit: b7442bf92eb6e1ae64a0f9644ffc2eee4597aad5 Parents: 746ad6e Author: Jian He Authored: Tue Dec 30 15:35:45 2014 -0800 Committer: Jian He Committed: Tue Dec 30 16:49:01 2014 -0800 ---------------------------------------------------------------------- hadoop-yarn-project/CHANGES.txt | 2 + .../nodelabels/CommonNodeLabelsManager.java | 21 +-- .../hadoop/yarn/nodelabels/NodeLabel.java | 96 +++++++++++++ .../hadoop/yarn/webapp/YarnWebParams.java | 1 + .../nodelabels/RMNodeLabelsManager.java | 36 +++-- .../server/resourcemanager/webapp/NavBlock.java | 3 +- .../resourcemanager/webapp/NodeLabelsPage.java | 91 ++++++++++++ .../resourcemanager/webapp/NodesPage.java | 139 ++++++++++--------- .../server/resourcemanager/webapp/RMWebApp.java | 1 + .../resourcemanager/webapp/RmController.java | 6 +- .../yarn/server/resourcemanager/MockNodes.java | 46 +++++- .../TestWorkPreservingRMRestart.java | 2 +- .../nodelabels/TestRMNodeLabelsManager.java | 33 +++++ .../resourcemanager/webapp/TestNodesPage.java | 45 ++++++ 14 files changed, 421 insertions(+), 101 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/b7442bf9/hadoop-yarn-project/CHANGES.txt ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index e707ea7..0ebf8a3 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -155,6 +155,8 @@ Release 2.7.0 - UNRELEASED YARN-2993. Several fixes (missing acl check, error log msg ...) and some refinement in AdminService. (Yi Liu via junping_du) + YARN-2943. Added node-labels page on RM web UI. (Wangda Tan via jianhe) + OPTIMIZATIONS BUG FIXES http://git-wip-us.apache.org/repos/asf/hadoop/blob/b7442bf9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/CommonNodeLabelsManager.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/CommonNodeLabelsManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/CommonNodeLabelsManager.java index 070aa1f..e888cc5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/CommonNodeLabelsManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/CommonNodeLabelsManager.java @@ -72,8 +72,8 @@ public class CommonNodeLabelsManager extends AbstractService { protected Dispatcher dispatcher; - protected ConcurrentMap labelCollections = - new ConcurrentHashMap(); + protected ConcurrentMap labelCollections = + new ConcurrentHashMap(); protected ConcurrentMap nodeCollections = new ConcurrentHashMap(); @@ -82,19 +82,6 @@ public class CommonNodeLabelsManager extends AbstractService { protected NodeLabelsStore store; - protected static class Label { - private Resource resource; - - protected Label() { - this.resource = Resource.newInstance(0, 0); - } - - public Resource getResource() { - return this.resource; - } - - } - /** * A Host can have multiple Nodes */ @@ -201,7 +188,7 @@ public class CommonNodeLabelsManager extends AbstractService { protected void serviceInit(Configuration conf) throws Exception { initNodeLabelStore(conf); - labelCollections.put(NO_LABEL, new Label()); + labelCollections.put(NO_LABEL, new NodeLabel(NO_LABEL)); } protected void initNodeLabelStore(Configuration conf) throws Exception { @@ -271,7 +258,7 @@ public class CommonNodeLabelsManager extends AbstractService { for (String label : labels) { // shouldn't overwrite it to avoid changing the Label.resource if (this.labelCollections.get(label) == null) { - this.labelCollections.put(label, new Label()); + this.labelCollections.put(label, new NodeLabel(label)); newLabels.add(label); } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/b7442bf9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/NodeLabel.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/NodeLabel.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/NodeLabel.java new file mode 100644 index 0000000..7668648 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/NodeLabel.java @@ -0,0 +1,96 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.nodelabels; + +import org.apache.commons.lang.StringUtils; +import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.util.resource.Resources; + +public class NodeLabel implements Comparable { + private Resource resource; + private int numActiveNMs; + private String labelName; + + public NodeLabel(String labelName) { + this(labelName, Resource.newInstance(0, 0), 0); + } + + protected NodeLabel(String labelName, Resource res, int activeNMs) { + this.labelName = labelName; + this.resource = res; + this.numActiveNMs = activeNMs; + } + + public void addNode(Resource nodeRes) { + Resources.addTo(resource, nodeRes); + numActiveNMs++; + } + + public void removeNode(Resource nodeRes) { + Resources.subtractFrom(resource, nodeRes); + numActiveNMs--; + } + + public Resource getResource() { + return this.resource; + } + + public int getNumActiveNMs() { + return numActiveNMs; + } + + public String getLabelName() { + return labelName; + } + + public NodeLabel getCopy() { + return new NodeLabel(labelName, resource, numActiveNMs); + } + + @Override + public int compareTo(NodeLabel o) { + // We should always put empty label entry first after sorting + if (labelName.isEmpty() != o.getLabelName().isEmpty()) { + if (labelName.isEmpty()) { + return -1; + } + return 1; + } + + return labelName.compareTo(o.getLabelName()); + } + + @Override + public boolean equals(Object obj) { + if (obj instanceof NodeLabel) { + NodeLabel other = (NodeLabel) obj; + return Resources.equals(resource, other.getResource()) + && StringUtils.equals(labelName, other.getLabelName()) + && (other.getNumActiveNMs() == numActiveNMs); + } + return false; + } + + @Override + public int hashCode() { + final int prime = 502357; + return (int) ((((long) labelName.hashCode() << 8) + + (resource.hashCode() << 4) + numActiveNMs) % prime); + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/b7442bf9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/YarnWebParams.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/YarnWebParams.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/YarnWebParams.java index 91d2a20..62c3c7a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/YarnWebParams.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/YarnWebParams.java @@ -32,4 +32,5 @@ public interface YarnWebParams { String APP_STATE = "app.state"; String QUEUE_NAME = "queue.name"; String NODE_STATE = "node.state"; + String NODE_LABEL = "node.label"; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/b7442bf9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/RMNodeLabelsManager.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/RMNodeLabelsManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/RMNodeLabelsManager.java index 646441a..828d1bc 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/RMNodeLabelsManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/RMNodeLabelsManager.java @@ -19,10 +19,12 @@ package org.apache.hadoop.yarn.server.resourcemanager.nodelabels; import java.io.IOException; +import java.util.ArrayList; import java.util.Collection; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; +import java.util.List; import java.util.Map; import java.util.Map.Entry; import java.util.Set; @@ -37,6 +39,7 @@ import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.event.Dispatcher; import org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager; +import org.apache.hadoop.yarn.nodelabels.NodeLabel; import org.apache.hadoop.yarn.server.resourcemanager.RMContext; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeLabelsUpdateSchedulerEvent; import org.apache.hadoop.yarn.util.resource.Resources; @@ -360,8 +363,8 @@ public class RMNodeLabelsManager extends CommonNodeLabelsManager { // no label in the past if (oldLabels.isEmpty()) { // update labels - Label label = labelCollections.get(NO_LABEL); - Resources.subtractFrom(label.getResource(), oldNM.resource); + NodeLabel label = labelCollections.get(NO_LABEL); + label.removeNode(oldNM.resource); // update queues, all queue can access this node for (Queue q : queueCollections.values()) { @@ -370,11 +373,11 @@ public class RMNodeLabelsManager extends CommonNodeLabelsManager { } else { // update labels for (String labelName : oldLabels) { - Label label = labelCollections.get(labelName); + NodeLabel label = labelCollections.get(labelName); if (null == label) { continue; } - Resources.subtractFrom(label.getResource(), oldNM.resource); + label.removeNode(oldNM.resource); } // update queues, only queue can access this node will be subtract @@ -395,8 +398,8 @@ public class RMNodeLabelsManager extends CommonNodeLabelsManager { // no label in the past if (newLabels.isEmpty()) { // update labels - Label label = labelCollections.get(NO_LABEL); - Resources.addTo(label.getResource(), newNM.resource); + NodeLabel label = labelCollections.get(NO_LABEL); + label.addNode(newNM.resource); // update queues, all queue can access this node for (Queue q : queueCollections.values()) { @@ -405,8 +408,8 @@ public class RMNodeLabelsManager extends CommonNodeLabelsManager { } else { // update labels for (String labelName : newLabels) { - Label label = labelCollections.get(labelName); - Resources.addTo(label.getResource(), newNM.resource); + NodeLabel label = labelCollections.get(labelName); + label.addNode(newNM.resource); } // update queues, only queue can access this node will be subtract @@ -475,4 +478,21 @@ public class RMNodeLabelsManager extends CommonNodeLabelsManager { public void setRMContext(RMContext rmContext) { this.rmContext = rmContext; } + + public List pullRMNodeLabelsInfo() { + try { + readLock.lock(); + List infos = new ArrayList(); + + for (Entry entry : labelCollections.entrySet()) { + NodeLabel label = entry.getValue(); + infos.add(label.getCopy()); + } + + Collections.sort(infos); + return infos; + } finally { + readLock.unlock(); + } + } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/b7442bf9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/NavBlock.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/NavBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/NavBlock.java index ce8fd9e..db00bb0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/NavBlock.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/NavBlock.java @@ -33,7 +33,8 @@ public class NavBlock extends HtmlBlock { h3("Cluster"). ul(). li().a(url("cluster"), "About")._(). - li().a(url("nodes"), "Nodes")._(); + li().a(url("nodes"), "Nodes")._(). + li().a(url("nodelabels"), "Node Labels")._(); UL>>> subAppsList = mainList. li().a(url("apps"), "Applications"). ul(); http://git-wip-us.apache.org/repos/asf/hadoop/blob/b7442bf9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/NodeLabelsPage.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/NodeLabelsPage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/NodeLabelsPage.java new file mode 100644 index 0000000..5e8c1ed --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/NodeLabelsPage.java @@ -0,0 +1,91 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.webapp; + +import static org.apache.hadoop.yarn.webapp.view.JQueryUI.DATATABLES_ID; + +import org.apache.hadoop.yarn.nodelabels.NodeLabel; +import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager; +import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager; +import org.apache.hadoop.yarn.webapp.SubView; +import org.apache.hadoop.yarn.webapp.YarnWebParams; +import org.apache.hadoop.yarn.webapp.hamlet.Hamlet; +import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.TABLE; +import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.TBODY; +import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.TR; +import org.apache.hadoop.yarn.webapp.view.HtmlBlock; + +import com.google.inject.Inject; + +public class NodeLabelsPage extends RmView { + static class NodeLabelsBlock extends HtmlBlock { + final ResourceManager rm; + + @Inject + NodeLabelsBlock(ResourceManager rm, ViewContext ctx) { + super(ctx); + this.rm = rm; + } + + @Override + protected void render(Block html) { + TBODY> tbody = html.table("#nodelabels"). + thead(). + tr(). + th(".name", "Label Name"). + th(".numOfActiveNMs", "Num Of Active NMs"). + th(".totalResource", "Total Resource"). + _()._(). + tbody(); + + RMNodeLabelsManager nlm = rm.getRMContext().getNodeLabelManager(); + for (NodeLabel info : nlm.pullRMNodeLabelsInfo()) { + TR>> row = + tbody.tr().td( + info.getLabelName().isEmpty() ? "" : info + .getLabelName()); + int nActiveNMs = info.getNumActiveNMs(); + if (nActiveNMs > 0) { + row = row.td() + .a(url("nodes", + "?" + YarnWebParams.NODE_LABEL + "=" + info.getLabelName()), + String.valueOf(nActiveNMs)) + ._(); + } else { + row = row.td(String.valueOf(nActiveNMs)); + } + row.td(info.getResource().toString())._(); + } + tbody._()._(); + } + } + + @Override protected void preHead(Page.HTML<_> html) { + commonPreHead(html); + String title = "Node labels of the cluster"; + setTitle(title); + set(DATATABLES_ID, "nodelabels"); + setTableStyles(html, "nodelabels", ".healthStatus {width:10em}", + ".healthReport {width:10em}"); + } + + @Override protected Class content() { + return NodeLabelsBlock.class; + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/b7442bf9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/NodesPage.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/NodesPage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/NodesPage.java index d3849ae..f28a9a8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/NodesPage.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/NodesPage.java @@ -1,24 +1,25 @@ /** -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.hadoop.yarn.server.resourcemanager.webapp; -import static org.apache.hadoop.yarn.server.resourcemanager.webapp.RMWebApp.NODE_STATE; +import static org.apache.hadoop.yarn.webapp.YarnWebParams.NODE_STATE; +import static org.apache.hadoop.yarn.webapp.YarnWebParams.NODE_LABEL; import static org.apache.hadoop.yarn.webapp.view.JQueryUI.DATATABLES; import static org.apache.hadoop.yarn.webapp.view.JQueryUI.DATATABLES_ID; import static org.apache.hadoop.yarn.webapp.view.JQueryUI.initID; @@ -28,7 +29,9 @@ import java.util.Collection; import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.yarn.api.records.NodeState; +import org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager; import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager; +import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager; import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NodeInfo; @@ -60,26 +63,20 @@ class NodesPage extends RmView { ResourceScheduler sched = rm.getResourceScheduler(); String type = $(NODE_STATE); - TBODY> tbody = html.table("#nodes"). - thead(). - tr(). - th(".nodelabels", "Node Labels"). - th(".rack", "Rack"). - th(".state", "Node State"). - th(".nodeaddress", "Node Address"). - th(".nodehttpaddress", "Node HTTP Address"). - th(".lastHealthUpdate", "Last health-update"). - th(".healthReport", "Health-report"). - th(".containers", "Containers"). - th(".mem", "Mem Used"). - th(".mem", "Mem Avail"). - th(".vcores", "VCores Used"). - th(".vcores", "VCores Avail"). - th(".nodeManagerVersion", "Version"). - _()._(). - tbody(); + String labelFilter = $(NODE_LABEL, CommonNodeLabelsManager.ANY).trim(); + TBODY> tbody = + html.table("#nodes").thead().tr().th(".nodelabels", "Node Labels") + .th(".rack", "Rack").th(".state", "Node State") + .th(".nodeaddress", "Node Address") + .th(".nodehttpaddress", "Node HTTP Address") + .th(".lastHealthUpdate", "Last health-update") + .th(".healthReport", "Health-report") + .th(".containers", "Containers").th(".mem", "Mem Used") + .th(".mem", "Mem Avail").th(".vcores", "VCores Used") + .th(".vcores", "VCores Avail") + .th(".nodeManagerVersion", "Version")._()._().tbody(); NodeState stateFilter = null; - if(type != null && !type.isEmpty()) { + if (type != null && !type.isEmpty()) { stateFilter = NodeState.valueOf(type.toUpperCase()); } Collection rmNodes = this.rm.getRMContext().getRMNodes().values(); @@ -97,9 +94,9 @@ class NodesPage extends RmView { } } for (RMNode ni : rmNodes) { - if(stateFilter != null) { + if (stateFilter != null) { NodeState state = ni.getState(); - if(!stateFilter.equals(state)) { + if (!stateFilter.equals(state)) { continue; } } else { @@ -109,61 +106,71 @@ class NodesPage extends RmView { continue; } } + // Besides state, we need to filter label as well. + if (!labelFilter.equals(RMNodeLabelsManager.ANY)) { + if (labelFilter.isEmpty()) { + // Empty label filter means only shows nodes without label + if (!ni.getNodeLabels().isEmpty()) { + continue; + } + } else if (!ni.getNodeLabels().contains(labelFilter)) { + // Only nodes have given label can show on web page. + continue; + } + } NodeInfo info = new NodeInfo(ni, sched); - int usedMemory = (int)info.getUsedMemory(); - int availableMemory = (int)info.getAvailableMemory(); - TR>> row = tbody.tr(). - td(StringUtils.join(",", info.getNodeLabels())). - td(info.getRack()). - td(info.getState()). - td(info.getNodeId()); + int usedMemory = (int) info.getUsedMemory(); + int availableMemory = (int) info.getAvailableMemory(); + TR>> row = + tbody.tr().td(StringUtils.join(",", info.getNodeLabels())) + .td(info.getRack()).td(info.getState()).td(info.getNodeId()); if (isInactive) { row.td()._("N/A")._(); } else { String httpAddress = info.getNodeHTTPAddress(); - row.td().a("//" + httpAddress, - httpAddress)._(); + row.td().a("//" + httpAddress, httpAddress)._(); } - row.td().br().$title(String.valueOf(info.getLastHealthUpdate()))._(). - _(Times.format(info.getLastHealthUpdate()))._(). - td(info.getHealthReport()). - td(String.valueOf(info.getNumContainers())). - td().br().$title(String.valueOf(usedMemory))._(). - _(StringUtils.byteDesc(usedMemory * BYTES_IN_MB))._(). - td().br().$title(String.valueOf(availableMemory))._(). - _(StringUtils.byteDesc(availableMemory * BYTES_IN_MB))._(). - td(String.valueOf(info.getUsedVirtualCores())). - td(String.valueOf(info.getAvailableVirtualCores())). - td(ni.getNodeManagerVersion()). - _(); + row.td().br().$title(String.valueOf(info.getLastHealthUpdate()))._() + ._(Times.format(info.getLastHealthUpdate()))._() + .td(info.getHealthReport()) + .td(String.valueOf(info.getNumContainers())).td().br() + .$title(String.valueOf(usedMemory))._() + ._(StringUtils.byteDesc(usedMemory * BYTES_IN_MB))._().td().br() + .$title(String.valueOf(availableMemory))._() + ._(StringUtils.byteDesc(availableMemory * BYTES_IN_MB))._() + .td(String.valueOf(info.getUsedVirtualCores())) + .td(String.valueOf(info.getAvailableVirtualCores())) + .td(ni.getNodeManagerVersion())._(); } tbody._()._(); } } - @Override protected void preHead(Page.HTML<_> html) { + @Override + protected void preHead(Page.HTML<_> html) { commonPreHead(html); String type = $(NODE_STATE); String title = "Nodes of the cluster"; - if(type != null && !type.isEmpty()) { - title = title+" ("+type+")"; + if (type != null && !type.isEmpty()) { + title = title + " (" + type + ")"; } setTitle(title); set(DATATABLES_ID, "nodes"); set(initID(DATATABLES, "nodes"), nodesTableInit()); setTableStyles(html, "nodes", ".healthStatus {width:10em}", - ".healthReport {width:10em}"); + ".healthReport {width:10em}"); } - @Override protected Class content() { + @Override + protected Class content() { return NodesBlock.class; } private String nodesTableInit() { StringBuilder b = tableInit().append(", aoColumnDefs: ["); b.append("{'bSearchable': false, 'aTargets': [ 6 ]}"); - b.append(", {'sType': 'title-numeric', 'bSearchable': false, " + - "'aTargets': [ 7, 8 ] }"); + b.append(", {'sType': 'title-numeric', 'bSearchable': false, " + + "'aTargets': [ 7, 8 ] }"); b.append(", {'sType': 'title-numeric', 'aTargets': [ 4 ]}"); b.append("]}"); return b.toString(); http://git-wip-us.apache.org/repos/asf/hadoop/blob/b7442bf9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebApp.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebApp.java index 67c73b8..c0e6834 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebApp.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebApp.java @@ -61,6 +61,7 @@ public class RMWebApp extends WebApp implements YarnWebParams { route(pajoin("/app", APPLICATION_ID), RmController.class, "app"); route("/scheduler", RmController.class, "scheduler"); route(pajoin("/queue", QUEUE_NAME), RmController.class, "queue"); + route("/nodelabels", RmController.class, "nodelabels"); } @Override http://git-wip-us.apache.org/repos/asf/hadoop/blob/b7442bf9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RmController.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RmController.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RmController.java index f186bf4..972432b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RmController.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RmController.java @@ -28,7 +28,6 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.Capacity import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler; import org.apache.hadoop.yarn.util.StringHelper; import org.apache.hadoop.yarn.webapp.Controller; -import org.apache.hadoop.yarn.webapp.WebAppException; import org.apache.hadoop.yarn.webapp.YarnWebParams; import com.google.inject.Inject; @@ -93,4 +92,9 @@ public class RmController extends Controller { public void submit() { setTitle("Application Submission Not Allowed"); } + + public void nodelabels() { + setTitle("Node Labels"); + render(NodeLabelsPage.class); + } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/b7442bf9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNodes.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNodes.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNodes.java index 278c151..2d863d1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNodes.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNodes.java @@ -30,11 +30,13 @@ import org.apache.hadoop.yarn.api.records.NodeState; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.factories.RecordFactory; import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; +import org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager; import org.apache.hadoop.yarn.server.api.protocolrecords.NodeHeartbeatResponse; import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager; import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode; import org.apache.hadoop.yarn.server.resourcemanager.rmnode.UpdatedContainerInfo; +import com.google.common.collect.ImmutableSet; import com.google.common.collect.Lists; /** @@ -53,7 +55,12 @@ public class MockNodes { // One unhealthy node per rack. list.add(nodeInfo(i, perNode, NodeState.UNHEALTHY)); } - list.add(newNodeInfo(i, perNode)); + if (j == 0) { + // One node with label + list.add(nodeInfo(i, perNode, NodeState.RUNNING, ImmutableSet.of("x"))); + } else { + list.add(newNodeInfo(i, perNode)); + } } } return list; @@ -100,10 +107,12 @@ public class MockNodes { private String healthReport; private long lastHealthReportTime; private NodeState state; + private Set labels; public MockRMNodeImpl(NodeId nodeId, String nodeAddr, String httpAddress, Resource perNode, String rackName, String healthReport, - long lastHealthReportTime, int cmdPort, String hostName, NodeState state) { + long lastHealthReportTime, int cmdPort, String hostName, NodeState state, + Set labels) { this.nodeId = nodeId; this.nodeAddr = nodeAddr; this.httpAddress = httpAddress; @@ -114,6 +123,7 @@ public class MockNodes { this.cmdPort = cmdPort; this.hostName = hostName; this.state = state; + this.labels = labels; } @Override @@ -207,16 +217,33 @@ public class MockNodes { @Override public Set getNodeLabels() { - return RMNodeLabelsManager.EMPTY_STRING_SET; + if (labels != null) { + return labels; + } + return CommonNodeLabelsManager.EMPTY_STRING_SET; } }; - private static RMNode buildRMNode(int rack, final Resource perNode, NodeState state, String httpAddr) { - return buildRMNode(rack, perNode, state, httpAddr, NODE_ID++, null, 123); + private static RMNode buildRMNode(int rack, final Resource perNode, + NodeState state, String httpAddr) { + return buildRMNode(rack, perNode, state, httpAddr, null); } - + + private static RMNode buildRMNode(int rack, final Resource perNode, + NodeState state, String httpAddr, Set labels) { + return buildRMNode(rack, perNode, state, httpAddr, NODE_ID++, null, 123, + labels); + } + private static RMNode buildRMNode(int rack, final Resource perNode, NodeState state, String httpAddr, int hostnum, String hostName, int port) { + return buildRMNode(rack, perNode, state, httpAddr, hostnum, hostName, port, + null); + } + + private static RMNode buildRMNode(int rack, final Resource perNode, + NodeState state, String httpAddr, int hostnum, String hostName, int port, + Set labels) { final String rackName = "rack"+ rack; final int nid = hostnum; final String nodeAddr = hostName + ":" + nid; @@ -228,13 +255,18 @@ public class MockNodes { final String httpAddress = httpAddr; String healthReport = (state == NodeState.UNHEALTHY) ? null : "HealthyMe"; return new MockRMNodeImpl(nodeID, nodeAddr, httpAddress, perNode, - rackName, healthReport, 0, nid, hostName, state); + rackName, healthReport, 0, nid, hostName, state, labels); } public static RMNode nodeInfo(int rack, final Resource perNode, NodeState state) { return buildRMNode(rack, perNode, state, "N/A"); } + + public static RMNode nodeInfo(int rack, final Resource perNode, + NodeState state, Set labels) { + return buildRMNode(rack, perNode, state, "N/A", labels); + } public static RMNode newNodeInfo(int rack, final Resource perNode) { return buildRMNode(rack, perNode, NodeState.RUNNING, "localhost:0"); http://git-wip-us.apache.org/repos/asf/hadoop/blob/b7442bf9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingRMRestart.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingRMRestart.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingRMRestart.java index 842eaec..e21fcf9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingRMRestart.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingRMRestart.java @@ -839,7 +839,7 @@ public class TestWorkPreservingRMRestart { // Test if RM on recovery receives the container release request from AM // before it receives the container status reported by NM for recovery. this // container should not be recovered. - @Test (timeout = 30000) + @Test (timeout = 50000) public void testReleasedContainerNotRecovered() throws Exception { MemoryRMStateStore memStore = new MemoryRMStateStore(); memStore.init(conf); http://git-wip-us.apache.org/repos/asf/hadoop/blob/b7442bf9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/TestRMNodeLabelsManager.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/TestRMNodeLabelsManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/TestRMNodeLabelsManager.java index ed675f3..e4cdc71 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/TestRMNodeLabelsManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/TestRMNodeLabelsManager.java @@ -20,6 +20,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.nodelabels; import java.io.IOException; import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.Set; @@ -27,6 +28,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager; +import org.apache.hadoop.yarn.nodelabels.NodeLabel; import org.apache.hadoop.yarn.nodelabels.NodeLabelTestBase; import org.apache.hadoop.yarn.util.resource.Resources; import org.junit.After; @@ -428,4 +430,35 @@ public class TestRMNodeLabelsManager extends NodeLabelTestBase { Assert.fail("IOException from removeLabelsFromNode " + e); } } + + private void checkNodeLabelInfo(List infos, String labelName, int activeNMs, int memory) { + for (NodeLabel info : infos) { + if (info.getLabelName().equals(labelName)) { + Assert.assertEquals(activeNMs, info.getNumActiveNMs()); + Assert.assertEquals(memory, info.getResource().getMemory()); + return; + } + } + Assert.fail("Failed to find info has label=" + labelName); + } + + @Test(timeout = 5000) + public void testPullRMNodeLabelsInfo() throws IOException { + mgr.addToCluserNodeLabels(toSet("x", "y", "z")); + mgr.activateNode(NodeId.newInstance("n1", 1), Resource.newInstance(10, 0)); + mgr.activateNode(NodeId.newInstance("n2", 1), Resource.newInstance(10, 0)); + mgr.activateNode(NodeId.newInstance("n3", 1), Resource.newInstance(10, 0)); + mgr.activateNode(NodeId.newInstance("n4", 1), Resource.newInstance(10, 0)); + mgr.activateNode(NodeId.newInstance("n5", 1), Resource.newInstance(10, 0)); + mgr.replaceLabelsOnNode(ImmutableMap.of(toNodeId("n1"), toSet("x"), + toNodeId("n2"), toSet("x"), toNodeId("n3"), toSet("y"))); + + // x, y, z and "" + List infos = mgr.pullRMNodeLabelsInfo(); + Assert.assertEquals(4, infos.size()); + checkNodeLabelInfo(infos, RMNodeLabelsManager.NO_LABEL, 2, 20); + checkNodeLabelInfo(infos, "x", 2, 20); + checkNodeLabelInfo(infos, "y", 1, 10); + checkNodeLabelInfo(infos, "z", 0, 0); + } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/b7442bf9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestNodesPage.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestNodesPage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestNodesPage.java index bb38079..62713cf 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestNodesPage.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestNodesPage.java @@ -106,4 +106,49 @@ public class TestNodesPage { * numberOfActualTableHeaders + numberOfThInMetricsTable)).print( "