Return-Path: X-Original-To: apmail-hadoop-common-commits-archive@www.apache.org Delivered-To: apmail-hadoop-common-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id BA7A0178AA for ; Wed, 13 May 2015 20:29:27 +0000 (UTC) Received: (qmail 811 invoked by uid 500); 13 May 2015 20:29:27 -0000 Delivered-To: apmail-hadoop-common-commits-archive@hadoop.apache.org Received: (qmail 747 invoked by uid 500); 13 May 2015 20:29:27 -0000 Mailing-List: contact common-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: common-dev@hadoop.apache.org Delivered-To: mailing list common-commits@hadoop.apache.org Received: (qmail 738 invoked by uid 99); 13 May 2015 20:29:27 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 13 May 2015 20:29:27 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 611CEE32B9; Wed, 13 May 2015 20:29:27 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: wangda@apache.org To: common-commits@hadoop.apache.org Message-Id: X-Mailer: ASF-Git Admin Mailer Subject: hadoop git commit: YARN-3579. CommonNodeLabelsManager should support NodeLabel instead of string label name when getting node-to-label/label-to-label mappings. (Sunil G via wangda) Date: Wed, 13 May 2015 20:29:27 +0000 (UTC) Repository: hadoop Updated Branches: refs/heads/trunk 0f9592144 -> d4f53fc96 YARN-3579. CommonNodeLabelsManager should support NodeLabel instead of string label name when getting node-to-label/label-to-label mappings. (Sunil G via wangda) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d4f53fc9 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d4f53fc9 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d4f53fc9 Branch: refs/heads/trunk Commit: d4f53fc9631d682cd79ba440aefa6750dcc898be Parents: 0f95921 Author: Wangda Tan Authored: Wed May 13 13:29:09 2015 -0700 Committer: Wangda Tan Committed: Wed May 13 13:29:09 2015 -0700 ---------------------------------------------------------------------- hadoop-yarn-project/CHANGES.txt | 3 + .../nodelabels/CommonNodeLabelsManager.java | 149 ++++++++++++++++--- .../hadoop/yarn/nodelabels/RMNodeLabel.java | 7 +- .../yarn/nodelabels/NodeLabelTestBase.java | 30 ++++ .../nodelabels/TestCommonNodeLabelsManager.java | 22 +++ 5 files changed, 186 insertions(+), 25 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/d4f53fc9/hadoop-yarn-project/CHANGES.txt ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index e5d5ecf..16c2dd9 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -420,6 +420,9 @@ Release 2.7.1 - UNRELEASED YARN-3539. Updated timeline server documentation and marked REST APIs evolving. (Steve Loughran via zjshen) + YARN-3579. CommonNodeLabelsManager should support NodeLabel instead of string + label name when getting node-to-label/label-to-label mappings. (Sunil G via wangda) + OPTIMIZATIONS BUG FIXES http://git-wip-us.apache.org/repos/asf/hadoop/blob/d4f53fc9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/CommonNodeLabelsManager.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/CommonNodeLabelsManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/CommonNodeLabelsManager.java index f2ff0f6..bf34837 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/CommonNodeLabelsManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/CommonNodeLabelsManager.java @@ -64,6 +64,8 @@ public class CommonNodeLabelsManager extends AbstractService { private static final int MAX_LABEL_LENGTH = 255; public static final Set EMPTY_STRING_SET = Collections .unmodifiableSet(new HashSet(0)); + public static final Set EMPTY_NODELABEL_SET = Collections + .unmodifiableSet(new HashSet(0)); public static final String ANY = "*"; public static final Set ACCESS_ANY_LABEL_SET = ImmutableSet.of(ANY); private static final Pattern LABEL_PATTERN = Pattern @@ -716,23 +718,53 @@ public class CommonNodeLabelsManager extends AbstractService { * @return nodes to labels map */ public Map> getNodeLabels() { + Map> nodeToLabels = + generateNodeLabelsInfoPerNode(String.class); + return nodeToLabels; + } + + /** + * Get mapping of nodes to label info + * + * @return nodes to labels map + */ + public Map> getNodeLabelsInfo() { + Map> nodeToLabels = + generateNodeLabelsInfoPerNode(NodeLabel.class); + return nodeToLabels; + } + + @SuppressWarnings("unchecked") + private Map> generateNodeLabelsInfoPerNode(Class type) { try { readLock.lock(); - Map> nodeToLabels = - new HashMap>(); + Map> nodeToLabels = new HashMap<>(); for (Entry entry : nodeCollections.entrySet()) { String hostName = entry.getKey(); Host host = entry.getValue(); for (NodeId nodeId : host.nms.keySet()) { - Set nodeLabels = getLabelsByNode(nodeId); - if (nodeLabels == null || nodeLabels.isEmpty()) { - continue; + if (type.isAssignableFrom(String.class)) { + Set nodeLabels = getLabelsByNode(nodeId); + if (nodeLabels == null || nodeLabels.isEmpty()) { + continue; + } + nodeToLabels.put(nodeId, (Set) nodeLabels); + } else { + Set nodeLabels = getLabelsInfoByNode(nodeId); + if (nodeLabels == null || nodeLabels.isEmpty()) { + continue; + } + nodeToLabels.put(nodeId, (Set) nodeLabels); } - nodeToLabels.put(nodeId, nodeLabels); } if (!host.labels.isEmpty()) { - nodeToLabels - .put(NodeId.newInstance(hostName, WILDCARD_PORT), host.labels); + if (type.isAssignableFrom(String.class)) { + nodeToLabels.put(NodeId.newInstance(hostName, WILDCARD_PORT), + (Set) host.labels); + } else { + nodeToLabels.put(NodeId.newInstance(hostName, WILDCARD_PORT), + (Set) createNodeLabelFromLabelNames(host.labels)); + } } } return Collections.unmodifiableMap(nodeToLabels); @@ -741,6 +773,7 @@ public class CommonNodeLabelsManager extends AbstractService { } } + /** * Get mapping of labels to nodes for all the labels. * @@ -765,28 +798,72 @@ public class CommonNodeLabelsManager extends AbstractService { public Map> getLabelsToNodes(Set labels) { try { readLock.lock(); - Map> labelsToNodes = - new HashMap>(); - for (String label : labels) { - if(label.equals(NO_LABEL)) { - continue; - } - RMNodeLabel nodeLabelInfo = labelCollections.get(label); - if(nodeLabelInfo != null) { - Set nodeIds = nodeLabelInfo.getAssociatedNodeIds(); - if (!nodeIds.isEmpty()) { - labelsToNodes.put(label, nodeIds); - } - } else { - LOG.warn("getLabelsToNodes : Label [" + label + "] cannot be found"); - } - } + Map> labelsToNodes = getLabelsToNodesMapping(labels, + String.class); + return Collections.unmodifiableMap(labelsToNodes); + } finally { + readLock.unlock(); + } + } + + + /** + * Get mapping of labels to nodes for all the labels. + * + * @return labels to nodes map + */ + public Map> getLabelsInfoToNodes() { + try { + readLock.lock(); + return getLabelsInfoToNodes(labelCollections.keySet()); + } finally { + readLock.unlock(); + } + } + + /** + * Get mapping of labels info to nodes for specified set of labels. + * + * @param nodelabels + * set of nodelabels for which labels to nodes mapping will be + * returned. + * @return labels to nodes map + */ + public Map> getLabelsInfoToNodes(Set labels) { + try { + readLock.lock(); + Map> labelsToNodes = getLabelsToNodesMapping( + labels, NodeLabel.class); return Collections.unmodifiableMap(labelsToNodes); } finally { readLock.unlock(); } } + private Map> getLabelsToNodesMapping(Set labels, + Class type) { + Map> labelsToNodes = new HashMap>(); + for (String label : labels) { + if (label.equals(NO_LABEL)) { + continue; + } + RMNodeLabel nodeLabelInfo = labelCollections.get(label); + if (nodeLabelInfo != null) { + Set nodeIds = nodeLabelInfo.getAssociatedNodeIds(); + if (!nodeIds.isEmpty()) { + if (type.isAssignableFrom(String.class)) { + labelsToNodes.put(type.cast(label), nodeIds); + } else { + labelsToNodes.put(type.cast(nodeLabelInfo.getNodeLabel()), nodeIds); + } + } + } else { + LOG.warn("getLabelsToNodes : Label [" + label + "] cannot be found"); + } + } + return labelsToNodes; + } + /** * Get existing valid labels in repository * @@ -914,6 +991,30 @@ public class CommonNodeLabelsManager extends AbstractService { } } + private Set getLabelsInfoByNode(NodeId nodeId) { + Set labels = getLabelsByNode(nodeId, nodeCollections); + if (labels.isEmpty()) { + return EMPTY_NODELABEL_SET; + } + Set nodeLabels = createNodeLabelFromLabelNames(labels); + return nodeLabels; + } + + private Set createNodeLabelFromLabelNames(Set labels) { + Set nodeLabels = new HashSet(); + for (String label : labels) { + if (label.equals(NO_LABEL)) { + continue; + } + RMNodeLabel rmLabel = labelCollections.get(label); + if (rmLabel == null) { + continue; + } + nodeLabels.add(rmLabel.getNodeLabel()); + } + return nodeLabels; + } + protected void createNodeIfNonExisted(NodeId nodeId) throws IOException { Host host = nodeCollections.get(nodeId.getHost()); if (null == host) { http://git-wip-us.apache.org/repos/asf/hadoop/blob/d4f53fc9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/RMNodeLabel.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/RMNodeLabel.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/RMNodeLabel.java index 1c4fd17..feeeaf1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/RMNodeLabel.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/RMNodeLabel.java @@ -25,7 +25,6 @@ import org.apache.commons.lang.StringUtils; import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.api.records.NodeLabel; import org.apache.hadoop.yarn.api.records.Resource; -import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.util.resource.Resources; public class RMNodeLabel implements Comparable { @@ -34,6 +33,7 @@ public class RMNodeLabel implements Comparable { private String labelName; private Set nodeIds; private boolean exclusive; + private NodeLabel nodeLabel; public RMNodeLabel(NodeLabel nodeLabel) { this(nodeLabel.getName(), Resource.newInstance(0, 0), 0, @@ -52,6 +52,7 @@ public class RMNodeLabel implements Comparable { this.numActiveNMs = activeNMs; this.nodeIds = new HashSet(); this.exclusive = exclusive; + this.nodeLabel = NodeLabel.newInstance(labelName, exclusive); } public void addNodeId(NodeId node) { @@ -100,6 +101,10 @@ public class RMNodeLabel implements Comparable { return new RMNodeLabel(labelName, resource, numActiveNMs, exclusive); } + public NodeLabel getNodeLabel() { + return this.nodeLabel; + } + @Override public int compareTo(RMNodeLabel o) { // We should always put empty label entry first after sorting http://git-wip-us.apache.org/repos/asf/hadoop/blob/d4f53fc9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/nodelabels/NodeLabelTestBase.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/nodelabels/NodeLabelTestBase.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/nodelabels/NodeLabelTestBase.java index 4d406fd..8301d96 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/nodelabels/NodeLabelTestBase.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/nodelabels/NodeLabelTestBase.java @@ -26,6 +26,7 @@ import java.util.Set; import java.util.Map.Entry; import org.apache.hadoop.yarn.api.records.NodeId; +import org.apache.hadoop.yarn.api.records.NodeLabel; import org.junit.Assert; import com.google.common.collect.ImmutableMap; @@ -41,6 +42,15 @@ public class NodeLabelTestBase { } } + public static void assertLabelInfoMapEquals(Map> m1, + ImmutableMap> m2) { + Assert.assertEquals(m1.size(), m2.size()); + for (NodeId k : m1.keySet()) { + Assert.assertTrue(m2.containsKey(k)); + assertNLCollectionEquals(m1.get(k), m2.get(k)); + } + } + public static void assertLabelsToNodesEquals(Map> m1, ImmutableMap> m2) { Assert.assertEquals(m1.size(), m2.size()); @@ -88,6 +98,14 @@ public class NodeLabelTestBase { Assert.assertTrue(s1.containsAll(s2)); } + public static void assertNLCollectionEquals(Collection c1, + Collection c2) { + Set s1 = new HashSet(c1); + Set s2 = new HashSet(c2); + Assert.assertEquals(s1, s2); + Assert.assertTrue(s1.containsAll(s2)); + } + @SuppressWarnings("unchecked") public static Set toSet(E... elements) { Set set = Sets.newHashSet(elements); @@ -105,4 +123,16 @@ public class NodeLabelTestBase { return NodeId.newInstance(str, CommonNodeLabelsManager.WILDCARD_PORT); } } + + public static void assertLabelsInfoToNodesEquals( + Map> m1, ImmutableMap> m2) { + Assert.assertEquals(m1.size(), m2.size()); + for (NodeLabel k : m1.keySet()) { + Assert.assertTrue(m2.containsKey(k)); + Set s1 = new HashSet(m1.get(k)); + Set s2 = new HashSet(m2.get(k)); + Assert.assertEquals(s1, s2); + Assert.assertTrue(s1.containsAll(s2)); + } + } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/d4f53fc9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/nodelabels/TestCommonNodeLabelsManager.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/nodelabels/TestCommonNodeLabelsManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/nodelabels/TestCommonNodeLabelsManager.java index 09838b4..c25b0bf 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/nodelabels/TestCommonNodeLabelsManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/nodelabels/TestCommonNodeLabelsManager.java @@ -579,4 +579,26 @@ public class TestCommonNodeLabelsManager extends NodeLabelTestBase { labelsByNode); Assert.assertTrue(labelsByNode.contains("p1")); } + + @Test(timeout = 5000) + public void testLabelsInfoToNodes() throws IOException { + mgr.addToCluserNodeLabels(Arrays.asList(NodeLabel.newInstance("p1", false), + NodeLabel.newInstance("p2", true), NodeLabel.newInstance("p3", true))); + mgr.addLabelsToNode(ImmutableMap.of(toNodeId("n1"), toSet("p1"))); + Map> labelsToNodes = mgr.getLabelsInfoToNodes(); + assertLabelsInfoToNodesEquals(labelsToNodes, ImmutableMap.of( + NodeLabel.newInstance("p1", false), toSet(toNodeId("n1")))); + } + + @Test(timeout = 5000) + public void testGetNodeLabelsInfo() throws IOException { + mgr.addToCluserNodeLabels(Arrays.asList(NodeLabel.newInstance("p1", false), + NodeLabel.newInstance("p2", true), NodeLabel.newInstance("p3", false))); + mgr.addLabelsToNode(ImmutableMap.of(toNodeId("n1"), toSet("p2"))); + mgr.addLabelsToNode(ImmutableMap.of(toNodeId("n2"), toSet("p3"))); + + assertLabelInfoMapEquals(mgr.getNodeLabelsInfo(), ImmutableMap.of( + toNodeId("n1"), toSet(NodeLabel.newInstance("p2", true)), + toNodeId("n2"), toSet(NodeLabel.newInstance("p3", false)))); + } }