Return-Path: X-Original-To: apmail-hadoop-common-commits-archive@www.apache.org Delivered-To: apmail-hadoop-common-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id DA09417621 for ; Mon, 12 Jan 2015 18:18:57 +0000 (UTC) Received: (qmail 24701 invoked by uid 500); 12 Jan 2015 18:18:45 -0000 Delivered-To: apmail-hadoop-common-commits-archive@hadoop.apache.org Received: (qmail 24179 invoked by uid 500); 12 Jan 2015 18:18:44 -0000 Mailing-List: contact common-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: common-dev@hadoop.apache.org Delivered-To: mailing list common-commits@hadoop.apache.org Received: (qmail 22911 invoked by uid 99); 12 Jan 2015 18:18:44 -0000 Received: from tyr.zones.apache.org (HELO tyr.zones.apache.org) (140.211.11.114) by apache.org (qpsmtpd/0.29) with ESMTP; Mon, 12 Jan 2015 18:18:44 +0000 Received: by tyr.zones.apache.org (Postfix, from userid 65534) id 20C31A01B49; Mon, 12 Jan 2015 18:18:44 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: zhz@apache.org To: common-commits@hadoop.apache.org Date: Mon, 12 Jan 2015 18:19:13 -0000 Message-Id: In-Reply-To: <80fa67df918545d79e507e849c37bdb8@git.apache.org> References: <80fa67df918545d79e507e849c37bdb8@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [31/34] hadoop git commit: YARN-3014. Replaces labels on a host should update all NM's labels on that host. Contributed by Wangda Tan YARN-3014. Replaces labels on a host should update all NM's labels on that host. Contributed by Wangda Tan Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d77a3971 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d77a3971 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d77a3971 Branch: refs/heads/HDFS-EC Commit: d77a397177c2a11ce9a7d3b93f9553398940d310 Parents: ce3d0c1 Author: Jian He Authored: Fri Jan 9 17:49:53 2015 -0800 Committer: Zhe Zhang Committed: Mon Jan 12 10:18:03 2015 -0800 ---------------------------------------------------------------------- hadoop-yarn-project/CHANGES.txt | 3 + .../nodelabels/CommonNodeLabelsManager.java | 163 ++++++++----------- .../nodelabels/TestCommonNodeLabelsManager.java | 38 +++++ .../nodelabels/RMNodeLabelsManager.java | 1 - 4 files changed, 113 insertions(+), 92 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/d77a3971/hadoop-yarn-project/CHANGES.txt ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 2cc3049..99c8514 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -344,6 +344,9 @@ Release 2.7.0 - UNRELEASED YARN-2997. Fixed NodeStatusUpdater to not send alreay-sent completed container statuses on heartbeat. (Chengbing Liu via jianhe) + YARN-3014. Replaces labels on a host should update all NM's labels on that + host. (Wangda Tan via jianhe) + Release 2.6.0 - 2014-11-18 INCOMPATIBLE CHANGES http://git-wip-us.apache.org/repos/asf/hadoop/blob/d77a3971/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/CommonNodeLabelsManager.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/CommonNodeLabelsManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/CommonNodeLabelsManager.java index e888cc5..aeefff1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/CommonNodeLabelsManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/CommonNodeLabelsManager.java @@ -21,6 +21,7 @@ package org.apache.hadoop.yarn.nodelabels; import java.io.IOException; import java.util.Collection; import java.util.Collections; +import java.util.EnumSet; import java.util.HashMap; import java.util.HashSet; import java.util.Map; @@ -130,6 +131,12 @@ public class CommonNodeLabelsManager extends AbstractService { return c; } } + + private enum NodeLabelUpdateOperation { + ADD, + REMOVE, + REPLACE + } private final class ForwardingEventHandler implements EventHandler { @@ -290,45 +297,6 @@ public class CommonNodeLabelsManager extends AbstractService { } } - @SuppressWarnings("unchecked") - protected void internalAddLabelsToNode( - Map> addedLabelsToNode) throws IOException { - // do add labels to nodes - Map> newNMToLabels = - new HashMap>(); - for (Entry> entry : addedLabelsToNode.entrySet()) { - NodeId nodeId = entry.getKey(); - Set labels = entry.getValue(); - - createHostIfNonExisted(nodeId.getHost()); - if (nodeId.getPort() == WILDCARD_PORT) { - Host host = nodeCollections.get(nodeId.getHost()); - host.labels.addAll(labels); - newNMToLabels.put(nodeId, host.labels); - } else { - createNodeIfNonExisted(nodeId); - Node nm = getNMInNodeSet(nodeId); - if (nm.labels == null) { - nm.labels = new HashSet(); - } - nm.labels.addAll(labels); - newNMToLabels.put(nodeId, nm.labels); - } - } - - if (null != dispatcher) { - dispatcher.getEventHandler().handle( - new UpdateNodeToLabelsMappingsEvent(newNMToLabels)); - } - - // shows node->labels we added - LOG.info("addLabelsToNode:"); - for (Entry> entry : newNMToLabels.entrySet()) { - LOG.info(" NM=" + entry.getKey() + ", labels=[" - + StringUtils.join(entry.getValue().iterator(), ",") + "]"); - } - } - /** * add more labels to nodes * @@ -338,7 +306,7 @@ public class CommonNodeLabelsManager extends AbstractService { throws IOException { addedLabelsToNode = normalizeNodeIdToLabels(addedLabelsToNode); checkAddLabelsToNode(addedLabelsToNode); - internalAddLabelsToNode(addedLabelsToNode); + internalUpdateLabelsOnNodes(addedLabelsToNode, NodeLabelUpdateOperation.ADD); } protected void checkRemoveFromClusterNodeLabels( @@ -469,24 +437,75 @@ public class CommonNodeLabelsManager extends AbstractService { } @SuppressWarnings("unchecked") - protected void internalRemoveLabelsFromNode( - Map> removeLabelsFromNode) { - // do remove labels from nodes + protected void internalUpdateLabelsOnNodes( + Map> nodeToLabels, NodeLabelUpdateOperation op) + throws IOException { + // do update labels from nodes Map> newNMToLabels = new HashMap>(); - for (Entry> entry : removeLabelsFromNode.entrySet()) { + for (Entry> entry : nodeToLabels.entrySet()) { NodeId nodeId = entry.getKey(); Set labels = entry.getValue(); + createHostIfNonExisted(nodeId.getHost()); if (nodeId.getPort() == WILDCARD_PORT) { Host host = nodeCollections.get(nodeId.getHost()); - host.labels.removeAll(labels); + switch (op) { + case REMOVE: + host.labels.removeAll(labels); + for (Node node : host.nms.values()) { + if (node.labels != null) { + node.labels.removeAll(labels); + } + } + break; + case ADD: + host.labels.addAll(labels); + for (Node node : host.nms.values()) { + if (node.labels != null) { + node.labels.addAll(labels); + } + } + break; + case REPLACE: + host.labels.clear(); + host.labels.addAll(labels); + for (Node node : host.nms.values()) { + node.labels = null; + } + break; + default: + break; + } newNMToLabels.put(nodeId, host.labels); } else { - Node nm = getNMInNodeSet(nodeId); - if (nm.labels != null) { - nm.labels.removeAll(labels); + if (EnumSet.of(NodeLabelUpdateOperation.ADD, + NodeLabelUpdateOperation.REPLACE).contains(op)) { + // Add and replace + createNodeIfNonExisted(nodeId); + Node nm = getNMInNodeSet(nodeId); + if (nm.labels == null) { + nm.labels = new HashSet(); + } + switch (op) { + case ADD: + nm.labels.addAll(labels); + break; + case REPLACE: + nm.labels.clear(); + nm.labels.addAll(labels); + break; + default: + break; + } newNMToLabels.put(nodeId, nm.labels); + } else { + // remove + Node nm = getNMInNodeSet(nodeId); + if (nm.labels != null) { + nm.labels.removeAll(labels); + newNMToLabels.put(nodeId, nm.labels); + } } } } @@ -497,7 +516,7 @@ public class CommonNodeLabelsManager extends AbstractService { } // shows node->labels we added - LOG.info("removeLabelsFromNode:"); + LOG.info(op.name() + " labels on nodes:"); for (Entry> entry : newNMToLabels.entrySet()) { LOG.info(" NM=" + entry.getKey() + ", labels=[" + StringUtils.join(entry.getValue().iterator(), ",") + "]"); @@ -517,7 +536,8 @@ public class CommonNodeLabelsManager extends AbstractService { checkRemoveLabelsFromNode(removeLabelsFromNode); - internalRemoveLabelsFromNode(removeLabelsFromNode); + internalUpdateLabelsOnNodes(removeLabelsFromNode, + NodeLabelUpdateOperation.REMOVE); } protected void checkReplaceLabelsOnNode( @@ -539,47 +559,7 @@ public class CommonNodeLabelsManager extends AbstractService { } } } - - @SuppressWarnings("unchecked") - protected void internalReplaceLabelsOnNode( - Map> replaceLabelsToNode) throws IOException { - // do replace labels to nodes - Map> newNMToLabels = new HashMap>(); - for (Entry> entry : replaceLabelsToNode.entrySet()) { - NodeId nodeId = entry.getKey(); - Set labels = entry.getValue(); - createHostIfNonExisted(nodeId.getHost()); - if (nodeId.getPort() == WILDCARD_PORT) { - Host host = nodeCollections.get(nodeId.getHost()); - host.labels.clear(); - host.labels.addAll(labels); - newNMToLabels.put(nodeId, host.labels); - } else { - createNodeIfNonExisted(nodeId); - Node nm = getNMInNodeSet(nodeId); - if (nm.labels == null) { - nm.labels = new HashSet(); - } - nm.labels.clear(); - nm.labels.addAll(labels); - newNMToLabels.put(nodeId, nm.labels); - } - } - - if (null != dispatcher) { - dispatcher.getEventHandler().handle( - new UpdateNodeToLabelsMappingsEvent(newNMToLabels)); - } - - // shows node->labels we added - LOG.info("setLabelsToNode:"); - for (Entry> entry : newNMToLabels.entrySet()) { - LOG.info(" NM=" + entry.getKey() + ", labels=[" - + StringUtils.join(entry.getValue().iterator(), ",") + "]"); - } - } - /** * replace labels to nodes * @@ -591,7 +571,8 @@ public class CommonNodeLabelsManager extends AbstractService { checkReplaceLabelsOnNode(replaceLabelsToNode); - internalReplaceLabelsOnNode(replaceLabelsToNode); + internalUpdateLabelsOnNodes(replaceLabelsToNode, + NodeLabelUpdateOperation.REPLACE); } /** http://git-wip-us.apache.org/repos/asf/hadoop/blob/d77a3971/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/nodelabels/TestCommonNodeLabelsManager.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/nodelabels/TestCommonNodeLabelsManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/nodelabels/TestCommonNodeLabelsManager.java index a56a595..c0b05e3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/nodelabels/TestCommonNodeLabelsManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/nodelabels/TestCommonNodeLabelsManager.java @@ -281,4 +281,42 @@ public class TestCommonNodeLabelsManager extends NodeLabelTestBase { mgr.removeLabelsFromNode(ImmutableMap.of(toNodeId("n1"), toSet(" p2 "))); Assert.assertTrue(mgr.getNodeLabels().isEmpty()); } + + @Test(timeout = 5000) + public void testReplaceLabelsOnHostsShouldUpdateNodesBelongTo() + throws IOException { + mgr.addToCluserNodeLabels(toSet("p1", "p2", "p3")); + mgr.addLabelsToNode(ImmutableMap.of(toNodeId("n1"), toSet("p1", "p2"))); + assertMapEquals( + mgr.getNodeLabels(), + ImmutableMap.of(toNodeId("n1"), toSet("p1", "p2"))); + + // Replace labels on n1:1 to P2 + mgr.replaceLabelsOnNode(ImmutableMap.of(toNodeId("n1:1"), toSet("p2"), + toNodeId("n1:2"), toSet("p2"))); + assertMapEquals(mgr.getNodeLabels(), ImmutableMap.of(toNodeId("n1"), + toSet("p1", "p2"), toNodeId("n1:1"), toSet("p2"), toNodeId("n1:2"), + toSet("p2"))); + + // Replace labels on n1 to P1, both n1:1/n1 will be P1 now + mgr.replaceLabelsOnNode(ImmutableMap.of(toNodeId("n1"), toSet("p1"))); + assertMapEquals(mgr.getNodeLabels(), ImmutableMap.of(toNodeId("n1"), + toSet("p1"), toNodeId("n1:1"), toSet("p1"), toNodeId("n1:2"), + toSet("p1"))); + + // Set labels on n1:1 to P2 again to verify if add/remove works + mgr.replaceLabelsOnNode(ImmutableMap.of(toNodeId("n1:1"), toSet("p2"))); + + // Add p3 to n1, should makes n1:1 to be p2/p3, and n1:2 to be p1/p3 + mgr.addLabelsToNode(ImmutableMap.of(toNodeId("n1"), toSet("p3"))); + assertMapEquals(mgr.getNodeLabels(), ImmutableMap.of(toNodeId("n1"), + toSet("p1", "p3"), toNodeId("n1:1"), toSet("p2", "p3"), + toNodeId("n1:2"), toSet("p1", "p3"))); + + // Remove P3 from n1, should makes n1:1 to be p2, and n1:2 to be p1 + mgr.removeLabelsFromNode(ImmutableMap.of(toNodeId("n1"), toSet("p3"))); + assertMapEquals(mgr.getNodeLabels(), ImmutableMap.of(toNodeId("n1"), + toSet("p1"), toNodeId("n1:1"), toSet("p2"), toNodeId("n1:2"), + toSet("p1"))); + } } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/d77a3971/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/RMNodeLabelsManager.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/RMNodeLabelsManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/RMNodeLabelsManager.java index 828d1bc..18478e3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/RMNodeLabelsManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/RMNodeLabelsManager.java @@ -37,7 +37,6 @@ import org.apache.hadoop.security.authorize.AccessControlList; import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.conf.YarnConfiguration; -import org.apache.hadoop.yarn.event.Dispatcher; import org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager; import org.apache.hadoop.yarn.nodelabels.NodeLabel; import org.apache.hadoop.yarn.server.resourcemanager.RMContext;