Return-Path: X-Original-To: apmail-hadoop-common-commits-archive@www.apache.org Delivered-To: apmail-hadoop-common-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id B596B181AF for ; Wed, 5 Aug 2015 22:23:19 +0000 (UTC) Received: (qmail 56425 invoked by uid 500); 5 Aug 2015 22:23:15 -0000 Delivered-To: apmail-hadoop-common-commits-archive@hadoop.apache.org Received: (qmail 56290 invoked by uid 500); 5 Aug 2015 22:23:15 -0000 Mailing-List: contact common-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: common-dev@hadoop.apache.org Delivered-To: mailing list common-commits@hadoop.apache.org Received: (qmail 54792 invoked by uid 99); 5 Aug 2015 22:23:15 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 05 Aug 2015 22:23:15 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 092DDE36E4; Wed, 5 Aug 2015 22:23:15 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: jianhe@apache.org To: common-commits@hadoop.apache.org Date: Wed, 05 Aug 2015 22:23:26 -0000 Message-Id: <4e79dee190044e719834c86575a93628@git.apache.org> In-Reply-To: <056ab9cba2974c82b747cd7b0675a1e6@git.apache.org> References: <056ab9cba2974c82b747cd7b0675a1e6@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [13/47] hadoop git commit: YARN-3971. Skip RMNodeLabelsManager#checkRemoveFromClusterNodeLabelsOfQueue on nodelabel recovery. (Bibin A Chundatt via wangda) YARN-3971. Skip RMNodeLabelsManager#checkRemoveFromClusterNodeLabelsOfQueue on nodelabel recovery. (Bibin A Chundatt via wangda) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/91b42e7d Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/91b42e7d Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/91b42e7d Branch: refs/heads/YARN-1197 Commit: 91b42e7d6e8169feb54dbe3715a3269eea6153a9 Parents: 8acb30b Author: Wangda Tan Authored: Thu Jul 30 10:00:31 2015 -0700 Committer: Wangda Tan Committed: Thu Jul 30 10:00:31 2015 -0700 ---------------------------------------------------------------------- hadoop-yarn-project/CHANGES.txt | 3 ++ .../nodelabels/RMNodeLabelsManager.java | 14 ++++-- .../nodelabels/TestRMNodeLabelsManager.java | 48 +++++++++++++++++++- 3 files changed, 59 insertions(+), 6 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/91b42e7d/hadoop-yarn-project/CHANGES.txt ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 4c59da6..f027c29 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -713,6 +713,9 @@ Release 2.8.0 - UNRELEASED YARN-3963. AddNodeLabel on duplicate label addition shows success. (Bibin A Chundatt via wangda) + YARN-3971. Skip RMNodeLabelsManager#checkRemoveFromClusterNodeLabelsOfQueue + on nodelabel recovery. (Bibin A Chundatt via wangda) + Release 2.7.2 - UNRELEASED INCOMPATIBLE CHANGES http://git-wip-us.apache.org/repos/asf/hadoop/blob/91b42e7d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/RMNodeLabelsManager.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/RMNodeLabelsManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/RMNodeLabelsManager.java index 10092ca..8587bda 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/RMNodeLabelsManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/RMNodeLabelsManager.java @@ -33,8 +33,8 @@ import java.util.concurrent.ConcurrentMap; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.service.Service; import org.apache.hadoop.yarn.api.records.NodeId; -import org.apache.hadoop.yarn.api.records.NodeLabel; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager; import org.apache.hadoop.yarn.nodelabels.RMNodeLabel; @@ -114,9 +114,15 @@ public class RMNodeLabelsManager extends CommonNodeLabelsManager { throws IOException { try { writeLock.lock(); - - checkRemoveFromClusterNodeLabelsOfQueue(labelsToRemove); - + if (getServiceState() == Service.STATE.STARTED) { + // We cannot remove node labels from collection when some queue(s) are + // using any of them. + // We will only do this check when service starting finished. Before + // service starting, we will replay edit logs and recover state. It is + // possible that a history operation removed some labels which were being + // used by some queues in the past but not used by current queues. + checkRemoveFromClusterNodeLabelsOfQueue(labelsToRemove); + } // copy before NMs Map before = cloneNodeMap(); http://git-wip-us.apache.org/repos/asf/hadoop/blob/91b42e7d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/TestRMNodeLabelsManager.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/TestRMNodeLabelsManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/TestRMNodeLabelsManager.java index 05bb1e5..7940865 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/TestRMNodeLabelsManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/TestRMNodeLabelsManager.java @@ -18,7 +18,10 @@ package org.apache.hadoop.yarn.server.resourcemanager.nodelabels; +import java.io.File; import java.io.IOException; +import java.util.Arrays; +import java.util.Collection; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -31,6 +34,7 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager; import org.apache.hadoop.yarn.nodelabels.RMNodeLabel; import org.apache.hadoop.yarn.nodelabels.NodeLabelTestBase; +import org.apache.hadoop.yarn.server.resourcemanager.MockRM; import org.apache.hadoop.yarn.util.resource.Resources; import org.junit.After; import org.junit.Assert; @@ -46,7 +50,8 @@ public class TestRMNodeLabelsManager extends NodeLabelTestBase { private final Resource LARGE_NODE = Resource.newInstance(1000, 0); NullRMNodeLabelsManager mgr = null; - + RMNodeLabelsManager lmgr = null; + boolean checkQueueCall = false; @Before public void before() { mgr = new NullRMNodeLabelsManager(); @@ -506,7 +511,46 @@ public class TestRMNodeLabelsManager extends NodeLabelTestBase { checkNodeLabelInfo(infos, "y", 1, 10); checkNodeLabelInfo(infos, "z", 0, 0); } - + + @Test(timeout = 60000) + public void testcheckRemoveFromClusterNodeLabelsOfQueue() throws Exception { + class TestRMLabelManger extends RMNodeLabelsManager { + @Override + protected void checkRemoveFromClusterNodeLabelsOfQueue( + Collection labelsToRemove) throws IOException { + checkQueueCall = true; + // Do nothing + } + + } + lmgr = new TestRMLabelManger(); + Configuration conf = new Configuration(); + File tempDir = File.createTempFile("nlb", ".tmp"); + tempDir.delete(); + tempDir.mkdirs(); + tempDir.deleteOnExit(); + conf.set(YarnConfiguration.FS_NODE_LABELS_STORE_ROOT_DIR, + tempDir.getAbsolutePath()); + conf.setBoolean(YarnConfiguration.NODE_LABELS_ENABLED, true); + MockRM rm = new MockRM(conf) { + @Override + public RMNodeLabelsManager createNodeLabelManager() { + return lmgr; + } + }; + lmgr.addToCluserNodeLabelsWithDefaultExclusivity(toSet("a")); + lmgr.removeFromClusterNodeLabels(Arrays.asList(new String[] { "a" })); + rm.getRMContext().setNodeLabelManager(lmgr); + rm.start(); + lmgr.addToCluserNodeLabelsWithDefaultExclusivity(toSet("a")); + Assert.assertEquals(false, checkQueueCall); + lmgr.removeFromClusterNodeLabels(Arrays.asList(new String[] { "a" })); + Assert.assertEquals(true, checkQueueCall); + lmgr.stop(); + lmgr.close(); + rm.stop(); + } + @Test(timeout = 5000) public void testLabelsToNodesOnNodeActiveDeactive() throws Exception { // Activate a node without assigning any labels