Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id 1D895200BB5 for ; Sun, 6 Nov 2016 17:31:20 +0100 (CET) Received: by cust-asf.ponee.io (Postfix) id 1C239160B0D; Sun, 6 Nov 2016 16:31:20 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id 6674B160AE8 for ; Sun, 6 Nov 2016 17:31:19 +0100 (CET) Received: (qmail 23924 invoked by uid 500); 6 Nov 2016 16:31:18 -0000 Mailing-List: contact common-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Delivered-To: mailing list common-commits@hadoop.apache.org Received: (qmail 23913 invoked by uid 99); 6 Nov 2016 16:31:18 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Sun, 06 Nov 2016 16:31:18 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 69069EEE43; Sun, 6 Nov 2016 16:31:18 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: varunsaxena@apache.org To: common-commits@hadoop.apache.org Date: Sun, 06 Nov 2016 16:31:18 -0000 Message-Id: <3655727bccc04a60923f996409b4b7cb@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [01/50] [abbrv] hadoop git commit: YARN-5837. NPE when getting node status of a decommissioned node after an RM restart. Contributed by Robert Kanter (cherry picked from commit 6bb741ff0ef208a8628bc64d6537999d4cd67955) [Forced Update!] archived-at: Sun, 06 Nov 2016 16:31:20 -0000 Repository: hadoop Updated Branches: refs/heads/YARN-5355-branch-2 bd800f2b9 -> ae0349edc (forced update) YARN-5837. NPE when getting node status of a decommissioned node after an RM restart. Contributed by Robert Kanter (cherry picked from commit 6bb741ff0ef208a8628bc64d6537999d4cd67955) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/27ed2c52 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/27ed2c52 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/27ed2c52 Branch: refs/heads/YARN-5355-branch-2 Commit: 27ed2c526ad7a1d025a99ef8e81abe49cb85ae52 Parents: 4cee535 Author: Jason Lowe Authored: Fri Nov 4 22:20:21 2016 +0000 Committer: Jason Lowe Committed: Fri Nov 4 22:21:45 2016 +0000 ---------------------------------------------------------------------- .../server/resourcemanager/NodesListManager.java | 4 +++- .../yarn/server/resourcemanager/MockNM.java | 4 ++++ .../server/resourcemanager/TestRMRestart.java | 19 +++++++++++++++++++ 3 files changed, 26 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/27ed2c52/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/NodesListManager.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/NodesListManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/NodesListManager.java index 99413bc..7d69f93 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/NodesListManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/NodesListManager.java @@ -44,6 +44,7 @@ import org.apache.hadoop.util.Time; import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.api.records.NodeState; +import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.event.EventHandler; import org.apache.hadoop.yarn.exceptions.YarnException; @@ -242,7 +243,8 @@ public class NodesListManager extends CompositeService implements for (final String host : excludeList) { NodeId nodeId = createUnknownNodeId(host); RMNodeImpl rmNode = new RMNodeImpl(nodeId, - rmContext, host, -1, -1, new UnknownNode(host), null, null); + rmContext, host, -1, -1, new UnknownNode(host), + Resource.newInstance(0, 0), "unknown"); rmContext.getInactiveRMNodes().put(nodeId, rmNode); rmNode.handle(new RMNodeEvent(nodeId, RMNodeEventType.DECOMMISSION)); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/27ed2c52/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNM.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNM.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNM.java index 2e2bef7..567db86 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNM.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNM.java @@ -256,4 +256,8 @@ public class MockNM { public int getvCores() { return vCores; } + + public String getVersion() { + return version; + } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/27ed2c52/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java index b793d32..19928e1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java @@ -40,6 +40,7 @@ import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; +import java.util.concurrent.ConcurrentMap; import org.apache.commons.io.FileUtils; import org.apache.commons.logging.Log; @@ -103,6 +104,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppImpl; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState; +import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.TestSchedulerUtils; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.YarnScheduler; @@ -1894,6 +1896,9 @@ public class TestRMRestart extends ParameterizedSchedulerTestBase { rm1.start(); MockNM nm1 = rm1.registerNode("localhost:1234", 8000); MockNM nm2 = rm1.registerNode("host2:1234", 8000); + Resource expectedCapability = + Resource.newInstance(nm1.getMemory(), nm1.getvCores()); + String expectedVersion = nm1.getVersion(); Assert .assertEquals(0, ClusterMetrics.getMetrics().getNumDecommisionedNMs()); @@ -1915,6 +1920,7 @@ public class TestRMRestart extends ParameterizedSchedulerTestBase { Assert .assertEquals(2, ClusterMetrics.getMetrics().getNumDecommisionedNMs()); + verifyNodesAfterDecom(rm1, 2, expectedCapability, expectedVersion); rm1.stop(); rm1 = null; Assert @@ -1928,6 +1934,7 @@ public class TestRMRestart extends ParameterizedSchedulerTestBase { Assert .assertEquals(2, ClusterMetrics.getMetrics().getNumDecommisionedNMs()); + verifyNodesAfterDecom(rm2, 2, Resource.newInstance(0, 0), "unknown"); } finally { if (rm1 != null) { rm1.stop(); @@ -1938,6 +1945,18 @@ public class TestRMRestart extends ParameterizedSchedulerTestBase { } } + private void verifyNodesAfterDecom(MockRM rm, int numNodes, + Resource expectedCapability, + String expectedVersion) { + ConcurrentMap inactiveRMNodes = + rm.getRMContext().getInactiveRMNodes(); + Assert.assertEquals(numNodes, inactiveRMNodes.size()); + for (RMNode rmNode : inactiveRMNodes.values()) { + Assert.assertEquals(expectedCapability, rmNode.getTotalCapability()); + Assert.assertEquals(expectedVersion, rmNode.getNodeManagerVersion()); + } + } + // Test Delegation token is renewed synchronously so that recover events // can be processed before any other external incoming events, specifically // the ContainerFinished event on NM re-registraton. --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org For additional commands, e-mail: common-commits-help@hadoop.apache.org