From common-commits-return-82472-archive-asf-public=cust-asf.ponee.io@hadoop.apache.org Tue May 8 20:38:05 2018 Return-Path: X-Original-To: archive-asf-public@cust-asf.ponee.io Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by mx-eu-01.ponee.io (Postfix) with SMTP id 50EDA18067B for ; Tue, 8 May 2018 20:38:04 +0200 (CEST) Received: (qmail 50049 invoked by uid 500); 8 May 2018 18:38:03 -0000 Mailing-List: contact common-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Delivered-To: mailing list common-commits@hadoop.apache.org Received: (qmail 50040 invoked by uid 99); 8 May 2018 18:38:03 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 08 May 2018 18:38:03 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 93228E0614; Tue, 8 May 2018 18:38:02 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: wangda@apache.org To: common-commits@hadoop.apache.org Message-Id: <5816cd5399ff428680e6cbadbf1410a8@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: hadoop git commit: YARN-8232. RMContainer lost queue name when RM HA happens. (Hu Ziqian via wangda) Date: Tue, 8 May 2018 18:38:02 +0000 (UTC) Repository: hadoop Updated Branches: refs/heads/branch-3.1 b69c65fc6 -> 0873cfcb5 YARN-8232. RMContainer lost queue name when RM HA happens. (Hu Ziqian via wangda) Change-Id: Ia21e1da6871570c993bbedde76ce32929e95970f (cherry picked from commit 6b96a73bb0f0ad1c877a062b19091e3e15a33ec4) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0873cfcb Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0873cfcb Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0873cfcb Branch: refs/heads/branch-3.1 Commit: 0873cfcb5601c601f97854fb15af641a479beadd Parents: b69c65f Author: Wangda Tan Authored: Tue May 8 11:34:45 2018 -0700 Committer: Wangda Tan Committed: Tue May 8 11:35:11 2018 -0700 ---------------------------------------------------------------------- .../scheduler/AbstractYarnScheduler.java | 6 +- .../scheduler/TestAbstractYarnScheduler.java | 60 ++++++++++++++++---- 2 files changed, 54 insertions(+), 12 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/0873cfcb/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java index e76287d..b2747f7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java @@ -531,7 +531,8 @@ public abstract class AbstractYarnScheduler } // create container - RMContainer rmContainer = recoverAndCreateContainer(container, nm); + RMContainer rmContainer = recoverAndCreateContainer(container, nm, + schedulerApp.getQueue().getQueueName()); // recover RMContainer rmContainer.handle( @@ -581,7 +582,7 @@ public abstract class AbstractYarnScheduler } private RMContainer recoverAndCreateContainer(NMContainerStatus status, - RMNode node) { + RMNode node, String queueName) { Container container = Container.newInstance(status.getContainerId(), node.getNodeID(), node.getHttpAddress(), status.getAllocatedResource(), @@ -596,6 +597,7 @@ public abstract class AbstractYarnScheduler SchedulerRequestKey.extractFrom(container), attemptId, node.getNodeID(), applications.get(attemptId.getApplicationId()).getUser(), rmContext, status.getCreationTime(), status.getNodeLabelExpression()); + ((RMContainerImpl) rmContainer).setQueueName(queueName); return rmContainer; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/0873cfcb/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestAbstractYarnScheduler.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestAbstractYarnScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestAbstractYarnScheduler.java index 979e68a..c0f8d39 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestAbstractYarnScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestAbstractYarnScheduler.java @@ -30,22 +30,14 @@ import java.util.Map; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.service.Service; -import org.apache.hadoop.yarn.api.records.ApplicationAccessType; -import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; -import org.apache.hadoop.yarn.api.records.Container; -import org.apache.hadoop.yarn.api.records.ContainerId; -import org.apache.hadoop.yarn.api.records.ContainerState; -import org.apache.hadoop.yarn.api.records.ContainerStatus; -import org.apache.hadoop.yarn.api.records.NodeId; -import org.apache.hadoop.yarn.api.records.Resource; -import org.apache.hadoop.yarn.api.records.ResourceOption; -import org.apache.hadoop.yarn.api.records.ResourceRequest; +import org.apache.hadoop.yarn.api.records.*; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.event.Dispatcher; import org.apache.hadoop.yarn.event.DrainDispatcher; import org.apache.hadoop.yarn.event.EventHandler; import org.apache.hadoop.yarn.factories.RecordFactory; import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; +import org.apache.hadoop.yarn.server.api.protocolrecords.NMContainerStatus; import org.apache.hadoop.yarn.server.api.protocolrecords.RegisterNodeManagerRequest; import org.apache.hadoop.yarn.server.resourcemanager.MockAM; import org.apache.hadoop.yarn.server.resourcemanager.MockNM; @@ -59,6 +51,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.RMContextImpl; import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager; import org.apache.hadoop.yarn.server.resourcemanager.ResourceTrackerService; import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager; +import org.apache.hadoop.yarn.server.resourcemanager.recovery.MemoryRMStateStore; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.MockRMApp; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState; @@ -874,4 +867,51 @@ public class TestAbstractYarnScheduler extends ParameterizedSchedulerTestBase { rm.stop(); } } + + @Test(timeout=60000) + public void testContainerRecoveredByNode() throws Exception { + System.out.println("Starting testContainerRecoveredByNode"); + final int maxMemory = 10 * 1024; + YarnConfiguration conf = getConf(); + conf.setBoolean(YarnConfiguration.RECOVERY_ENABLED, true); + conf.setBoolean( + YarnConfiguration.RM_WORK_PRESERVING_RECOVERY_ENABLED, true); + conf.set( + YarnConfiguration.RM_STORE, MemoryRMStateStore.class.getName()); + MockRM rm1 = new MockRM(conf); + try { + rm1.start(); + RMApp app1 = + rm1.submitApp(200, "name", "user", + new HashMap(), false, "default", + -1, null, "Test", false, true); + MockNM nm1 = + new MockNM("127.0.0.1:1234", 10240, rm1.getResourceTrackerService()); + nm1.registerNode(); + MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1); + am1.allocate("127.0.0.1", 8192, 1, new ArrayList()); + + YarnScheduler scheduler = rm1.getResourceScheduler(); + + RMNode node1 = MockNodes.newNodeInfo( + 0, Resources.createResource(maxMemory), 1, "127.0.0.2"); + ContainerId containerId = ContainerId.newContainerId( + app1.getCurrentAppAttempt().getAppAttemptId(), 2); + NMContainerStatus containerReport = + NMContainerStatus.newInstance(containerId, 0, ContainerState.RUNNING, + Resource.newInstance(1024, 1), "recover container", 0, + Priority.newInstance(0), 0); + List containerReports = new ArrayList<>(); + containerReports.add(containerReport); + scheduler.handle(new NodeAddedSchedulerEvent(node1, containerReports)); + RMContainer rmContainer = scheduler.getRMContainer(containerId); + + //verify queue name when rmContainer is recovered + Assert.assertEquals(app1.getQueue(), rmContainer.getQueueName()); + + } finally { + rm1.stop(); + System.out.println("Stopping testContainerRecoveredByNode"); + } + } } --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org For additional commands, e-mail: common-commits-help@hadoop.apache.org