Return-Path: X-Original-To: apmail-hadoop-common-commits-archive@www.apache.org Delivered-To: apmail-hadoop-common-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 5E48E18CDC for ; Tue, 23 Feb 2016 20:51:58 +0000 (UTC) Received: (qmail 56338 invoked by uid 500); 23 Feb 2016 20:51:58 -0000 Delivered-To: apmail-hadoop-common-commits-archive@hadoop.apache.org Received: (qmail 56270 invoked by uid 500); 23 Feb 2016 20:51:58 -0000 Mailing-List: contact common-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: common-dev@hadoop.apache.org Delivered-To: mailing list common-commits@hadoop.apache.org Received: (qmail 56254 invoked by uid 99); 23 Feb 2016 20:51:58 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 23 Feb 2016 20:51:58 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id EF1D8E8E87; Tue, 23 Feb 2016 20:51:57 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: jlowe@apache.org To: common-commits@hadoop.apache.org Message-Id: X-Mailer: ASF-Git Admin Mailer Subject: hadoop git commit: YARN-2046. Out of band heartbeats are sent only on container kill and possibly too early. Contributed by Ming Ma (cherry picked from commit d284e187b8db43056236032ebc2114ee462c27f6) Date: Tue, 23 Feb 2016 20:51:57 +0000 (UTC) Repository: hadoop Updated Branches: refs/heads/branch-2.8 7812cfdd8 -> 6f3f5a8c3 YARN-2046. Out of band heartbeats are sent only on container kill and possibly too early. Contributed by Ming Ma (cherry picked from commit d284e187b8db43056236032ebc2114ee462c27f6) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6f3f5a8c Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6f3f5a8c Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6f3f5a8c Branch: refs/heads/branch-2.8 Commit: 6f3f5a8c38d912619cb4cf17d63e9d7d980844df Parents: 7812cfd Author: Jason Lowe Authored: Tue Feb 23 20:49:09 2016 +0000 Committer: Jason Lowe Committed: Tue Feb 23 20:51:41 2016 +0000 ---------------------------------------------------------------------- hadoop-yarn-project/CHANGES.txt | 6 +++++ .../hadoop/yarn/server/nodemanager/Context.java | 2 ++ .../yarn/server/nodemanager/NodeManager.java | 10 +++++++ .../containermanager/ContainerManagerImpl.java | 12 +++------ .../container/ContainerImpl.java | 25 ++++++++++------- .../yarn/server/nodemanager/TestEventFlow.java | 1 + .../nodemanager/TestNodeStatusUpdater.java | 28 +++++++++++++------- .../amrmproxy/BaseAMRMProxyTest.java | 5 ++++ .../BaseContainerManagerTest.java | 1 + .../containermanager/TestAuxServices.java | 7 +++-- .../container/TestContainer.java | 16 ++++++++--- .../nodemanager/webapp/TestNMWebServer.java | 5 ++-- 12 files changed, 83 insertions(+), 35 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f3f5a8c/hadoop-yarn-project/CHANGES.txt ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 6d49f39..aa02fc2 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -1310,6 +1310,9 @@ Release 2.7.3 - UNRELEASED YARN-4707. Remove the extra char (>) from SecureContainer.md. (Brahma Reddy Battula via aajisaka) + YARN-2046. Out of band heartbeats are sent only on container kill and + possibly too early (Ming Ma via jlowe) + Release 2.7.2 - 2016-01-25 INCOMPATIBLE CHANGES @@ -2167,6 +2170,9 @@ Release 2.6.5 - UNRELEASED BUG FIXES + YARN-2046. Out of band heartbeats are sent only on container kill and + possibly too early (Ming Ma via jlowe) + Release 2.6.4 - 2016-02-11 INCOMPATIBLE CHANGES http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f3f5a8c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/Context.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/Context.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/Context.java index 9c2d1fb..d3251ae 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/Context.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/Context.java @@ -87,4 +87,6 @@ public interface Context { ConcurrentLinkedQueue getLogAggregationStatusForApps(); + + NodeStatusUpdater getNodeStatusUpdater(); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f3f5a8c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java index 04e383f..15907ee 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java @@ -352,6 +352,7 @@ public class NodeManager extends CompositeService // StatusUpdater should be added last so that it get started last // so that we make sure everything is up before registering with RM. addService(nodeStatusUpdater); + ((NMContext) context).setNodeStatusUpdater(nodeStatusUpdater); super.serviceInit(conf); // TODO add local dirs to del @@ -461,6 +462,7 @@ public class NodeManager extends CompositeService private boolean isDecommissioned = false; private final ConcurrentLinkedQueue logAggregationReportForApps; + private NodeStatusUpdater nodeStatusUpdater; public NMContext(NMContainerTokenSecretManager containerTokenSecretManager, NMTokenSecretManagerInNM nmTokenSecretManager, @@ -588,6 +590,14 @@ public class NodeManager extends CompositeService getLogAggregationStatusForApps() { return this.logAggregationReportForApps; } + + public NodeStatusUpdater getNodeStatusUpdater() { + return this.nodeStatusUpdater; + } + + public void setNodeStatusUpdater(NodeStatusUpdater nodeStatusUpdater) { + this.nodeStatusUpdater = nodeStatusUpdater; + } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f3f5a8c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java index d0663d5..e3d8f1d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java @@ -347,9 +347,9 @@ public class ContainerManagerImpl extends CompositeService implements Credentials credentials = YarnServerSecurityUtils.parseCredentials(launchContext); Container container = new ContainerImpl(getConfig(), dispatcher, - context.getNMStateStore(), req.getContainerLaunchContext(), + req.getContainerLaunchContext(), credentials, metrics, token, rcs.getStatus(), rcs.getExitCode(), - rcs.getDiagnostics(), rcs.getKilled(), rcs.getCapability()); + rcs.getDiagnostics(), rcs.getKilled(), rcs.getCapability(), context); context.getContainers().put(containerId, container); dispatcher.getEventHandler().handle( new ApplicationContainerInitEvent(container)); @@ -907,8 +907,8 @@ public class ContainerManagerImpl extends CompositeService implements Container container = new ContainerImpl(getConfig(), this.dispatcher, - context.getNMStateStore(), launchContext, - credentials, metrics, containerTokenIdentifier); + launchContext, credentials, metrics, containerTokenIdentifier, + context); ApplicationId applicationID = containerId.getApplicationAttemptId().getApplicationId(); if (context.getContainers().putIfAbsent(containerId, container) != null) { @@ -1179,10 +1179,6 @@ public class ContainerManagerImpl extends CompositeService implements NMAuditLogger.logSuccess(container.getUser(), AuditConstants.STOP_CONTAINER, "ContainerManageImpl", containerID .getApplicationAttemptId().getApplicationId(), containerID); - - // TODO: Move this code to appropriate place once kill_container is - // implemented. - nodeStatusUpdater.sendOutofBandHeartBeat(); } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f3f5a8c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java index 6186b94..dbc7de8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java @@ -67,6 +67,7 @@ import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.shar import org.apache.hadoop.yarn.server.nodemanager.containermanager.loghandler.event.LogHandlerContainerFinishedEvent; import org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor.ContainerStartMonitoringEvent; import org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor.ContainerStopMonitoringEvent; +import org.apache.hadoop.yarn.server.nodemanager.Context; import org.apache.hadoop.yarn.server.nodemanager.metrics.NodeManagerMetrics; import org.apache.hadoop.yarn.server.nodemanager.recovery.NMStateStoreService; import org.apache.hadoop.yarn.server.nodemanager.recovery.NMStateStoreService.RecoveredContainerStatus; @@ -125,14 +126,15 @@ public class ContainerImpl implements Container { RecoveredContainerStatus.REQUESTED; // whether container was marked as killed after recovery private boolean recoveredAsKilled = false; + private Context context; public ContainerImpl(Configuration conf, Dispatcher dispatcher, - NMStateStoreService stateStore, ContainerLaunchContext launchContext, - Credentials creds, NodeManagerMetrics metrics, - ContainerTokenIdentifier containerTokenIdentifier) { + ContainerLaunchContext launchContext, Credentials creds, + NodeManagerMetrics metrics, + ContainerTokenIdentifier containerTokenIdentifier, Context context) { this.daemonConf = conf; this.dispatcher = dispatcher; - this.stateStore = stateStore; + this.stateStore = context.getNMStateStore(); this.launchContext = launchContext; this.containerTokenIdentifier = containerTokenIdentifier; this.containerId = containerTokenIdentifier.getContainerID(); @@ -144,19 +146,21 @@ public class ContainerImpl implements Container { ReadWriteLock readWriteLock = new ReentrantReadWriteLock(); this.readLock = readWriteLock.readLock(); this.writeLock = readWriteLock.writeLock(); + this.context = context; stateMachine = stateMachineFactory.make(this); } // constructor for a recovered container public ContainerImpl(Configuration conf, Dispatcher dispatcher, - NMStateStoreService stateStore, ContainerLaunchContext launchContext, - Credentials creds, NodeManagerMetrics metrics, + ContainerLaunchContext launchContext, Credentials creds, + NodeManagerMetrics metrics, ContainerTokenIdentifier containerTokenIdentifier, RecoveredContainerStatus recoveredStatus, int exitCode, - String diagnostics, boolean wasKilled, Resource recoveredCapability) { - this(conf, dispatcher, stateStore, launchContext, creds, metrics, - containerTokenIdentifier); + String diagnostics, boolean wasKilled, Resource recoveredCapability, + Context context) { + this(conf, dispatcher, launchContext, creds, metrics, + containerTokenIdentifier, context); this.recoveredStatus = recoveredStatus; this.exitCode = exitCode; this.recoveredAsKilled = wasKilled; @@ -988,11 +992,12 @@ public class ContainerImpl implements Container { container.sendFinishedEvents(); //if the current state is NEW it means the CONTAINER_INIT was never // sent for the event, thus no need to send the CONTAINER_STOP - if (container.getCurrentState() + if (container.getCurrentState() != org.apache.hadoop.yarn.api.records.ContainerState.NEW) { container.dispatcher.getEventHandler().handle(new AuxServicesEvent (AuxServicesEventType.CONTAINER_STOP, container)); } + container.context.getNodeStatusUpdater().sendOutofBandHeartBeat(); } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f3f5a8c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestEventFlow.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestEventFlow.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestEventFlow.java index 3dc62bc..ddbfbb9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestEventFlow.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestEventFlow.java @@ -131,6 +131,7 @@ public class TestEventFlow { nodeStatusUpdater.init(conf); ((NMContext)context).setContainerManager(containerManager); nodeStatusUpdater.start(); + ((NMContext)context).setNodeStatusUpdater(nodeStatusUpdater); containerManager.init(conf); containerManager.start(); http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f3f5a8c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java index 90804b8..b1f826b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java @@ -252,8 +252,10 @@ public class TestNodeStatusUpdater { firstContainerID, InetAddress.getByName("localhost") .getCanonicalHostName(), 1234, user, resource, currentTime + 10000, 123, "password".getBytes(), currentTime)); + Context context = mock(Context.class); + when(context.getNMStateStore()).thenReturn(stateStore); Container container = new ContainerImpl(conf, mockDispatcher, - stateStore, launchContext, null, mockMetrics, containerToken); + launchContext, null, mockMetrics, containerToken, context); this.context.getContainers().put(firstContainerID, container); } else if (heartBeatID == 2) { // Checks on the RM end @@ -291,8 +293,10 @@ public class TestNodeStatusUpdater { secondContainerID, InetAddress.getByName("localhost") .getCanonicalHostName(), 1234, user, resource, currentTime + 10000, 123, "password".getBytes(), currentTime)); + Context context = mock(Context.class); + when(context.getNMStateStore()).thenReturn(stateStore); Container container = new ContainerImpl(conf, mockDispatcher, - stateStore, launchContext, null, mockMetrics, containerToken); + launchContext, null, mockMetrics, containerToken, context); this.context.getContainers().put(secondContainerID, container); } else if (heartBeatID == 3) { // Checks on the RM end @@ -1007,8 +1011,9 @@ public class TestNodeStatusUpdater { BuilderUtils.newResource(1024, 1), 0, 123, "password".getBytes(), 0); Container anyCompletedContainer = new ContainerImpl(conf, null, - null, null, null, null, - BuilderUtils.newContainerTokenIdentifier(containerToken)) { + null, null, null, + BuilderUtils.newContainerTokenIdentifier(containerToken), + nm.getNMContext()) { @Override public ContainerState getCurrentState() { @@ -1028,8 +1033,9 @@ public class TestNodeStatusUpdater { 1234, "anyUser", BuilderUtils.newResource(1024, 1), 0, 123, "password".getBytes(), 0); Container runningContainer = - new ContainerImpl(conf, null, null, null, null, null, - BuilderUtils.newContainerTokenIdentifier(runningContainerToken)) { + new ContainerImpl(conf, null, null, null, null, + BuilderUtils.newContainerTokenIdentifier(runningContainerToken), + nm.getNMContext()) { @Override public ContainerState getCurrentState() { return ContainerState.RUNNING; @@ -1086,8 +1092,9 @@ public class TestNodeStatusUpdater { BuilderUtils.newResource(1024, 1), 0, 123, "password".getBytes(), 0); Container completedContainer = new ContainerImpl(conf, null, - null, null, null, null, - BuilderUtils.newContainerTokenIdentifier(containerToken)) { + null, null, null, + BuilderUtils.newContainerTokenIdentifier(containerToken), + nm.getNMContext()) { @Override public ContainerState getCurrentState() { return ContainerState.COMPLETE; @@ -1123,8 +1130,9 @@ public class TestNodeStatusUpdater { BuilderUtils.newResource(1024, 1), 0, 123, "password".getBytes(), 0); Container anyCompletedContainer = new ContainerImpl(conf, null, - null, null, null, null, - BuilderUtils.newContainerTokenIdentifier(containerToken)) { + null, null, null, + BuilderUtils.newContainerTokenIdentifier(containerToken), + nm.getNMContext()) { @Override public ContainerState getCurrentState() { http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f3f5a8c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/BaseAMRMProxyTest.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/BaseAMRMProxyTest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/BaseAMRMProxyTest.java index 9bc23f6..1aea9d2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/BaseAMRMProxyTest.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/BaseAMRMProxyTest.java @@ -63,6 +63,7 @@ import org.apache.hadoop.yarn.server.api.records.NodeHealthStatus; import org.apache.hadoop.yarn.server.nodemanager.Context; import org.apache.hadoop.yarn.server.nodemanager.LocalDirsHandlerService; import org.apache.hadoop.yarn.server.nodemanager.NodeResourceMonitor; +import org.apache.hadoop.yarn.server.nodemanager.NodeStatusUpdater; import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.Application; import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container; import org.apache.hadoop.yarn.server.nodemanager.recovery.NMStateStoreService; @@ -678,5 +679,9 @@ public abstract class BaseAMRMProxyTest { return null; } + @Override + public NodeStatusUpdater getNodeStatusUpdater() { + return null; + } } } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f3f5a8c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/BaseContainerManagerTest.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/BaseContainerManagerTest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/BaseContainerManagerTest.java index c902fd5..787778e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/BaseContainerManagerTest.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/BaseContainerManagerTest.java @@ -188,6 +188,7 @@ public abstract class BaseContainerManagerTest { nodeStatusUpdater.init(conf); containerManager.init(conf); nodeStatusUpdater.start(); + ((NMContext)context).setNodeStatusUpdater(nodeStatusUpdater); } protected ContainerManagerImpl http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f3f5a8c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestAuxServices.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestAuxServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestAuxServices.java index 1380752..91466e8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestAuxServices.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestAuxServices.java @@ -25,6 +25,7 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; +import static org.mockito.Mockito.mock; import java.io.File; import java.io.IOException; @@ -53,6 +54,7 @@ import org.apache.hadoop.yarn.server.api.ApplicationTerminationContext; import org.apache.hadoop.yarn.server.api.AuxiliaryService; import org.apache.hadoop.yarn.server.api.ContainerInitializationContext; import org.apache.hadoop.yarn.server.api.ContainerTerminationContext; +import org.apache.hadoop.yarn.server.nodemanager.Context; import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container; import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerImpl; import org.junit.Assert; @@ -192,8 +194,9 @@ public class TestAuxServices { ContainerTokenIdentifier cti = new ContainerTokenIdentifier( ContainerId.newContainerId(attemptId, 1), "", "", Resource.newInstance(1, 1), 0,0,0, Priority.newInstance(0), 0); - Container container = new ContainerImpl(null, null, null, null, null, - null, cti); + Context context = mock(Context.class); + Container container = new ContainerImpl(null, null, null, null, + null, cti, context); ContainerId containerId = container.getContainerId(); Resource resource = container.getResource(); event = new AuxServicesEvent(AuxServicesEventType.CONTAINER_INIT,container); http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f3f5a8c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/TestContainer.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/TestContainer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/TestContainer.java index 2ab9842..3e06236 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/TestContainer.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/TestContainer.java @@ -88,6 +88,7 @@ import org.apache.hadoop.yarn.server.nodemanager.containermanager.loghandler.eve import org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor.ContainersMonitorEvent; import org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor.ContainersMonitorEventType; import org.apache.hadoop.yarn.server.nodemanager.metrics.NodeManagerMetrics; +import org.apache.hadoop.yarn.server.nodemanager.NodeStatusUpdater; import org.apache.hadoop.yarn.server.nodemanager.recovery.NMNullStateStoreService; import org.apache.hadoop.yarn.server.utils.BuilderUtils; import org.junit.Assert; @@ -261,6 +262,7 @@ public class TestContainer { wc.containerSuccessful(); wc.containerResourcesCleanup(); assertEquals(ContainerState.DONE, wc.c.getContainerState()); + verifyOutofBandHeartBeat(wc); assertNull(wc.c.getLocalizedResources()); // Now in DONE, issue INIT wc.initContainer(); @@ -290,6 +292,7 @@ public class TestContainer { wc.containerSuccessful(); wc.containerResourcesCleanup(); assertEquals(ContainerState.DONE, wc.c.getContainerState()); + verifyOutofBandHeartBeat(wc); assertNull(wc.c.getLocalizedResources()); // Now in DONE, issue RESOURCE_FAILED as done by LocalizeRunner wc.resourceFailedContainer(); @@ -336,6 +339,7 @@ public class TestContainer { int killed = metrics.getKilledContainers(); wc.killContainer(); assertEquals(ContainerState.DONE, wc.c.getContainerState()); + verifyOutofBandHeartBeat(wc); assertEquals(ContainerExitStatus.KILLED_BY_RESOURCEMANAGER, wc.c.cloneAndGetContainerStatus().getExitStatus()); assertTrue(wc.c.cloneAndGetContainerStatus().getDiagnostics() @@ -653,6 +657,10 @@ public class TestContainer { verify(wc.localizerBus).handle(argThat(matchesReq)); } + private void verifyOutofBandHeartBeat(WrappedContainer wc) { + verify(wc.context.getNodeStatusUpdater()).sendOutofBandHeartBeat(); + } + private static class ResourcesReleasedMatcher extends ArgumentMatcher { final HashSet resources = @@ -779,6 +787,7 @@ public class TestContainer { final Container c; final Map localResources; final Map serviceData; + final Context context = mock(Context.class); WrappedContainer(int appId, long timestamp, int id, String user) throws IOException { @@ -804,11 +813,12 @@ public class TestContainer { dispatcher.register(ApplicationEventType.class, appBus); dispatcher.register(LogHandlerEventType.class, LogBus); - Context context = mock(Context.class); when(context.getApplications()).thenReturn( new ConcurrentHashMap()); NMNullStateStoreService stateStore = new NMNullStateStoreService(); when(context.getNMStateStore()).thenReturn(stateStore); + NodeStatusUpdater nodeStatusUpdater = mock(NodeStatusUpdater.class); + when(context.getNodeStatusUpdater()).thenReturn(nodeStatusUpdater); ContainerExecutor executor = mock(ContainerExecutor.class); launcher = new ContainersLauncher(context, dispatcher, executor, null, null); @@ -864,8 +874,8 @@ public class TestContainer { } when(ctxt.getServiceData()).thenReturn(serviceData); - c = new ContainerImpl(conf, dispatcher, new NMNullStateStoreService(), - ctxt, null, metrics, identifier); + c = new ContainerImpl(conf, dispatcher, ctxt, null, metrics, identifier, + context); dispatcher.register(ContainerEventType.class, new EventHandler() { @Override http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f3f5a8c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServer.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServer.java index e1845c7..f0c7cbc 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServer.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServer.java @@ -215,10 +215,11 @@ public class TestNMWebServer { BuilderUtils.newContainerToken(containerId, "127.0.0.1", 1234, user, BuilderUtils.newResource(1024, 1), currentTime + 10000L, 123, "password".getBytes(), currentTime); + Context context = mock(Context.class); Container container = - new ContainerImpl(conf, dispatcher, stateStore, launchContext, + new ContainerImpl(conf, dispatcher, launchContext, null, metrics, - BuilderUtils.newContainerTokenIdentifier(containerToken)) { + BuilderUtils.newContainerTokenIdentifier(containerToken), context) { @Override public ContainerState getContainerState() {