Return-Path: X-Original-To: apmail-hadoop-common-commits-archive@www.apache.org Delivered-To: apmail-hadoop-common-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id CCC8211F9F for ; Sat, 6 Sep 2014 04:37:29 +0000 (UTC) Received: (qmail 99744 invoked by uid 500); 6 Sep 2014 04:37:24 -0000 Delivered-To: apmail-hadoop-common-commits-archive@hadoop.apache.org Received: (qmail 99675 invoked by uid 500); 6 Sep 2014 04:37:24 -0000 Mailing-List: contact common-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: common-dev@hadoop.apache.org Delivered-To: mailing list common-commits@hadoop.apache.org Received: (qmail 99640 invoked by uid 99); 6 Sep 2014 04:37:24 -0000 Received: from tyr.zones.apache.org (HELO tyr.zones.apache.org) (140.211.11.114) by apache.org (qpsmtpd/0.29) with ESMTP; Sat, 06 Sep 2014 04:37:24 +0000 Received: by tyr.zones.apache.org (Postfix, from userid 65534) id A2898A0C133; Sat, 6 Sep 2014 04:37:21 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: arp@apache.org To: common-commits@hadoop.apache.org Date: Sat, 06 Sep 2014 04:37:31 -0000 Message-Id: <9ed133c26048437fa377b0af9256958b@git.apache.org> In-Reply-To: <34ae924c08e44aa688ccd1f75ff3f114@git.apache.org> References: <34ae924c08e44aa688ccd1f75ff3f114@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [12/28] git commit: YARN-2431. NM restart: cgroup is not removed for reacquired containers. Contributed by Jason Lowe YARN-2431. NM restart: cgroup is not removed for reacquired containers. Contributed by Jason Lowe Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3fa5f728 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3fa5f728 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3fa5f728 Branch: refs/heads/HDFS-6581 Commit: 3fa5f728c4d6944302621965d6a0376827af5b51 Parents: b44b2ee Author: Jason Lowe Authored: Thu Sep 4 21:11:27 2014 +0000 Committer: Jason Lowe Committed: Thu Sep 4 21:11:27 2014 +0000 ---------------------------------------------------------------------- hadoop-yarn-project/CHANGES.txt | 3 + .../nodemanager/LinuxContainerExecutor.java | 11 ++++ .../nodemanager/TestLinuxContainerExecutor.java | 60 ++++++++++++++++++++ 3 files changed, 74 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/3fa5f728/hadoop-yarn-project/CHANGES.txt ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index b0e4a01..f6f5c4b 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -278,6 +278,9 @@ Release 2.6.0 - UNRELEASED YARN-2462. TestNodeManagerResync#testBlockNewContainerRequestsOnStartAndResync should have a test timeout (Eric Payne via jlowe) + YARN-2431. NM restart: cgroup is not removed for reacquired containers + (jlowe) + Release 2.5.1 - UNRELEASED INCOMPATIBLE CHANGES http://git-wip-us.apache.org/repos/asf/hadoop/blob/3fa5f728/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java index 804864e..6b8dd28 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java @@ -19,6 +19,7 @@ package org.apache.hadoop.yarn.server.nodemanager; import com.google.common.base.Optional; + import java.io.File; import java.io.IOException; import java.net.InetSocketAddress; @@ -342,6 +343,16 @@ public class LinuxContainerExecutor extends ContainerExecutor { } @Override + public int reacquireContainer(String user, ContainerId containerId) + throws IOException { + try { + return super.reacquireContainer(user, containerId); + } finally { + resourcesHandler.postExecute(containerId); + } + } + + @Override public boolean signalContainer(String user, String pid, Signal signal) throws IOException { http://git-wip-us.apache.org/repos/asf/hadoop/blob/3fa5f728/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutor.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutor.java index a5ec43b..c02212e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutor.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutor.java @@ -30,6 +30,8 @@ import java.io.FileOutputStream; import java.io.IOException; import java.io.PrintWriter; import java.util.HashMap; +import java.util.HashSet; +import java.util.Set; import org.junit.Assert; import org.apache.commons.logging.Log; @@ -42,11 +44,15 @@ import org.apache.hadoop.fs.FileUtil; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; +import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; +import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor.Signal; import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container; +import org.apache.hadoop.yarn.server.nodemanager.util.LCEResourcesHandler; import org.junit.After; import org.junit.Before; import org.junit.Test; @@ -353,4 +359,58 @@ public class TestLinuxContainerExecutor { } } + @Test(timeout=10000) + public void testPostExecuteAfterReacquisition() throws Exception { + // make up some bogus container ID + ApplicationId appId = ApplicationId.newInstance(12345, 67890); + ApplicationAttemptId attemptId = + ApplicationAttemptId.newInstance(appId, 54321); + ContainerId cid = ContainerId.newInstance(attemptId, 9876); + + Configuration conf = new YarnConfiguration(); + conf.setClass(YarnConfiguration.NM_LINUX_CONTAINER_RESOURCES_HANDLER, + TestResourceHandler.class, LCEResourcesHandler.class); + LinuxContainerExecutor lce = new LinuxContainerExecutor(); + lce.setConf(conf); + try { + lce.init(); + } catch (IOException e) { + // expected if LCE isn't setup right, but not necessary for this test + } + lce.reacquireContainer("foouser", cid); + Assert.assertTrue("postExec not called after reacquisition", + TestResourceHandler.postExecContainers.contains(cid)); + } + + private static class TestResourceHandler implements LCEResourcesHandler { + static Set postExecContainers = new HashSet(); + + @Override + public void setConf(Configuration conf) { + } + + @Override + public Configuration getConf() { + return null; + } + + @Override + public void init(LinuxContainerExecutor lce) throws IOException { + } + + @Override + public void preExecute(ContainerId containerId, Resource containerResource) + throws IOException { + } + + @Override + public void postExecute(ContainerId containerId) { + postExecContainers.add(containerId); + } + + @Override + public String getResourcesOption(ContainerId containerId) { + return null; + } + } }