Return-Path: X-Original-To: apmail-hadoop-common-commits-archive@www.apache.org Delivered-To: apmail-hadoop-common-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 772D018EB4 for ; Wed, 29 Jul 2015 15:17:20 +0000 (UTC) Received: (qmail 28646 invoked by uid 500); 29 Jul 2015 15:17:20 -0000 Delivered-To: apmail-hadoop-common-commits-archive@hadoop.apache.org Received: (qmail 28582 invoked by uid 500); 29 Jul 2015 15:17:20 -0000 Mailing-List: contact common-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: common-dev@hadoop.apache.org Delivered-To: mailing list common-commits@hadoop.apache.org Received: (qmail 28573 invoked by uid 99); 29 Jul 2015 15:17:20 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 29 Jul 2015 15:17:20 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 24AB6DFAF5; Wed, 29 Jul 2015 15:17:20 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: jlowe@apache.org To: common-commits@hadoop.apache.org Message-Id: <685bf85a51844a95a155f3e0924f4d5c@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: hadoop git commit: YARN-3950. Add unique SHELL_ID environment variable to DistributedShell. Contributed by Robert Kanter Date: Wed, 29 Jul 2015 15:17:20 +0000 (UTC) Repository: hadoop Updated Branches: refs/heads/trunk 6374ee0db -> 2b2bd9214 YARN-3950. Add unique SHELL_ID environment variable to DistributedShell. Contributed by Robert Kanter Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/2b2bd921 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/2b2bd921 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/2b2bd921 Branch: refs/heads/trunk Commit: 2b2bd9214604bc2e14e41e08d30bf86f512151bd Parents: 6374ee0 Author: Jason Lowe Authored: Wed Jul 29 15:16:40 2015 +0000 Committer: Jason Lowe Committed: Wed Jul 29 15:16:40 2015 +0000 ---------------------------------------------------------------------- hadoop-yarn-project/CHANGES.txt | 3 ++ .../distributedshell/ApplicationMaster.java | 31 +++++++++++++++----- .../distributedshell/TestDSAppMaster.java | 11 ++++++- 3 files changed, 36 insertions(+), 9 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2bd921/hadoop-yarn-project/CHANGES.txt ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index b4666e8..be6a50c 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -358,6 +358,9 @@ Release 2.8.0 - UNRELEASED YARN-3026. Move application-specific container allocation logic from LeafQueue to FiCaSchedulerApp. (Wangda Tan via jianhe) + YARN-3950. Add unique SHELL_ID environment variable to DistributedShell + (Robert Kanter via jlowe) + OPTIMIZATIONS YARN-3339. TestDockerContainerExecutor should pull a single image and not http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2bd921/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java index b28c0c9..5d2d6c2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java @@ -182,6 +182,8 @@ public class ApplicationMaster { DS_APP_ATTEMPT, DS_CONTAINER } + private static final String YARN_SHELL_ID = "YARN_SHELL_ID"; + // Configuration private Configuration conf; @@ -279,6 +281,8 @@ public class ApplicationMaster { private final String linux_bash_command = "bash"; private final String windows_command = "cmd /c"; + private int yarnShellIdCounter = 1; + @VisibleForTesting protected final Set launchedContainers = Collections.newSetFromMap(new ConcurrentHashMap()); @@ -803,8 +807,11 @@ public class ApplicationMaster { + allocatedContainers.size()); numAllocatedContainers.addAndGet(allocatedContainers.size()); for (Container allocatedContainer : allocatedContainers) { + String yarnShellId = Integer.toString(yarnShellIdCounter); + yarnShellIdCounter++; LOG.info("Launching shell command on a new container." + ", containerId=" + allocatedContainer.getId() + + ", yarnShellId=" + yarnShellId + ", containerNode=" + allocatedContainer.getNodeId().getHost() + ":" + allocatedContainer.getNodeId().getPort() + ", containerNodeURI=" + allocatedContainer.getNodeHttpAddress() @@ -815,7 +822,8 @@ public class ApplicationMaster { // + ", containerToken" // +allocatedContainer.getContainerToken().getIdentifier().toString()); - Thread launchThread = createLaunchContainerThread(allocatedContainer); + Thread launchThread = createLaunchContainerThread(allocatedContainer, + yarnShellId); // launch and start the container on a separate thread to keep // the main thread unblocked @@ -927,7 +935,8 @@ public class ApplicationMaster { private class LaunchContainerRunnable implements Runnable { // Allocated container - Container container; + private Container container; + private String shellId; NMCallbackHandler containerListener; @@ -935,10 +944,11 @@ public class ApplicationMaster { * @param lcontainer Allocated container * @param containerListener Callback handler of the container */ - public LaunchContainerRunnable( - Container lcontainer, NMCallbackHandler containerListener) { + public LaunchContainerRunnable(Container lcontainer, + NMCallbackHandler containerListener, String shellId) { this.container = lcontainer; this.containerListener = containerListener; + this.shellId = shellId; } @Override @@ -949,7 +959,7 @@ public class ApplicationMaster { */ public void run() { LOG.info("Setting up container launch container for containerid=" - + container.getId()); + + container.getId() + " with shellid=" + shellId); // Set the local resources Map localResources = new HashMap(); @@ -1038,8 +1048,11 @@ public class ApplicationMaster { // download anyfiles in the distributed file-system. The tokens are // otherwise also useful in cases, for e.g., when one is running a // "hadoop dfs" command inside the distributed shell. + Map myShellEnv = new HashMap(shellEnv); + myShellEnv.put(YARN_SHELL_ID, shellId); ContainerLaunchContext ctx = ContainerLaunchContext.newInstance( - localResources, shellEnv, commands, null, allTokens.duplicate(), null); + localResources, myShellEnv, commands, null, allTokens.duplicate(), + null); containerListener.addContainer(container.getId(), container); nmClientAsync.startContainerAsync(container, ctx); } @@ -1189,9 +1202,11 @@ public class ApplicationMaster { } @VisibleForTesting - Thread createLaunchContainerThread(Container allocatedContainer) { + Thread createLaunchContainerThread(Container allocatedContainer, + String shellId) { LaunchContainerRunnable runnableLaunchContainer = - new LaunchContainerRunnable(allocatedContainer, containerListener); + new LaunchContainerRunnable(allocatedContainer, containerListener, + shellId); return new Thread(runnableLaunchContainer); } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2bd921/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDSAppMaster.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDSAppMaster.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDSAppMaster.java index 0fed14d..2789d04 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDSAppMaster.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDSAppMaster.java @@ -41,6 +41,7 @@ import org.mockito.Mockito; import java.io.IOException; import java.util.ArrayList; +import java.util.Arrays; import java.util.List; /** @@ -51,11 +52,14 @@ public class TestDSAppMaster { static class TestAppMaster extends ApplicationMaster { private int threadsLaunched = 0; + public List yarnShellIds = new ArrayList(); @Override - protected Thread createLaunchContainerThread(Container allocatedContainer) { + protected Thread createLaunchContainerThread(Container allocatedContainer, + String shellId) { threadsLaunched++; launchedContainers.add(allocatedContainer.getId()); + yarnShellIds.add(shellId); return new Thread(); } @@ -101,6 +105,8 @@ public class TestDSAppMaster { Mockito.verifyZeroInteractions(mockClient); Assert.assertEquals("Incorrect number of threads launched", 1, master.threadsLaunched); + Assert.assertEquals("Incorrect YARN Shell IDs", + Arrays.asList("1"), master.yarnShellIds); // now send 3 extra containers containers.clear(); @@ -117,6 +123,9 @@ public class TestDSAppMaster { Assert.assertEquals("Incorrect number of threads launched", 4, master.threadsLaunched); + Assert.assertEquals("Incorrect YARN Shell IDs", + Arrays.asList("1", "2", "3", "4"), master.yarnShellIds); + // make sure we handle completion events correctly List status = new ArrayList<>(); status.add(generateContainerStatus(id1, ContainerExitStatus.SUCCESS));