Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id 4D3C8200CF3 for ; Wed, 30 Aug 2017 00:29:39 +0200 (CEST) Received: by cust-asf.ponee.io (Postfix) id 4A012167BC7; Tue, 29 Aug 2017 22:29:39 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id 68166167BC6 for ; Wed, 30 Aug 2017 00:29:38 +0200 (CEST) Received: (qmail 49431 invoked by uid 500); 29 Aug 2017 22:29:36 -0000 Mailing-List: contact common-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Delivered-To: mailing list common-commits@hadoop.apache.org Received: (qmail 49422 invoked by uid 99); 29 Aug 2017 22:29:36 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 29 Aug 2017 22:29:36 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id AD816E7DFA; Tue, 29 Aug 2017 22:29:35 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: wangda@apache.org To: common-commits@hadoop.apache.org Message-Id: <26aac8af0eac4ddb9e54f6505e981d9a@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: hadoop git commit: YARN-5219. When an export var command fails in launch_container.sh, the full container launch should fail. (Sunil G via wangda) Date: Tue, 29 Aug 2017 22:29:35 +0000 (UTC) archived-at: Tue, 29 Aug 2017 22:29:39 -0000 Repository: hadoop Updated Branches: refs/heads/trunk ad45d1999 -> f59332b97 YARN-5219. When an export var command fails in launch_container.sh, the full container launch should fail. (Sunil G via wangda) Change-Id: Iaa6b978bb89482e9d1d77ba57f4adfdc48e39a3c Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f59332b9 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f59332b9 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f59332b9 Branch: refs/heads/trunk Commit: f59332b97b9a57e3cf1dcdeb47d7838d287100eb Parents: ad45d19 Author: Wangda Tan Authored: Tue Aug 29 15:29:22 2017 -0700 Committer: Wangda Tan Committed: Tue Aug 29 15:29:22 2017 -0700 ---------------------------------------------------------------------- .../server/nodemanager/ContainerExecutor.java | 3 + .../launcher/ContainerLaunch.java | 9 ++ .../launcher/TestContainerLaunch.java | 105 +++++++++++++++++++ 3 files changed, 117 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/f59332b9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java index 9767fb9..b6fb4ec 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java @@ -337,6 +337,9 @@ public abstract class ContainerExecutor implements Configurable { whitelist.add(param); } + // Add "set -o pipefail -e" to validate launch_container script. + sb.setExitOnFailure(); + if (environment != null) { for (Map.Entry env : environment.entrySet()) { if (!whitelist.contains(env.getKey())) { http://git-wip-us.apache.org/repos/asf/hadoop/blob/f59332b9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java index 3f50c3f..50443f3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java @@ -931,6 +931,10 @@ public class ContainerLaunch implements Callable { sb.append(LINE_SEPARATOR); } + public void setExitOnFailure() { + // Dummy implementation + } + protected abstract void link(Path src, Path dst) throws IOException; protected abstract void mkdir(Path path) throws IOException; @@ -1008,6 +1012,11 @@ public class ContainerLaunch implements Callable { output.toString(), "\""); line("find -L . -maxdepth 5 -type l -ls 1>>\"", output.toString(), "\""); } + + @Override + public void setExitOnFailure() { + line("set -o pipefail -e"); + } } private static final class WindowsShellScriptBuilder http://git-wip-us.apache.org/repos/asf/hadoop/blob/f59332b9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java index e71ce75..085b60f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java @@ -39,6 +39,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.HashMap; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.StringTokenizer; @@ -1529,4 +1530,108 @@ public class TestContainerLaunch extends BaseContainerManagerTest { verify(updaterNoCall, never()).reportException(any()); } + + /* + * ${foo.version} is substituted to suffix a specific version number + */ + @Test + public void testInvalidEnvVariableSubstitutionType1() throws IOException { + Map env = new HashMap(); + // invalid env + env.put("testVar", "version${foo.version}"); + validateShellExecutorForDifferentEnvs(env); + } + + /* + * Multiple paths are substituted in a path variable + */ + @Test + public void testInvalidEnvVariableSubstitutionType2() throws IOException { + Map env = new HashMap(); + // invalid env + env.put("testPath", "/abc:/${foo.path}:/$bar"); + validateShellExecutorForDifferentEnvs(env); + } + + private void validateShellExecutorForDifferentEnvs(Map env) + throws IOException { + File shellFile = null; + try { + shellFile = Shell.appendScriptExtension(tmpDir, "hello"); + Map> resources = new HashMap>(); + FileOutputStream fos = new FileOutputStream(shellFile); + FileUtil.setExecutable(shellFile, true); + + List commands = new ArrayList(); + DefaultContainerExecutor executor = new DefaultContainerExecutor(); + executor.setConf(new Configuration()); + executor.writeLaunchEnv(fos, env, resources, commands, + new Path(localLogDir.getAbsolutePath()), user); + fos.flush(); + fos.close(); + + // It is supposed that LANG is set as C. + Map cmdEnv = new HashMap(); + cmdEnv.put("LANG", "C"); + Shell.ShellCommandExecutor shexc = new Shell.ShellCommandExecutor( + new String[] { shellFile.getAbsolutePath() }, tmpDir, cmdEnv); + try { + shexc.execute(); + Assert.fail("Should catch exception"); + } catch (ExitCodeException e) { + Assert.assertTrue(shexc.getExitCode() != 0); + } + } finally { + // cleanup + if (shellFile != null && shellFile.exists()) { + shellFile.delete(); + } + } + } + + @Test + public void testValidEnvVariableSubstitution() throws IOException { + File shellFile = null; + try { + shellFile = Shell.appendScriptExtension(tmpDir, "hello"); + Map> resources = + new HashMap>(); + FileOutputStream fos = new FileOutputStream(shellFile); + FileUtil.setExecutable(shellFile, true); + + Map env = new LinkedHashMap(); + // valid env + env.put( + "foo", "2.4.6" ); + env.put( + "testVar", "version${foo}" ); + List commands = new ArrayList(); + DefaultContainerExecutor executor = new DefaultContainerExecutor(); + executor.setConf(new Configuration()); + executor.writeLaunchEnv(fos, env, resources, commands, + new Path(localLogDir.getAbsolutePath()), user); + fos.flush(); + fos.close(); + + // It is supposed that LANG is set as C. + Map cmdEnv = new HashMap(); + cmdEnv.put("LANG", "C"); + Shell.ShellCommandExecutor shexc + = new Shell.ShellCommandExecutor(new String[]{shellFile.getAbsolutePath()}, + tmpDir, cmdEnv); + try { + shexc.execute(); + } catch(ExitCodeException e){ + Assert.fail("Should not catch exception"); + } + Assert.assertTrue(shexc.getExitCode() == 0); + } + finally { + // cleanup + if (shellFile != null + && shellFile.exists()) { + shellFile.delete(); + } + } + } } --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org For additional commands, e-mail: common-commits-help@hadoop.apache.org