Return-Path: X-Original-To: apmail-hadoop-common-commits-archive@www.apache.org Delivered-To: apmail-hadoop-common-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 1699819EE9 for ; Fri, 29 Apr 2016 20:20:19 +0000 (UTC) Received: (qmail 80373 invoked by uid 500); 29 Apr 2016 20:20:02 -0000 Delivered-To: apmail-hadoop-common-commits-archive@hadoop.apache.org Received: (qmail 79761 invoked by uid 500); 29 Apr 2016 20:20:01 -0000 Mailing-List: contact common-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: common-dev@hadoop.apache.org Delivered-To: mailing list common-commits@hadoop.apache.org Received: (qmail 75067 invoked by uid 99); 29 Apr 2016 20:19:58 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 29 Apr 2016 20:19:58 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 413C5E943F; Fri, 29 Apr 2016 20:19:58 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: arp@apache.org To: common-commits@hadoop.apache.org Date: Fri, 29 Apr 2016 20:20:43 -0000 Message-Id: <5430a5ad51334e04b6fc498f13e40fad@git.apache.org> In-Reply-To: <7bf7ef6c73cb43eba069a697ad6f21ed@git.apache.org> References: <7bf7ef6c73cb43eba069a697ad6f21ed@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [47/50] [abbrv] hadoop git commit: Remove parent's env vars from child processes Remove parent's env vars from child processes Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/9d4d3024 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/9d4d3024 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/9d4d3024 Branch: refs/heads/HDFS-7240 Commit: 9d4d30243b0fc9630da51a2c17b543ef671d035c Parents: af9b000 Author: Robert Kanter Authored: Thu Apr 28 19:24:38 2016 -0700 Committer: Robert Kanter Committed: Fri Apr 29 09:25:51 2016 -0700 ---------------------------------------------------------------------- .../main/java/org/apache/hadoop/util/Shell.java | 23 ++++++++++-- .../java/org/apache/hadoop/util/TestShell.java | 37 ++++++++++++++++++++ .../nodemanager/DefaultContainerExecutor.java | 4 ++- .../nodemanager/DockerContainerExecutor.java | 4 ++- .../nodemanager/LinuxContainerExecutor.java | 2 +- .../privileged/PrivilegedOperationExecutor.java | 8 +++-- .../runtime/DefaultLinuxContainerRuntime.java | 4 +-- .../runtime/DockerLinuxContainerRuntime.java | 4 +-- .../runtime/TestDockerContainerRuntime.java | 2 +- 9 files changed, 75 insertions(+), 13 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/9d4d3024/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Shell.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Shell.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Shell.java index 0af3752..ea8db07 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Shell.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Shell.java @@ -34,6 +34,7 @@ import java.util.concurrent.atomic.AtomicBoolean; import com.google.common.annotations.VisibleForTesting; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.security.alias.AbstractJavaKeyStoreProvider; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -362,6 +363,9 @@ public abstract class Shell { /** If or not script timed out*/ private final AtomicBoolean timedOut = new AtomicBoolean(false); + /** Indicates if the parent env vars should be inherited or not*/ + protected boolean inheritParentEnv = true; + /** * Centralized logic to discover and validate the sanity of the Hadoop * home directory. @@ -854,9 +858,16 @@ public abstract class Shell { timedOut.set(false); completed.set(false); + // Remove all env vars from the Builder to prevent leaking of env vars from + // the parent process. + if (!inheritParentEnv) { + builder.environment().clear(); + } + if (environment != null) { builder.environment().putAll(this.environment); } + if (dir != null) { builder.directory(this.dir); } @@ -1084,6 +1095,11 @@ public abstract class Shell { this(execString, dir, env , 0L); } + public ShellCommandExecutor(String[] execString, File dir, + Map env, long timeout) { + this(execString, dir, env , timeout, true); + } + /** * Create a new instance of the ShellCommandExecutor to execute a command. * @@ -1096,10 +1112,12 @@ public abstract class Shell { * environment is not modified. * @param timeout Specifies the time in milliseconds, after which the * command will be killed and the status marked as timed-out. - * If 0, the command will not be timed out. + * If 0, the command will not be timed out. + * @param inheritParentEnv Indicates if the process should inherit the env + * vars from the parent process or not. */ public ShellCommandExecutor(String[] execString, File dir, - Map env, long timeout) { + Map env, long timeout, boolean inheritParentEnv) { command = execString.clone(); if (dir != null) { setWorkingDirectory(dir); @@ -1108,6 +1126,7 @@ public abstract class Shell { setEnvironment(env); } timeOutInterval = timeout; + this.inheritParentEnv = inheritParentEnv; } /** http://git-wip-us.apache.org/repos/asf/hadoop/blob/9d4d3024/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestShell.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestShell.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestShell.java index f20c140..16ec867 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestShell.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestShell.java @@ -18,6 +18,7 @@ package org.apache.hadoop.util; import org.apache.commons.io.FileUtils; +import org.apache.hadoop.security.alias.AbstractJavaKeyStoreProvider; import org.junit.Assert; import java.io.BufferedReader; @@ -29,6 +30,8 @@ import java.io.PrintWriter; import java.lang.management.ManagementFactory; import java.lang.management.ThreadInfo; import java.lang.management.ThreadMXBean; +import java.util.HashMap; +import java.util.Map; import org.apache.hadoop.fs.FileUtil; import org.apache.hadoop.test.GenericTestUtils; @@ -145,6 +148,40 @@ public class TestShell extends Assert { shellFile.delete(); assertTrue("Script did not timeout" , shexc.isTimedOut()); } + + @Test + public void testEnvVarsWithInheritance() throws Exception { + Assume.assumeFalse(WINDOWS); + testEnvHelper(true); + } + + @Test + public void testEnvVarsWithoutInheritance() throws Exception { + Assume.assumeFalse(WINDOWS); + testEnvHelper(false); + } + + private void testEnvHelper(boolean inheritParentEnv) throws Exception { + Map customEnv = new HashMap<>(); + customEnv.put("AAA" + System.currentTimeMillis(), "AAA"); + customEnv.put("BBB" + System.currentTimeMillis(), "BBB"); + customEnv.put("CCC" + System.currentTimeMillis(), "CCC"); + Shell.ShellCommandExecutor command = new ShellCommandExecutor( + new String[]{"env"}, null, customEnv, 0L, inheritParentEnv); + command.execute(); + String[] varsArr = command.getOutput().split("\n"); + Map vars = new HashMap<>(); + for (String var : varsArr) { + int eqIndex = var.indexOf('='); + vars.put(var.substring(0, eqIndex), var.substring(eqIndex + 1)); + } + Map expectedEnv = new HashMap<>(); + expectedEnv.putAll(customEnv); + if (inheritParentEnv) { + expectedEnv.putAll(System.getenv()); + } + assertEquals(expectedEnv, vars); + } private static int countTimerThreads() { ThreadMXBean threadBean = ManagementFactory.getThreadMXBean(); http://git-wip-us.apache.org/repos/asf/hadoop/blob/9d4d3024/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java index 49398e4..c8048e9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java @@ -284,7 +284,9 @@ public class DefaultContainerExecutor extends ContainerExecutor { return new ShellCommandExecutor( command, wordDir, - environment); + environment, + 0L, + false); } protected LocalWrapperScriptBuilder getLocalWrapperScriptBuilder( http://git-wip-us.apache.org/repos/asf/hadoop/blob/9d4d3024/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DockerContainerExecutor.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DockerContainerExecutor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DockerContainerExecutor.java index b089947..72da236 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DockerContainerExecutor.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DockerContainerExecutor.java @@ -284,7 +284,9 @@ public class DockerContainerExecutor extends ContainerExecutor { shExec = new ShellCommandExecutor( command, new File(containerWorkDir.toUri().getPath()), - container.getLaunchContext().getEnvironment()); // sanitized env + container.getLaunchContext().getEnvironment(), // sanitized env + 0L, + false); if (isContainerActive(containerId)) { shExec.execute(); } else { http://git-wip-us.apache.org/repos/asf/hadoop/blob/9d4d3024/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java index 5a48e09..e46ce56 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java @@ -282,7 +282,7 @@ public class LinuxContainerExecutor extends ContainerExecutor { PrivilegedOperationExecutor.getInstance(conf); privilegedOperationExecutor.executePrivilegedOperation(prefixCommands, - initializeContainerOp, null, null, false); + initializeContainerOp, null, null, false, true); } catch (PrivilegedOperationException e) { int exitCode = e.getExitCode(); http://git-wip-us.apache.org/repos/asf/hadoop/blob/9d4d3024/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/privileged/PrivilegedOperationExecutor.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/privileged/PrivilegedOperationExecutor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/privileged/PrivilegedOperationExecutor.java index 7370daa..f865c14 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/privileged/PrivilegedOperationExecutor.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/privileged/PrivilegedOperationExecutor.java @@ -133,18 +133,19 @@ public class PrivilegedOperationExecutor { * @param workingDir (optional) working directory for execution * @param env (optional) env of the command will include specified vars * @param grabOutput return (possibly large) shell command output + * @param inheritParentEnv inherit the env vars from the parent process * @return stdout contents from shell executor - useful for some privileged * operations - e.g --tc_read * @throws org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.privileged.PrivilegedOperationException */ public String executePrivilegedOperation(List prefixCommands, PrivilegedOperation operation, File workingDir, - Map env, boolean grabOutput) + Map env, boolean grabOutput, boolean inheritParentEnv) throws PrivilegedOperationException { String[] fullCommandArray = getPrivilegedOperationExecutionCommand (prefixCommands, operation); ShellCommandExecutor exec = new ShellCommandExecutor(fullCommandArray, - workingDir, env); + workingDir, env, 0L, inheritParentEnv); try { exec.execute(); @@ -199,7 +200,8 @@ public class PrivilegedOperationExecutor { */ public String executePrivilegedOperation(PrivilegedOperation operation, boolean grabOutput) throws PrivilegedOperationException { - return executePrivilegedOperation(null, operation, null, null, grabOutput); + return executePrivilegedOperation(null, operation, null, null, grabOutput, + true); } //Utility functions for squashing together operations in supported ways http://git-wip-us.apache.org/repos/asf/hadoop/blob/9d4d3024/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DefaultLinuxContainerRuntime.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DefaultLinuxContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DefaultLinuxContainerRuntime.java index 3862b92..e78f460 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DefaultLinuxContainerRuntime.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DefaultLinuxContainerRuntime.java @@ -102,7 +102,7 @@ public class DefaultLinuxContainerRuntime implements LinuxContainerRuntime { try { privilegedOperationExecutor.executePrivilegedOperation(prefixCommands, launchOp, null, container.getLaunchContext().getEnvironment(), - false); + false, false); } catch (PrivilegedOperationException e) { LOG.warn("Launch container failed. Exception: ", e); @@ -134,7 +134,7 @@ public class DefaultLinuxContainerRuntime implements LinuxContainerRuntime { executor.executePrivilegedOperation(null, signalOp, null, container.getLaunchContext().getEnvironment(), - false); + false, true); } catch (PrivilegedOperationException e) { //Don't log the failure here. Some kinds of signaling failures are // acceptable. Let the calling executor decide what to do. http://git-wip-us.apache.org/repos/asf/hadoop/blob/9d4d3024/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java index c66189d..681cae2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java @@ -331,7 +331,7 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime { try { privilegedOperationExecutor.executePrivilegedOperation(null, launchOp, null, container.getLaunchContext().getEnvironment(), - false); + false, false); } catch (PrivilegedOperationException e) { LOG.warn("Launch container failed. Exception: ", e); @@ -360,7 +360,7 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime { executor.executePrivilegedOperation(null, signalOp, null, container.getLaunchContext().getEnvironment(), - false); + false, true); } catch (PrivilegedOperationException e) { LOG.warn("Signal container failed. Exception: ", e); http://git-wip-us.apache.org/repos/asf/hadoop/blob/9d4d3024/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java index e05719c..d1bdabe 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java @@ -179,7 +179,7 @@ public class TestDockerContainerRuntime { // warning annotation on the entire method verify(mockExecutor, times(1)) .executePrivilegedOperation(anyList(), opCaptor.capture(), any( - File.class), any(Map.class), eq(false)); + File.class), any(Map.class), eq(false), eq(false)); PrivilegedOperation op = opCaptor.getValue(); --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org For additional commands, e-mail: common-commits-help@hadoop.apache.org