Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id 926BC200D12 for ; Fri, 22 Sep 2017 20:27:04 +0200 (CEST) Received: by cust-asf.ponee.io (Postfix) id 911391609BE; Fri, 22 Sep 2017 18:27:04 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id 8EC1C1609E5 for ; Fri, 22 Sep 2017 20:27:03 +0200 (CEST) Received: (qmail 95770 invoked by uid 500); 22 Sep 2017 18:27:01 -0000 Mailing-List: contact common-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Delivered-To: mailing list common-commits@hadoop.apache.org Received: (qmail 95488 invoked by uid 99); 22 Sep 2017 18:27:01 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 22 Sep 2017 18:27:01 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 89271F5AC2; Fri, 22 Sep 2017 18:27:00 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: jhung@apache.org To: common-commits@hadoop.apache.org Date: Fri, 22 Sep 2017 18:27:09 -0000 Message-Id: <2dbed64655a841e99bd0f22a743655f1@git.apache.org> In-Reply-To: <53e2fc90c3594a03b5a27004d84fcb29@git.apache.org> References: <53e2fc90c3594a03b5a27004d84fcb29@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [10/30] hadoop git commit: YARN-7034. DefaultLinuxContainerRuntime and DockerLinuxContainerRuntime sends client environment variables to container-executor. Contributed by Miklos Szegedi. archived-at: Fri, 22 Sep 2017 18:27:04 -0000 YARN-7034. DefaultLinuxContainerRuntime and DockerLinuxContainerRuntime sends client environment variables to container-executor. Contributed by Miklos Szegedi. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e5e1851d Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e5e1851d Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e5e1851d Branch: refs/heads/YARN-5734 Commit: e5e1851d803bf8d8b96fec1b5c0058014e9329d0 Parents: a92ef03 Author: Junping Du Authored: Thu Sep 21 14:01:16 2017 -0700 Committer: Junping Du Committed: Thu Sep 21 14:01:16 2017 -0700 ---------------------------------------------------------------------- .../privileged/PrivilegedOperationExecutor.java | 2 +- .../runtime/DefaultLinuxContainerRuntime.java | 8 +-- .../runtime/DockerLinuxContainerRuntime.java | 9 +-- .../TestLinuxContainerExecutorWithMocks.java | 67 +++++++++++++++++++- .../runtime/TestDockerContainerRuntime.java | 3 +- 5 files changed, 73 insertions(+), 16 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/e5e1851d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/privileged/PrivilegedOperationExecutor.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/privileged/PrivilegedOperationExecutor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/privileged/PrivilegedOperationExecutor.java index 5a3ce74..9f13a5e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/privileged/PrivilegedOperationExecutor.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/privileged/PrivilegedOperationExecutor.java @@ -204,7 +204,7 @@ public class PrivilegedOperationExecutor { public String executePrivilegedOperation(PrivilegedOperation operation, boolean grabOutput) throws PrivilegedOperationException { return executePrivilegedOperation(null, operation, null, null, grabOutput, - true); + false); } //Utility functions for squashing together operations in supported ways http://git-wip-us.apache.org/repos/asf/hadoop/blob/e5e1851d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DefaultLinuxContainerRuntime.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DefaultLinuxContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DefaultLinuxContainerRuntime.java index d09e4a1..6c3ae85 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DefaultLinuxContainerRuntime.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DefaultLinuxContainerRuntime.java @@ -80,7 +80,6 @@ public class DefaultLinuxContainerRuntime implements LinuxContainerRuntime { @Override public void launchContainer(ContainerRuntimeContext ctx) throws ContainerExecutionException { - Container container = ctx.getContainer(); PrivilegedOperation launchOp = new PrivilegedOperation( PrivilegedOperation.OperationType.LAUNCH_CONTAINER); @@ -116,8 +115,7 @@ public class DefaultLinuxContainerRuntime implements LinuxContainerRuntime { try { privilegedOperationExecutor.executePrivilegedOperation(prefixCommands, - launchOp, null, container.getLaunchContext().getEnvironment(), - false, false); + launchOp, null, null, false, false); } catch (PrivilegedOperationException e) { LOG.warn("Launch container failed. Exception: ", e); @@ -129,7 +127,6 @@ public class DefaultLinuxContainerRuntime implements LinuxContainerRuntime { @Override public void signalContainer(ContainerRuntimeContext ctx) throws ContainerExecutionException { - Container container = ctx.getContainer(); PrivilegedOperation signalOp = new PrivilegedOperation( PrivilegedOperation.OperationType.SIGNAL_CONTAINER); @@ -148,8 +145,7 @@ public class DefaultLinuxContainerRuntime implements LinuxContainerRuntime { .getInstance(conf); executor.executePrivilegedOperation(null, - signalOp, null, container.getLaunchContext().getEnvironment(), - false, true); + signalOp, null, null, false, false); } catch (PrivilegedOperationException e) { //Don't log the failure here. Some kinds of signaling failures are // acceptable. Let the calling executor decide what to do. http://git-wip-us.apache.org/repos/asf/hadoop/blob/e5e1851d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java index 1ccd365..6491a99 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java @@ -549,8 +549,7 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime { try { privilegedOperationExecutor.executePrivilegedOperation(null, - launchOp, null, container.getLaunchContext().getEnvironment(), - false, false); + launchOp, null, null, false, false); } catch (PrivilegedOperationException e) { LOG.warn("Launch container failed. Exception: ", e); LOG.info("Docker command used: " + runCommand.getCommandWithArguments()); @@ -563,7 +562,6 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime { @Override public void signalContainer(ContainerRuntimeContext ctx) throws ContainerExecutionException { - Container container = ctx.getContainer(); ContainerExecutor.Signal signal = ctx.getExecutionAttribute(SIGNAL); PrivilegedOperation privOp = null; @@ -594,8 +592,7 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime { try { privilegedOperationExecutor.executePrivilegedOperation(null, - privOp, null, container.getLaunchContext().getEnvironment(), - false, false); + privOp, null, null, false, false); } catch (PrivilegedOperationException e) { throw new ContainerExecutionException("Signal container failed", e .getExitCode(), e.getOutput(), e.getErrorOutput()); @@ -623,7 +620,7 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime { privOp.appendArgs(commandFile); String output = privilegedOperationExecutor .executePrivilegedOperation(null, privOp, null, - container.getLaunchContext().getEnvironment(), true, false); + null, true, false); LOG.info("Docker inspect output for " + containerId + ": " + output); int index = output.lastIndexOf(','); if (index == -1) { http://git-wip-us.apache.org/repos/asf/hadoop/blob/e5e1851d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutorWithMocks.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutorWithMocks.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutorWithMocks.java index 6ca96db..9fa22d8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutorWithMocks.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutorWithMocks.java @@ -72,10 +72,16 @@ import org.junit.Assert; import org.junit.After; import org.junit.Before; import org.junit.Test; +import org.mockito.ArgumentCaptor; +import org.mockito.Mockito; import org.mockito.invocation.InvocationOnMock; import org.mockito.stubbing.Answer; import static java.nio.file.StandardCopyOption.REPLACE_EXISTING; +import static org.mockito.Matchers.anyListOf; +import static org.mockito.Matchers.eq; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; public class TestLinuxContainerExecutorWithMocks { @@ -91,6 +97,8 @@ public class TestLinuxContainerExecutorWithMocks { private String tmpMockExecutor; private LinuxContainerExecutor mockExec = null; + private LinuxContainerExecutor mockExecMockRuntime = null; + private PrivilegedOperationExecutor mockPrivilegedExec; private final File mockParamFile = new File("./params.txt"); private LocalDirsHandlerService dirsHandler; @@ -140,22 +148,28 @@ public class TestLinuxContainerExecutorWithMocks { Configuration conf = new YarnConfiguration(); LinuxContainerRuntime linuxContainerRuntime; + LinuxContainerRuntime mockLinuxContainerRuntime; setupMockExecutor(MOCK_EXECUTOR, conf); linuxContainerRuntime = new DefaultLinuxContainerRuntime( PrivilegedOperationExecutor.getInstance(conf)); + mockPrivilegedExec = Mockito.mock(PrivilegedOperationExecutor.class); + mockLinuxContainerRuntime = new DefaultLinuxContainerRuntime( + mockPrivilegedExec); dirsHandler = new LocalDirsHandlerService(); dirsHandler.init(conf); linuxContainerRuntime.initialize(conf); mockExec = new LinuxContainerExecutor(linuxContainerRuntime); mockExec.setConf(conf); + mockExecMockRuntime = new LinuxContainerExecutor(mockLinuxContainerRuntime); + mockExecMockRuntime.setConf(conf); } @After public void tearDown() { deleteMockParamFile(); } - + @Test public void testContainerLaunch() throws IOException, ConfigurationException { @@ -168,7 +182,7 @@ public class TestLinuxContainerExecutorWithMocks { ContainerId cId = mock(ContainerId.class); ContainerLaunchContext context = mock(ContainerLaunchContext.class); HashMap env = new HashMap(); - + when(container.getContainerId()).thenReturn(cId); when(container.getLaunchContext()).thenReturn(context); @@ -605,4 +619,53 @@ public class TestLinuxContainerExecutorWithMocks { e.getMessage().contains("exit code")); } } + + @Test + public void testContainerLaunchEnvironment() + throws IOException, ConfigurationException, + PrivilegedOperationException { + String appSubmitter = "nobody"; + String appId = "APP_ID"; + String containerId = "CONTAINER_ID"; + Container container = mock(Container.class); + ContainerId cId = mock(ContainerId.class); + ContainerLaunchContext context = mock(ContainerLaunchContext.class); + HashMap env = new HashMap(); + env.put("FROM_CLIENT", "1"); + + when(container.getContainerId()).thenReturn(cId); + when(container.getLaunchContext()).thenReturn(context); + + when(cId.toString()).thenReturn(containerId); + + when(context.getEnvironment()).thenReturn(env); + + Path scriptPath = new Path("file:///bin/echo"); + Path tokensPath = new Path("file:///dev/null"); + Path workDir = new Path("/tmp"); + Path pidFile = new Path(workDir, "pid.txt"); + + mockExecMockRuntime.activateContainer(cId, pidFile); + mockExecMockRuntime.launchContainer(new ContainerStartContext.Builder() + .setContainer(container) + .setNmPrivateContainerScriptPath(scriptPath) + .setNmPrivateTokensPath(tokensPath) + .setUser(appSubmitter) + .setAppId(appId) + .setContainerWorkDir(workDir) + .setLocalDirs(dirsHandler.getLocalDirs()) + .setLogDirs(dirsHandler.getLogDirs()) + .setFilecacheDirs(new ArrayList<>()) + .setUserLocalDirs(new ArrayList<>()) + .setContainerLocalDirs(new ArrayList<>()) + .setContainerLogDirs(new ArrayList<>()) + .build()); + ArgumentCaptor opCaptor = ArgumentCaptor.forClass( + PrivilegedOperation.class); + // Verify that + verify(mockPrivilegedExec, times(1)) + .executePrivilegedOperation(anyListOf( + String.class), opCaptor.capture(), any( + File.class), eq(null), eq(false), eq(false)); + } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/e5e1851d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java index 3f4bb5c..0e314bf 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java @@ -116,6 +116,7 @@ public class TestDockerContainerRuntime { cId = mock(ContainerId.class); context = mock(ContainerLaunchContext.class); env = new HashMap(); + env.put("FROM_CLIENT", "1"); image = "busybox:latest"; env.put(DockerLinuxContainerRuntime.ENV_DOCKER_CONTAINER_IMAGE, image); @@ -204,7 +205,7 @@ public class TestDockerContainerRuntime { // warning annotation on the entire method verify(mockExecutor, times(1)) .executePrivilegedOperation(anyList(), opCaptor.capture(), any( - File.class), any(Map.class), eq(false), eq(false)); + File.class), eq(null), eq(false), eq(false)); //verification completed. we need to isolate specific invications. // hence, reset mock here --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org For additional commands, e-mail: common-commits-help@hadoop.apache.org