Return-Path: X-Original-To: apmail-hadoop-common-commits-archive@www.apache.org Delivered-To: apmail-hadoop-common-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id BF6F317A8F for ; Fri, 22 Jan 2016 09:18:33 +0000 (UTC) Received: (qmail 81625 invoked by uid 500); 22 Jan 2016 09:18:33 -0000 Delivered-To: apmail-hadoop-common-commits-archive@hadoop.apache.org Received: (qmail 81561 invoked by uid 500); 22 Jan 2016 09:18:33 -0000 Mailing-List: contact common-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: common-dev@hadoop.apache.org Delivered-To: mailing list common-commits@hadoop.apache.org Received: (qmail 81552 invoked by uid 99); 22 Jan 2016 09:18:33 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 22 Jan 2016 09:18:33 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 56A04E0098; Fri, 22 Jan 2016 09:18:33 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: vvasudev@apache.org To: common-commits@hadoop.apache.org Date: Fri, 22 Jan 2016 09:18:33 -0000 Message-Id: <09a11b9628e747c1923b02cd2e2784ff@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [1/2] hadoop git commit: YARN-4578. Directories that are mounted in docker containers need to be more restrictive/container-specific. Contributed by Sidharta Seethana. Repository: hadoop Updated Branches: refs/heads/branch-2 73ef77c12 -> 85561207f refs/heads/trunk ae9c61ff0 -> b41a7e89d YARN-4578. Directories that are mounted in docker containers need to be more restrictive/container-specific. Contributed by Sidharta Seethana. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b41a7e89 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b41a7e89 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b41a7e89 Branch: refs/heads/trunk Commit: b41a7e89d1eb8650975ac7092532ed9563ac60f2 Parents: ae9c61f Author: Varun Vasudev Authored: Fri Jan 22 14:43:14 2016 +0530 Committer: Varun Vasudev Committed: Fri Jan 22 14:43:14 2016 +0530 ---------------------------------------------------------------------- hadoop-yarn-project/CHANGES.txt | 3 +++ .../nodemanager/LinuxContainerExecutor.java | 4 ++++ .../launcher/ContainerLaunch.java | 11 +++++++++ .../runtime/DockerLinuxContainerRuntime.java | 10 ++++++-- .../runtime/LinuxContainerRuntimeConstants.java | 4 ++++ .../executor/ContainerStartContext.java | 24 ++++++++++++++++++++ .../runtime/TestDockerContainerRuntime.java | 12 ++++++++-- 7 files changed, 64 insertions(+), 4 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/b41a7e89/hadoop-yarn-project/CHANGES.txt ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index a7a63b1..77b4eb4 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -140,6 +140,9 @@ Release 2.9.0 - UNRELEASED YARN-4584. RM startup failure when AM attempts greater than max-attempts. (Bibin A Chundatt via rohithsharmaks) + YARN-4578. Directories that are mounted in docker containers need to be more + restrictive/container-specific. (Sidharta Seethana via vvasudev) + Release 2.8.0 - UNRELEASED INCOMPATIBLE CHANGES http://git-wip-us.apache.org/repos/asf/hadoop/blob/b41a7e89/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java index 8549230..ff82e97 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java @@ -291,6 +291,8 @@ public class LinuxContainerExecutor extends ContainerExecutor { Path containerWorkDir = ctx.getContainerWorkDir(); List localDirs = ctx.getLocalDirs(); List logDirs = ctx.getLogDirs(); + List containerLocalDirs = ctx.getContainerLocalDirs(); + List containerLogDirs = ctx.getContainerLogDirs(); Map> localizedResources = ctx.getLocalizedResources(); verifyUsernamePattern(user); @@ -375,6 +377,8 @@ public class LinuxContainerExecutor extends ContainerExecutor { .setExecutionAttribute(PID_FILE_PATH, pidFilePath) .setExecutionAttribute(LOCAL_DIRS, localDirs) .setExecutionAttribute(LOG_DIRS, logDirs) + .setExecutionAttribute(CONTAINER_LOCAL_DIRS, containerLocalDirs) + .setExecutionAttribute(CONTAINER_LOG_DIRS, containerLogDirs) .setExecutionAttribute(RESOURCES_OPTIONS, resourcesOptions); if (tcCommandFile != null) { http://git-wip-us.apache.org/repos/asf/hadoop/blob/b41a7e89/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java index 6371b21..64689dd 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java @@ -253,6 +253,7 @@ public class ContainerLaunch implements Callable { + dirsHandler.getDisksHealthReport(false)); } + List containerLocalDirs = new ArrayList<>(localDirs.size()); try { // /////////// Write out the container-script in the nmPrivate space. List appDirs = new ArrayList(localDirs.size()); @@ -261,6 +262,14 @@ public class ContainerLaunch implements Callable { Path userdir = new Path(usersdir, user); Path appsdir = new Path(userdir, ContainerLocalizer.APPCACHE); appDirs.add(new Path(appsdir, appIdStr)); + + String containerLocalDir = localDir + Path.SEPARATOR + + ContainerLocalizer.USERCACHE + Path.SEPARATOR + user + + Path.SEPARATOR + + ContainerLocalizer.APPCACHE + Path.SEPARATOR + appIdStr + + Path.SEPARATOR; + + containerLocalDirs.add(containerLocalDir); } containerScriptOutStream = lfs.create(nmPrivateContainerScriptPath, @@ -317,6 +326,8 @@ public class ContainerLaunch implements Callable { .setContainerWorkDir(containerWorkDir) .setLocalDirs(localDirs) .setLogDirs(logDirs) + .setContainerLocalDirs(containerLocalDirs) + .setContainerLogDirs(containerLogDirs) .build()); } } catch (Throwable e) { http://git-wip-us.apache.org/repos/asf/hadoop/blob/b41a7e89/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java index f8aae81..2dee663 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java @@ -223,6 +223,12 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime { List localDirs = ctx.getExecutionAttribute(LOCAL_DIRS); @SuppressWarnings("unchecked") List logDirs = ctx.getExecutionAttribute(LOG_DIRS); + @SuppressWarnings("unchecked") + List containerLocalDirs = ctx.getExecutionAttribute( + CONTAINER_LOCAL_DIRS); + @SuppressWarnings("unchecked") + List containerLogDirs = ctx.getExecutionAttribute( + CONTAINER_LOG_DIRS); Set capabilities = new HashSet<>(Arrays.asList(conf.getStrings( YarnConfiguration.NM_DOCKER_CONTAINER_CAPABILITIES, YarnConfiguration.DEFAULT_NM_DOCKER_CONTAINER_CAPABILITIES))); @@ -235,10 +241,10 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime { .setNetworkType("host") .setCapabilities(capabilities) .addMountLocation("/etc/passwd", "/etc/password:ro"); - List allDirs = new ArrayList<>(localDirs); + List allDirs = new ArrayList<>(containerLocalDirs); allDirs.add(containerWorkDir.toString()); - allDirs.addAll(logDirs); + allDirs.addAll(containerLogDirs); for (String dir: allDirs) { runCommand.addMountLocation(dir, dir); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/b41a7e89/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/LinuxContainerRuntimeConstants.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/LinuxContainerRuntimeConstants.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/LinuxContainerRuntimeConstants.java index d2069a9..96ff40d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/LinuxContainerRuntimeConstants.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/LinuxContainerRuntimeConstants.java @@ -55,6 +55,10 @@ public final class LinuxContainerRuntimeConstants { List.class, "local_dirs"); public static final Attribute LOG_DIRS = Attribute.attribute( List.class, "log_dirs"); + public static final Attribute CONTAINER_LOCAL_DIRS = Attribute + .attribute(List.class, "container_local_dirs"); + public static final Attribute CONTAINER_LOG_DIRS = Attribute.attribute( + List.class, "container_log_dirs"); public static final Attribute RESOURCES_OPTIONS = Attribute.attribute( String.class, "resources_options"); public static final Attribute TC_COMMAND_FILE = Attribute.attribute( http://git-wip-us.apache.org/repos/asf/hadoop/blob/b41a7e89/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/executor/ContainerStartContext.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/executor/ContainerStartContext.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/executor/ContainerStartContext.java index ffcc519..b14a520 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/executor/ContainerStartContext.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/executor/ContainerStartContext.java @@ -45,6 +45,8 @@ public final class ContainerStartContext { private final Path containerWorkDir; private final List localDirs; private final List logDirs; + private final List containerLocalDirs; + private final List containerLogDirs; public static final class Builder { private Container container; @@ -56,6 +58,8 @@ public final class ContainerStartContext { private Path containerWorkDir; private List localDirs; private List logDirs; + private List containerLocalDirs; + private List containerLogDirs; public Builder() { } @@ -107,6 +111,16 @@ public final class ContainerStartContext { return this; } + public Builder setContainerLocalDirs(List containerLocalDirs) { + this.containerLocalDirs = containerLocalDirs; + return this; + } + + public Builder setContainerLogDirs(List containerLogDirs) { + this.containerLogDirs = containerLogDirs; + return this; + } + public ContainerStartContext build() { return new ContainerStartContext(this); } @@ -122,6 +136,8 @@ public final class ContainerStartContext { this.containerWorkDir = builder.containerWorkDir; this.localDirs = builder.localDirs; this.logDirs = builder.logDirs; + this.containerLocalDirs = builder.containerLocalDirs; + this.containerLogDirs = builder.containerLogDirs; } public Container getContainer() { @@ -163,4 +179,12 @@ public final class ContainerStartContext { public List getLogDirs() { return Collections.unmodifiableList(this.logDirs); } + + public List getContainerLocalDirs() { + return this.containerLocalDirs; + } + + public List getContainerLogDirs() { + return this.containerLogDirs; + } } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/b41a7e89/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java index 7fc0158..6898634 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java @@ -81,6 +81,8 @@ public class TestDockerContainerRuntime { Path pidFilePath; List localDirs; List logDirs; + List containerLocalDirs; + List containerLogDirs; String resourcesOptions; ContainerRuntimeContext.Builder builder; String submittingUser = "anakin"; @@ -123,9 +125,13 @@ public class TestDockerContainerRuntime { localDirs = new ArrayList<>(); logDirs = new ArrayList<>(); resourcesOptions = "cgroups=none"; + containerLocalDirs = new ArrayList<>(); + containerLogDirs = new ArrayList<>(); localDirs.add("/test_local_dir"); logDirs.add("/test_log_dir"); + containerLocalDirs.add("/test_container_local_dir"); + containerLogDirs.add("/test_container_log_dir"); builder = new ContainerRuntimeContext .Builder(container); @@ -141,6 +147,8 @@ public class TestDockerContainerRuntime { .setExecutionAttribute(PID_FILE_PATH, pidFilePath) .setExecutionAttribute(LOCAL_DIRS, localDirs) .setExecutionAttribute(LOG_DIRS, logDirs) + .setExecutionAttribute(CONTAINER_LOCAL_DIRS, containerLocalDirs) + .setExecutionAttribute(CONTAINER_LOG_DIRS, containerLogDirs) .setExecutionAttribute(RESOURCES_OPTIONS, resourcesOptions); } @@ -245,8 +253,8 @@ public class TestDockerContainerRuntime { .append("bash %8$s/launch_container.sh"); String expectedCommand = String.format(expectedCommandTemplate.toString(), - containerId, runAsUser, containerWorkDir, localDirs.get(0), - containerWorkDir, logDirs.get(0), image, containerWorkDir); + containerId, runAsUser, containerWorkDir, containerLocalDirs.get(0), + containerWorkDir, containerLogDirs.get(0), image, containerWorkDir); List dockerCommands = Files.readAllLines(Paths.get (dockerCommandFile), Charset.forName("UTF-8"));